From 9d9e754ae95c76f9cf6c7f617f15ed80fc6aeb03 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Fri, 23 Oct 2026 03:46:16 -0700 Subject: [PATCH 001/111] fix(resolver): link Go imports of a tracked module across repos The go.mod module-path bridge linked Go calls between tracked repositories, but file-to-package import edges into a tracked module stayed on dep:: terminals, even after a full warmup. The resolver's own comments say tracked sibling imports resolve through the file graph; the dep fallback is for modules with no matching files. The fallback also collapses subpackage paths to the module ID. Add an import-only bridge. It reads the retained import statement nodes for the exact package path, picks the provider package with the same module and workspace rules as the call bridge, and points the import edge at the provider's package file with the origin and confidence of other resolved cross-repo imports. Dependency contract nodes and their consumes edges are unchanged. It runs on full and mutation-frontier passes and reuses the module revision retry state, and it handles consumers whose go.mod is in a subdirectory. Re-indexing a provider file deletes incoming import edges (existing behavior in SQLite file eviction). The bridge records which consumer import statements point into each provider and re-creates those edges when the provider is re-indexed. After a restart the record is rebuilt once, lazily, by filtering import node IDs by path before loading any file; later edits do not scan. Package identities are cached per module revision. Repository sweeps filter by import path before hydrating files and release the resolve lock between batches. Tests cover both track orders, IndexAll, consumer edits and re-index, subdirectory go.mod consumers, provider file and repository re-index, an unchanged warm restart followed by a provider edit, duplicate paths and counts, and lock handling under panic. They fail before this change on memory and SQLite stores. --- internal/indexer/go_module_import_test.go | 134 ++++++++ internal/resolver/cross_repo_incremental.go | 26 +- internal/resolver/go_module_imports.go | 331 ++++++++++++++++++++ internal/resolver/go_module_imports_test.go | 304 ++++++++++++++++++ internal/resolver/go_modules.go | 2 +- internal/resolver/go_modules_cross_repo.go | 88 ++++-- 6 files changed, 865 insertions(+), 20 deletions(-) create mode 100644 internal/indexer/go_module_import_test.go create mode 100644 internal/resolver/go_module_imports.go create mode 100644 internal/resolver/go_module_imports_test.go diff --git a/internal/indexer/go_module_import_test.go b/internal/indexer/go_module_import_test.go new file mode 100644 index 000000000..33fe69f70 --- /dev/null +++ b/internal/indexer/go_module_import_test.go @@ -0,0 +1,134 @@ +package indexer + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/search" +) + +func TestMultiIndexer_GoModuleImport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"track", "reverse", "all", "sub_track", "sub_reverse"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + consumerRoot, provider := t.TempDir(), t.TempDir() + consumer, consumerFile := consumerRoot, "main.go" + if mode == "sub_track" || mode == "sub_reverse" { + consumer = filepath.Join(consumerRoot, "svc") + consumerFile = "svc/main.go" + require.NoError(t, os.MkdirAll(consumer, 0o755)) + } + writeFile(t, filepath.Join(consumer, "go.mod"), "module example.com/consumer\nrequire example.com/lib v1.0.0\n") + writeFile(t, filepath.Join(consumer, "main.go"), `package main +import "example.com/lib" +func main() { lib.F() } +`) + writeFile(t, filepath.Join(provider, "go.mod"), "module example.com/lib\n") + writeFile(t, filepath.Join(provider, "lib.go"), "package lib\nfunc F() {}\n") + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + cm := newTestConfigManager(t) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + entries := []config.RepoEntry{ + {Path: consumerRoot, Name: "consumer", Workspace: "shared"}, + {Path: provider, Name: "provider", Workspace: "shared"}, + } + if mode == "all" { + cm.Global().Repos = entries + _, err := mi.IndexAll() + require.NoError(t, err) + } else { + if mode == "reverse" || mode == "sub_reverse" { + entries[0], entries[1] = entries[1], entries[0] + } + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + } + mi.RunGlobalResolve() + var imports []*graph.Edge + for _, e := range g.GetOutEdges("consumer/" + consumerFile) { + if e.Kind == graph.EdgeImports { + imports = append(imports, e) + } + } + require.Len(t, imports, 1) + t.Logf("import target after track + global resolve: %s", imports[0].To) + assert.Equal(t, "provider/lib.go", imports[0].To) + assert.True(t, imports[0].CrossRepo) + contractID := "dep::example.com/lib" + if mode == "all" { + contractID = "dep::provider::lib" + } + if consumerFile == "main.go" { + require.NotNil(t, g.GetNode(contractID), "keep the go.mod dependency contract") + } + var calls []string + for _, e := range g.GetOutEdges("consumer/" + consumerFile + "::main") { + if e.Kind == graph.EdgeCalls { + calls = append(calls, e.To) + } + } + assert.Equal(t, []string{"provider/lib.go::F"}, calls) + for _, mode := range []string{"file", "repo"} { + writeFile(t, filepath.Join(consumer, "main.go"), "// edited\npackage main\nimport \"example.com/lib\"\nfunc main() { lib.F() }\n") + switch mode { + case "file": + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "main.go"))) + case "repo": + _, err := mi.IndexRepo("consumer") + require.NoError(t, err) + } + var targets []string + for _, e := range g.GetOutEdges("consumer/" + consumerFile) { + if e.Kind == graph.EdgeImports { + targets = append(targets, e.To) + assert.True(t, e.CrossRepo) + } + } + assert.Equal(t, []string{"provider/lib.go"}, targets, mode) + mi.RunGlobalResolve() + var afterGlobal []string + for _, e := range g.GetOutEdges("consumer/" + consumerFile) { + if e.Kind == graph.EdgeImports { + afterGlobal = append(afterGlobal, e.To) + } + } + assert.Equal(t, []string{"provider/lib.go"}, afterGlobal, mode+" global") + } + writeFile(t, filepath.Join(provider, "lib.go"), "package lib\nfunc F() { /* edited */ }\n") + require.NoError(t, mi.GetIndexer("provider").IndexFile(filepath.Join(provider, "lib.go"))) + for _, phase := range []string{"provider_file", "provider_repo"} { + if phase == "provider_repo" { + _, err := mi.IndexRepo("provider") + require.NoError(t, err) + } + var targets []string + for _, e := range g.GetOutEdges("consumer/" + consumerFile) { + if e.Kind == graph.EdgeImports { + targets = append(targets, e.To) + } + } + assert.Equal(t, []string{"provider/lib.go"}, targets, phase) + } + }) + } + } +} diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 8727395c7..f9e9e9032 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -67,6 +67,15 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed cr.mu.Lock() defer cr.mu.Unlock() + if len(definitionFiles) > 0 && len(cr.goModules) > 0 { + repos := make(map[string]bool) + for _, file := range definitionFiles { + if repo := graph.RepoPrefixOfID(file); cr.goModules[repo] != "" && !repos[repo] { + repos[repo] = true + cr.restoreGoModuleImports(repo, stats) + } + } + } if len(resolutionFiles) > 0 { frontier := collectIncrementalFileFrontier(cr.graph, resolutionFiles, nil) admission = CrossRepoIncomingAdmission{ @@ -100,8 +109,23 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed pending = append(pending, edge) } if len(pending) > 0 { - stats = cr.resolveScopedLocked(pending) + resolved := cr.resolveScopedLocked(pending) + stats.Resolved += resolved.Resolved + stats.Unresolved += resolved.Unresolved + stats.CrossRepoEdges += resolved.CrossRepoEdges + for repo, count := range resolved.ByRepo { + stats.ByRepo[repo] += count + } + } + // Preserve ordinary local-file resolution, but count each import only + // once if the module bridge subsequently lifts its dependency fallback. + counted := make(map[graph.EdgeIdentity]bool) + for _, e := range pending { + if e.Kind == graph.EdgeImports { + counted[graph.EdgeIdentityFor(e)] = !graph.IsUnresolvedTarget(e.To) + } } + _ = cr.resolveGoModuleImports(context.Background(), resolutionFiles, stats, counted) } if len(definitionFiles) > 0 && len(cr.goModules) > 0 && !admission.Refused { budget := &graph.IncomingSourceBudget{} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go new file mode 100644 index 000000000..d8738e4b4 --- /dev/null +++ b/internal/resolver/go_module_imports.go @@ -0,0 +1,331 @@ +package resolver + +import ( + "context" + "iter" + "maps" + "strings" + + "github.com/zzet/gortex/internal/graph" +) + +// resolveGoModuleImports preserves dependency contracts while lifting Go source +// imports onto a tracked package's representative file in a file frontier. +// counted records whether the ordinary frontier pass already counted an import +// as resolved (true) or unresolved (false), before this pass retargets it. +func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files []string, stats *CrossRepoStats, counted map[graph.EdgeIdentity]bool) error { + if len(cr.goModules) == 0 || len(files) == 0 { + return nil + } + flush := func(paths []string) { + nodes := cr.graph.GetFileNodesByPaths(paths) + outgoing := cr.graph.GetOutEdgesByNodeIDs(paths) + var batch []graph.EdgeReindex + for _, path := range paths { + cr.rememberGoModuleImports(nodes[path]) + for _, e := range outgoing[path] { + if e.Kind != graph.EdgeImports { + continue + } + importPath := goModuleImportPath(e, nodes[path]) + if importPath == "" || !trackedGoImport(cr.goModules, importPath) { + continue + } + source := cr.graph.GetNode(e.From) + caller := "" + if source != nil { + caller = source.RepoPrefix + } + match := cr.goModuleImportCandidate(importPath, caller, candidateWorkspaceID(source)) + if match == nil || match.ID == e.To { + continue + } + copy := *e + copy.Meta = maps.Clone(e.Meta) + copy.To = match.ID + copy.CrossRepo = caller != "" && match.RepoPrefix != caller + clearEdgeTerminal(©) + batch = append(batch, graph.EdgeReindex{Edge: ©, OldTo: e.To, RefreshIdentity: true, OldFilePath: e.FilePath, OldLine: e.Line}) + if len(batch) == externalCallMutationChunk { + cr.applyGoModuleImportReindexes(batch, stats, counted) + batch = nil + } + } + } + cr.applyGoModuleImportReindexes(batch, stats, counted) + } + unique := make(map[string]bool) + var goFiles []string + for _, path := range files { + if strings.HasSuffix(path, ".go") && !unique[path] { + unique[path] = true + goFiles = append(goFiles, path) + } + } + files = goFiles + for start := 0; start < len(files); start += externalCallMutationChunk { + if err := ctx.Err(); err != nil { + return err + } + flush(files[start:min(start+externalCallMutationChunk, len(files))]) + } + return ctx.Err() +} + +// A dep contract has lost the package suffix. Recover it from the retained +// import statement, but never guess when several same-line imports fit. +func goModuleImportPath(e *graph.Edge, nodes []*graph.Node) string { + target := graph.UnresolvedName(e.To) + if target == "" { + target = e.To + } + target = strings.TrimPrefix(target, externalCallPrefix) + if path, ok := strings.CutPrefix(target, "import::"); ok { + return path + } + if path, ok := strings.CutPrefix(target, "external::"); ok { + return path + } + module, dep := strings.CutPrefix(target, "dep::") + if !dep || strings.Contains(module, "::") { + return "" + } + path := "" + for _, n := range nodes { + if n.Kind != graph.KindImport || n.Language != "go" || n.StartLine != e.Line { + continue + } + candidate, _ := n.Meta["path"].(string) + if candidate != module && !strings.HasPrefix(candidate, module+"/") { + continue + } + if path != "" && path != candidate { + return "" + } + path = candidate + } + return path +} + +// Package identities are cached across resolver objects by module revision. +// A provider mutation invalidates only that provider's projection. +func (cr *CrossRepoResolver) goModuleImportCandidate(importPath, caller, workspace string) *graph.Node { + state := cr.goModuleImportState() + var candidates []*graph.Node + for repo, module := range cr.goModules { + if module == "" || (importPath != module && !strings.HasPrefix(importPath, module+"/")) { + continue + } + packages, cached := state.packages[repo] + if !cached { + packages = make(map[string]graph.FileNodeIdentity) + for file := range graph.FileNodeIdentitiesSeq(cr.graph, []string{repo}) { + if !strings.HasSuffix(file.FilePath, ".go") || strings.HasSuffix(file.FilePath, "_test.go") { + continue + } + dir := filePathDir(file.FilePath) + if prior, ok := packages[dir]; !ok || file.ID < prior.ID { + packages[dir] = file + } + } + state.packages[repo] = packages + } + if file, ok := packages[repo+strings.TrimPrefix(importPath, module)]; ok { + candidates = append(candidates, &graph.Node{ID: file.ID, Kind: graph.KindFile, FilePath: file.FilePath, RepoPrefix: file.RepoPrefix, WorkspaceID: file.WorkspaceID, Language: "go"}) + } + } + return goModuleCandidate(cr.goModules, importPath, candidates, caller, workspace, cr.crossWorkspaceLookup) +} + +func (cr *CrossRepoResolver) applyGoModuleImportReindexes(batch []graph.EdgeReindex, stats *CrossRepoStats, counted map[graph.EdgeIdentity]bool) { + if !cr.applyGoModuleReindexes(batch) { + originals := make(map[graph.EdgeIdentity]*graph.Edge) + for _, row := range batch { + original := *row.Edge + original.To = row.OldTo + original.Meta = maps.Clone(row.Edge.Meta) + originals[graph.EdgeIdentityFor(&original)] = &original + } + cr.queueGoModuleEdges(originals) + cr.logger.Warn("Go module import batch dropped; queued for a later module pass") + return + } + if stats == nil { + return + } + for _, row := range batch { + original := graph.EdgeIdentityFor(row.Edge) + original.To = row.OldTo + resolved, alreadyCounted := counted[original] + if !alreadyCounted || !resolved { + stats.Resolved++ + } + if alreadyCounted && !resolved { + stats.Unresolved-- + } + if row.Edge.CrossRepo { + stats.CrossRepoEdges++ + if target := cr.graph.GetNode(row.Edge.To); target != nil { + stats.ByRepo[target.RepoPrefix]++ + } + } + } +} + +// Filter identity-only rows before loading any file payload or adjacency. +func (cr *CrossRepoResolver) goModuleImportFileBatches(repos []string) iter.Seq[[]string] { + return func(yield func([]string) bool) { + seen := make(map[string]bool) + var files []string + for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, graph.KindImport) { + file, path, ok := strings.Cut(n.ID, "::import::") + if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModules, path) || seen[file] { + continue + } + seen[file] = true + files = append(files, file) + if len(files) == externalCallMutationChunk { + if !yield(files) { + return + } + files = nil + } + } + if len(files) > 0 { + yield(files) + } + } +} + +// Import indexes share the store's resolve mutex, not the retry queue mutex. +// Retain statement IDs, not edges: consumer edits and removals are revalidated. +func (cr *CrossRepoResolver) goModuleImportState() *GoModuleDepRetry { + if cr.goModuleRetry == nil { + cr.goModuleRetry = &GoModuleDepRetry{} + } + state := cr.goModuleRetry + if state.packageRevision != cr.goModuleRevision || !maps.Equal(state.packageModules, cr.goModules) { + state.packageModules = maps.Clone(cr.goModules) + state.packageRevision = cr.goModuleRevision + state.packages = nil + state.imports = nil + state.importsSeeded = false + } + if state.packages == nil { + state.packages = make(map[string]map[string]graph.FileNodeIdentity) + } + if state.imports == nil { + state.imports = make(map[string]map[string]struct{}) + } + return state +} + +// An unchanged warm restart skips global resolution. Seed once, lazily, if +// the first operation is a provider edit; reuse the same seed in legacy retry. +// Callers hold the resolve mutex. Each candidate batch releases it afterward. +func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, stats *CrossRepoStats) error { + state := cr.goModuleImportState() + if state.importsSeeded { + return nil + } + err := func() error { + cr.mu.Unlock() + // Reacquire on panic too, before the caller's deferred Unlock runs. + defer cr.mu.Lock() + for files := range cr.goModuleImportFileBatches(nil) { + if err := ctx.Err(); err != nil { + return err + } + func() { + cr.mu.Lock() + defer cr.mu.Unlock() + _ = cr.resolveGoModuleImports(ctx, files, stats, nil) + }() + } + return ctx.Err() + }() + if err == nil && state.packageRevision == cr.goModuleRevision && maps.Equal(state.packageModules, cr.goModules) { + state.importsSeeded = true + } + return err +} + +func (cr *CrossRepoResolver) rememberGoModuleImports(nodes []*graph.Node) { + state := cr.goModuleImportState() + for _, n := range nodes { + if n.Kind != graph.KindImport || n.Language != "go" { + continue + } + _, path, ok := strings.Cut(n.ID, "::import::") + if !ok { + continue + } + for repo, module := range cr.goModules { + if module == "" || (path != module && !strings.HasPrefix(path, module+"/")) { + continue + } + if state.imports[repo] == nil { + state.imports[repo] = make(map[string]struct{}) + } + state.imports[repo][n.ID] = struct{}{} + } + } +} + +// Only retained statements registered for this provider are probed on edits. +// No global node scan is needed to reconstruct imports deleted by eviction. +func (cr *CrossRepoResolver) goModuleImportIDs(repo string) []string { + state := cr.goModuleImportState() + delete(state.packages, repo) + _ = cr.resolveGoModuleImportSweep(context.Background(), nil) + var ids []string + for id := range state.imports[repo] { + ids = append(ids, id) + } + return ids +} + +func (cr *CrossRepoResolver) restoreGoModuleImports(repo string, stats *CrossRepoStats) { + ids := cr.goModuleImportIDs(repo) + for start := 0; start < len(ids); start += externalCallMutationChunk { + cr.restoreGoModuleImportBatch(repo, ids[start:min(start+externalCallMutationChunk, len(ids))], stats) + } +} + +func (cr *CrossRepoResolver) restoreGoModuleImportBatch(repo string, ids []string, stats *CrossRepoStats) { + state := cr.goModuleImportState() + nodes := cr.graph.GetNodesByIDs(ids) + paths := make(map[string]bool) + var files []string + for _, n := range nodes { + if n != nil && n.Kind == graph.KindImport && n.Language == "go" && !paths[n.FilePath] { + paths[n.FilePath] = true + files = append(files, n.FilePath) + } + } + outgoing := cr.graph.GetOutEdgesByNodeIDs(files) + for _, id := range ids { + n := nodes[id] + if n == nil || n.Kind != graph.KindImport || n.Language != "go" { + delete(state.imports[repo], id) + continue + } + _, path, ok := strings.Cut(n.ID, "::import::") + module := cr.goModules[repo] + if !ok || module == "" || (path != module && !strings.HasPrefix(path, module+"/")) { + delete(state.imports[repo], id) + continue + } + present := false + for _, e := range outgoing[n.FilePath] { + if e.Kind == graph.EdgeImports && e.Line == n.StartLine { + present = true + break + } + } + if !present { + cr.graph.AddEdge(&graph.Edge{From: n.FilePath, To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: n.FilePath, Line: n.StartLine}) + } + } + _ = cr.resolveGoModuleImports(context.Background(), files, stats, nil) +} diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go new file mode 100644 index 000000000..2802c02d8 --- /dev/null +++ b/internal/resolver/go_module_imports_test.go @@ -0,0 +1,304 @@ +package resolver + +import ( + "context" + "fmt" + "iter" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestGoModuleImportRetry(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, tc := range []struct { + name, targetWorkspace, declared, want string + modules map[string]string + }{ + {"sub_package", "shared", "", "b/sub/a.go", map[string]string{"b": "example.com/lib"}}, + {"denied_workspace", "other", "", "dep::example.com/lib", map[string]string{"b": "example.com/lib"}}, + {"declared_workspace", "other", "example.com/lib", "b/sub/a.go", map[string]string{"b": "example.com/lib"}}, + {"wrong_declaration", "other", "example.com/lib2", "dep::example.com/lib", map[string]string{"b": "example.com/lib"}}, + {"wrong_module", "shared", "", "dep::example.com/lib", map[string]string{"b": "example.com/lib2"}}, + {"duplicate", "shared", "", "dep::example.com/lib", map[string]string{"b": "example.com/lib", "fork": "example.com/lib"}}, + {"longest", "shared", "", "child/a.go", map[string]string{"b": "example.com/lib", "child": "example.com/lib/sub"}}, + } { + t.Run(backend+"/"+tc.name, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", WorkspaceID: "shared", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib/sub", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib/sub"}}) + addDepNode(t, g, "a", "example.com/lib") + g.AddEdge(&graph.Edge{From: "a/go.mod", To: "dep::example.com/lib", Kind: graph.EdgeConsumes, FilePath: "a/go.mod"}) + for repo := range tc.modules { + dir := repo + "/sub" + if repo == "child" { + dir = repo + } + for _, name := range []string{"z.go", "a.go", "0_test.go"} { + file := dir + "/" + name + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: repo, WorkspaceID: tc.targetWorkspace, Language: "go"}) + } + } + g.AddEdge(&graph.Edge{From: "a/main.go", To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + state := &GoModuleDepRetry{} + resolve := func() { + cr := NewCrossRepo(g) + cr.SetGoModules(tc.modules) + cr.SetGoModuleDepRetry(state, 1) + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { + if tc.declared == "" { + return nil + } + return []CrossWorkspaceDepRule{{Workspace: "other", Modules: []string{tc.declared}}} + }) + require.NoError(t, cr.RetryGoModuleDeps(context.Background())) + } + resolve() + resolve() + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + assert.Equal(t, tc.want, edges[0].To) + assert.Equal(t, tc.want != "dep::example.com/lib", edges[0].CrossRepo) + if edges[0].CrossRepo { + assert.Empty(t, edges[0].Origin) + assert.Zero(t, edges[0].Confidence) + } + require.NotNil(t, g.GetNode("dep::example.com/lib")) + contracts := g.GetOutEdges("a/go.mod") + require.Len(t, contracts, 1) + assert.Equal(t, "dep::example.com/lib", contracts[0].To) + assert.Equal(t, graph.EdgeConsumes, contracts[0].Kind) + }) + } + } +} + +func TestGoModuleImportPathAmbiguousLine(t *testing.T) { + edge := &graph.Edge{To: "dep::example.com/lib", Kind: graph.EdgeImports, Line: 2} + nodes := []*graph.Node{ + {Kind: graph.KindImport, Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib/a"}}, + {Kind: graph.KindImport, Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib/b"}}, + } + assert.Empty(t, goModuleImportPath(edge, nodes), "a legacy module-only target is not enough evidence for either package") +} + +func TestGoModuleImportDroppedBatch(t *testing.T) { + g := &goModuleDroppingStore{Store: graph.New(), drop: true} + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib"}}) + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + state := &GoModuleDepRetry{} + for pass := range 2 { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + cr.SetGoModuleDepRetry(state, 1) + require.NoError(t, cr.RetryGoModuleDeps(context.Background())) + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + if pass == 0 { + assert.Equal(t, "dep::example.com/lib", edges[0].To) + assert.Len(t, state.pending, 1) + } else { + assert.Equal(t, "b/lib.go", edges[0].To) + assert.Empty(t, state.pending) + } + } +} + +// Count the expensive projections, not elapsed time. +type goImportCountingStore struct { + graph.Store + fileScans, hydrated int + importScans, importRows int + afterBatch func() +} + +func (s *goImportCountingStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { + s.fileScans++ + return graph.FileNodeIdentitiesSeq(s.Store, repos) +} +func (s *goImportCountingStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { + return func(yield func(graph.RepoNodeIdentity) bool) { + imports := len(kinds) == 1 && kinds[0] == graph.KindImport && repos == nil + if imports { + s.importScans++ + } + visited := 0 + for row := range graph.RepoNodeIdentitiesSeq(s.Store, repos, kinds...) { + if imports { + s.importRows++ + } + if !yield(row) { + return + } + visited++ + if visited == externalCallMutationChunk && s.afterBatch != nil { + s.afterBatch() + } + } + } +} +func (s *goImportCountingStore) GetFileNodesByPaths(paths []string) map[string][]*graph.Node { + s.hydrated += len(paths) + return s.Store.GetFileNodesByPaths(paths) +} +func TestGoModuleImportLookupBudget(t *testing.T) { + for _, mode := range []string{"cache", "filter", "dedup", "frontier_stats"} { + t.Run(mode, func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib"}}) + e := &graph.Edge{From: "a/main.go", To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2} + g.AddEdge(e) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + switch mode { + case "cache": + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"a/main.go"}, nil, nil)) + g.RemoveEdge("a/main.go", "b/lib.go", graph.EdgeImports) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"a/main.go"}, nil, nil)) + assert.Equal(t, 1, g.fileScans) + case "filter": + g.RemoveEdge(e.From, e.To, e.Kind) + g.EvictFile("a/main.go") + g.AddNode(&graph.Node{ID: "a/unrelated.go::import::fmt", Kind: graph.KindImport, FilePath: "a/unrelated.go", RepoPrefix: "a", Language: "go"}) + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("b")) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + assert.Zero(t, g.hydrated) + case "dedup": + stats := &CrossRepoStats{ByRepo: map[string]int{}} + cr.resolveGoModuleEdgesLocked([]*graph.Edge{e, e}, stats) + assert.Equal(t, 1, stats.Resolved) + assert.Equal(t, 1, stats.CrossRepoEdges) + case "frontier_stats": + g.RemoveEdge(e.From, e.To, e.Kind) + addDepNode(t, g, "a", "example.com/lib") + e.To = "unresolved::import::example.com/lib" + g.AddEdge(e) + stats := cr.ResolveMutationFrontiers([]string{"a/main.go"}, nil, nil) + assert.Equal(t, 1, stats.Resolved) + assert.Equal(t, 1, stats.CrossRepoEdges) + } + }) + } +} + +func TestGoModuleImportWarmProviderEdit(t *testing.T) { + for _, route := range []string{"mutation", "post_publish"} { + t.Run(route, func(t *testing.T) { + db := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := store_sqlite.Open(db) + require.NoError(t, err) + s.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + s.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib"}}) + s.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + // Persist an already-bound graph, with no retry state carried over. + s.AddEdge(&graph.Edge{From: "a/main.go", To: "b/lib.go", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2, CrossRepo: true}) + require.NoError(t, s.Close()) + s, err = store_sqlite.Open(db) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g := &goImportCountingStore{Store: s} + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + // The unchanged warm-restart fast path skips global resolution. + // Include unrelated imports to expose the initial scan's exact cost. + for _, path := range []string{"fmt", "strings", "example.com/untracked"} { + s.AddNode(&graph.Node{ID: "a/other.go::import::" + path, Kind: graph.KindImport, FilePath: "a/other.go", RepoPrefix: "a", Language: "go"}) + } + s.EvictFile("b/lib.go") + s.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + if route == "mutation" { + cr.ResolveMutationFrontiers(nil, nil, []string{"b/lib.go"}) + } else { + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("b")) + } + edges := s.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + assert.Equal(t, "b/lib.go", edges[0].To) + assert.True(t, edges[0].CrossRepo) + assert.Equal(t, 1, g.importScans) + assert.Equal(t, 4, g.importRows, "one tracked and three unrelated import identities") + t.Logf("lazy seed: global scans=%d, import identities visited=%d", g.importScans, g.importRows) + s.EvictFile("b/lib.go") + s.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + if route == "mutation" { + cr.ResolveMutationFrontiers(nil, nil, []string{"b/lib.go"}) + } else { + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("b")) + } + require.Len(t, s.GetOutEdges("a/main.go"), 1) + assert.Equal(t, 1, g.importScans, "later edits must not repeat the global identity scan") + assert.Equal(t, 4, g.importRows) + }) + } +} + +type goImportPanicStore struct{ graph.Store } + +func (s *goImportPanicStore) GetFileNodesByPaths([]string) map[string][]*graph.Node { + panic("import-store-panic") +} + +func TestGoModuleImportPanicUnlock(t *testing.T) { + if os.Getenv("GORTEX_TEST_IMPORT_PANIC") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=^TestGoModuleImportPanicUnlock$", "-test.v") + cmd.Env = append(os.Environ(), "GORTEX_TEST_IMPORT_PANIC=1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", output) + return + } + g := &goImportPanicStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + assert.PanicsWithValue(t, "import-store-panic", func() { _ = cr.RetryGoModuleDeps(t.Context()) }) + require.True(t, cr.mu.TryLock(), "the store resolve mutex must be released after panic") + cr.mu.Unlock() + require.True(t, cr.goModuleRetry.mu.TryLock(), "the retry queue mutex must remain usable") + cr.goModuleRetry.mu.Unlock() +} + +func TestGoModuleImportBatchUnlock(t *testing.T) { + for _, route := range []string{"retry", "post_publish"} { + t.Run(route, func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + for i := 0; i <= externalCallMutationChunk; i++ { + file := fmt.Sprintf("a/f%d.go", i) + g.AddNode(&graph.Node{ID: file + "::import::example.com/lib", Kind: graph.KindImport, FilePath: file, RepoPrefix: "a", Language: "go"}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"a": "example.com/consumer", "b": "example.com/lib"}) + probes := 0 + g.afterBatch = func() { + probes++ + available := cr.mu.TryLock() + assert.True(t, available, "the next identity page must not hold the resolve mutex") + if available { + cr.mu.Unlock() + } + } + if route == "retry" { + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + } else { + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("a")) + } + assert.Positive(t, probes) + }) + } +} diff --git a/internal/resolver/go_modules.go b/internal/resolver/go_modules.go index f34424afd..37ded3b3a 100644 --- a/internal/resolver/go_modules.go +++ b/internal/resolver/go_modules.go @@ -158,7 +158,7 @@ func goModuleCandidate(modules map[string]string, importPath string, candidates ambiguous := false priority := -1 for _, c := range candidates { - if (c.Language != "" && c.Language != "go") || (c.Kind != graph.KindFunction && c.Kind != graph.KindType && c.Kind != graph.KindInterface) { + if (c.Language != "" && c.Language != "go") || (c.Kind != graph.KindFunction && c.Kind != graph.KindType && c.Kind != graph.KindInterface && c.Kind != graph.KindFile) { continue // a package selector does not name a receiver method } module := modules[c.RepoPrefix] diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index a9955d8b0..e5f9d3838 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -52,6 +52,12 @@ type GoModuleDepRetry struct { revision uint64 complete bool pending map[graph.EdgeIdentity]*graph.Edge + // These compact import indexes are protected by the store resolve mutex. + packageModules map[string]string + packageRevision uint64 + packages map[string]map[string]graph.FileNodeIdentity + imports map[string]map[string]struct{} + importsSeeded bool } func (cr *CrossRepoResolver) SetGoModuleDepRetry(state *GoModuleDepRetry, revision uint64) { @@ -107,15 +113,21 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross } cr.resolveGoModuleEdgesLocked(deferred, stats) state.mu.Lock() - defer state.mu.Unlock() if len(cr.goModules) == 0 { state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, true state.replacements = maps.Clone(cr.goModuleReplacements) + state.mu.Unlock() return nil } - if state.complete && state.revision == cr.goModuleRevision && maps.Equal(state.modules, cr.goModules) && !changedReplacements { + complete := state.complete && state.revision == cr.goModuleRevision && maps.Equal(state.modules, cr.goModules) && !changedReplacements + state.mu.Unlock() + if complete { return nil } + err := cr.resolveGoModuleImportSweep(ctx, stats) + if err != nil { + return err + } var batch []graph.EdgeReindex originals := make(map[graph.EdgeIdentity]*graph.Edge) names := make(map[string][]*graph.Node) @@ -126,10 +138,7 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross if len(batch) > 0 { if !cr.applyGoModuleReindexes(batch) { cr.logger.Warn("Go module terminal batch dropped; queued for a later module pass") - if state.pending == nil { - state.pending = make(map[graph.EdgeIdentity]*graph.Edge) - } - maps.Copy(state.pending, originals) + cr.queueGoModuleEdges(originals) } batch = nil clear(originals) @@ -212,6 +221,8 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross if err := ctx.Err(); err != nil { return err } + state.mu.Lock() + defer state.mu.Unlock() state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, true state.replacements = maps.Clone(cr.goModuleReplacements) return nil @@ -395,6 +406,17 @@ func goModuleEdgeSpec(e *graph.Edge) string { } func (cr *CrossRepoResolver) resolveGoModuleEdgesLocked(edges []*graph.Edge, stats *CrossRepoStats) { + var importFiles []string + seenImports := make(map[string]bool) + for _, e := range edges { + if e != nil && e.Kind == graph.EdgeImports && !seenImports[e.FilePath] { + seenImports[e.FilePath] = true + importFiles = append(importFiles, e.FilePath) + } + } + if len(importFiles) > 0 { + _ = cr.resolveGoModuleImports(context.Background(), importFiles, stats, nil) + } var batch []graph.EdgeReindex names := make(map[[2]string][]*graph.Node) sources := make(map[string]*graph.Node) @@ -459,15 +481,7 @@ func (cr *CrossRepoResolver) resolveGoModuleEdgesLocked(edges []*graph.Edge, sta batch = cr.filterLiveReindex(batch) if !cr.applyGoModuleReindexes(batch) { cr.logger.Warn("Go module rebind batch dropped; queued for a later module pass") - if cr.goModuleRetry == nil { - cr.goModuleRetry = &GoModuleDepRetry{} - } - cr.goModuleRetry.mu.Lock() - if cr.goModuleRetry.pending == nil { - cr.goModuleRetry.pending = make(map[graph.EdgeIdentity]*graph.Edge) - } - maps.Copy(cr.goModuleRetry.pending, originals) - cr.goModuleRetry.mu.Unlock() + cr.queueGoModuleEdges(originals) return } if stats == nil { @@ -493,10 +507,35 @@ func (cr *CrossRepoResolver) RetryGoModuleDeps(ctx context.Context) error { return cr.retryGoModuleDeps(ctx, &CrossRepoStats{ByRepo: make(map[string]int)}) } -// ResolveGoModuleIncomingForRepo probes exact targets for the provider's definitions. -// Reindexing has no changed-symbol receipt, so stream only its definition IDs and -// hydrate at most one batch; never load the repository's nodes or edges as a set. +// ResolveGoModuleIncomingForRepo probes the provider's definitions and retained +// consumer import statements. Stream identities and hydrate at most one batch; +// never load the repository's nodes or edges as a set. func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { + if len(cr.goModules) == 0 { + return nil + } + if cr.goModules[repo] != "" { + var importIDs []string + func() { + cr.mu.Lock() + defer cr.mu.Unlock() + importIDs = cr.goModuleImportIDs(repo) + }() + for start := 0; start < len(importIDs); start += externalCallMutationChunk { + func() { + cr.mu.Lock() + defer cr.mu.Unlock() + cr.restoreGoModuleImportBatch(repo, importIDs[start:min(start+externalCallMutationChunk, len(importIDs))], nil) + }() + } + } + for files := range cr.goModuleImportFileBatches([]string{repo}) { + func() { + cr.mu.Lock() + defer cr.mu.Unlock() + _ = cr.resolveGoModuleImports(context.Background(), files, nil, nil) + }() + } if cr.goModules[repo] == "" { return nil } @@ -867,3 +906,16 @@ func (state *GoModuleDepRetry) Invalidate() { defer state.mu.Unlock() state.complete = false } + +func (cr *CrossRepoResolver) queueGoModuleEdges(originals map[graph.EdgeIdentity]*graph.Edge) { + if cr.goModuleRetry == nil { + cr.goModuleRetry = &GoModuleDepRetry{} + } + state := cr.goModuleRetry + state.mu.Lock() + defer state.mu.Unlock() + if state.pending == nil { + state.pending = make(map[graph.EdgeIdentity]*graph.Edge) + } + maps.Copy(state.pending, originals) +} From 57481f11dda8986cbee279337866f586737b5854 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Fri, 23 Oct 2026 04:15:59 -0700 Subject: [PATCH 002/111] fix(resolver): bound Go import recovery to other tracked repos The import recovery recorded a repository's imports of its own module, so each save in a repository that imports its own packages reloaded nearly every file, even with one tracked repository. Record and restore only imports whose provider is a different tracked repository, and skip restore when fewer than two repositories are in the graph. The provider package cache was not cleared on the whole-graph fallback paths, so a consumer edit after a provider file was deleted could bind the import to the deleted file. Clear the cache on those paths and apply import rebinds through the same live-target check as call rebinds. An import lifted from an external:: fallback is now counted as resolved, not unresolved. The one-time lazy seed runs from the public entry points before the resolve lock is taken, so no lookup helper releases its caller's lock. New tests fail before this change. --- internal/resolver/cross_repo.go | 11 ++ internal/resolver/cross_repo_incremental.go | 10 +- internal/resolver/go_module_imports.go | 56 +++++----- internal/resolver/go_module_imports_test.go | 109 +++++++++++++++++++- internal/resolver/go_modules_cross_repo.go | 21 ++-- 5 files changed, 170 insertions(+), 37 deletions(-) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 4fe26a8da..8e18a075d 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -393,11 +393,22 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS if err := ctx.Err(); err != nil { return stats, err } + // Whole-graph fallback has no exact provider frontier. Discard package + // projections even if the module revision's terminal retry is complete. + cr.mu.Lock() + clear(cr.goModuleImportState().packages) + cr.mu.Unlock() + if err := cr.resolveGoModuleImportSweep(ctx, stats); err != nil { + return stats, err + } cr.mu.Lock() defer cr.mu.Unlock() if err := ctx.Err(); err != nil { return stats, err } + for repo := range cr.goModules { + cr.restoreGoModuleImports(repo, stats) + } if err := cr.retryGoModuleDeps(ctx, stats); err != nil { return stats, err } diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index f9e9e9032..df81243af 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -2,6 +2,7 @@ package resolver import ( "context" + "strings" "go.uber.org/zap" @@ -64,6 +65,13 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed return stats, admission } + // Provider recovery needs the lazy seed before the pass lock is taken. + // Ordinary consumer frontiers keep local resolution ahead of the bridge. + if len(definitionFiles) > 0 { + if err := cr.resolveGoModuleImportSweep(context.Background(), stats); err != nil { + cr.logger.Error("cross-repo resolve: seed Go module imports", zap.Error(err)) + } + } cr.mu.Lock() defer cr.mu.Unlock() @@ -122,7 +130,7 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed counted := make(map[graph.EdgeIdentity]bool) for _, e := range pending { if e.Kind == graph.EdgeImports { - counted[graph.EdgeIdentityFor(e)] = !graph.IsUnresolvedTarget(e.To) + counted[graph.EdgeIdentityFor(e)] = !graph.IsUnresolvedTarget(e.To) && !strings.HasPrefix(e.To, "external::") } } _ = cr.resolveGoModuleImports(context.Background(), resolutionFiles, stats, counted) diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index d8738e4b4..ada330f1d 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -138,6 +138,7 @@ func (cr *CrossRepoResolver) goModuleImportCandidate(importPath, caller, workspa } func (cr *CrossRepoResolver) applyGoModuleImportReindexes(batch []graph.EdgeReindex, stats *CrossRepoStats, counted map[graph.EdgeIdentity]bool) { + batch = cr.filterLiveReindex(batch) if !cr.applyGoModuleReindexes(batch) { originals := make(map[graph.EdgeIdentity]*graph.Edge) for _, row := range batch { @@ -182,6 +183,9 @@ func (cr *CrossRepoResolver) goModuleImportFileBatches(repos []string) iter.Seq[ if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModules, path) || seen[file] { continue } + if module := cr.goModules[n.RepoPrefix]; module != "" && (path == module || strings.HasPrefix(path, module+"/")) { + continue + } seen[file] = true files = append(files, file) if len(files) == externalCallMutationChunk { @@ -220,34 +224,34 @@ func (cr *CrossRepoResolver) goModuleImportState() *GoModuleDepRetry { return state } -// An unchanged warm restart skips global resolution. Seed once, lazily, if -// the first operation is a provider edit; reuse the same seed in legacy retry. -// Callers hold the resolve mutex. Each candidate batch releases it afterward. +// An unchanged warm restart skips global resolution. Public entry points seed +// once before taking the pass lock; each candidate batch takes its own lock. func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, stats *CrossRepoStats) error { + cr.mu.Lock() state := cr.goModuleImportState() - if state.importsSeeded { - return nil + seeded := state.importsSeeded + cr.mu.Unlock() + if seeded || len(cr.goModules) == 0 { + return ctx.Err() } - err := func() error { - cr.mu.Unlock() - // Reacquire on panic too, before the caller's deferred Unlock runs. - defer cr.mu.Lock() - for files := range cr.goModuleImportFileBatches(nil) { - if err := ctx.Err(); err != nil { - return err - } - func() { - cr.mu.Lock() - defer cr.mu.Unlock() - _ = cr.resolveGoModuleImports(ctx, files, stats, nil) - }() + for files := range cr.goModuleImportFileBatches(nil) { + if err := ctx.Err(); err != nil { + return err } - return ctx.Err() - }() - if err == nil && state.packageRevision == cr.goModuleRevision && maps.Equal(state.packageModules, cr.goModules) { + if err := func() error { + cr.mu.Lock() + defer cr.mu.Unlock() + return cr.resolveGoModuleImports(ctx, files, stats, nil) + }(); err != nil { + return err + } + } + cr.mu.Lock() + defer cr.mu.Unlock() + if ctx.Err() == nil && state.packageRevision == cr.goModuleRevision && maps.Equal(state.packageModules, cr.goModules) { state.importsSeeded = true } - return err + return ctx.Err() } func (cr *CrossRepoResolver) rememberGoModuleImports(nodes []*graph.Node) { @@ -261,7 +265,7 @@ func (cr *CrossRepoResolver) rememberGoModuleImports(nodes []*graph.Node) { continue } for repo, module := range cr.goModules { - if module == "" || (path != module && !strings.HasPrefix(path, module+"/")) { + if repo == n.RepoPrefix || module == "" || (path != module && !strings.HasPrefix(path, module+"/")) { continue } if state.imports[repo] == nil { @@ -277,7 +281,6 @@ func (cr *CrossRepoResolver) rememberGoModuleImports(nodes []*graph.Node) { func (cr *CrossRepoResolver) goModuleImportIDs(repo string) []string { state := cr.goModuleImportState() delete(state.packages, repo) - _ = cr.resolveGoModuleImportSweep(context.Background(), nil) var ids []string for id := range state.imports[repo] { ids = append(ids, id) @@ -286,6 +289,9 @@ func (cr *CrossRepoResolver) goModuleImportIDs(repo string) []string { } func (cr *CrossRepoResolver) restoreGoModuleImports(repo string, stats *CrossRepoStats) { + if len(cr.graph.RepoPrefixes()) < 2 { + return + } ids := cr.goModuleImportIDs(repo) for start := 0; start < len(ids); start += externalCallMutationChunk { cr.restoreGoModuleImportBatch(repo, ids[start:min(start+externalCallMutationChunk, len(ids))], stats) @@ -312,7 +318,7 @@ func (cr *CrossRepoResolver) restoreGoModuleImportBatch(repo string, ids []strin } _, path, ok := strings.Cut(n.ID, "::import::") module := cr.goModules[repo] - if !ok || module == "" || (path != module && !strings.HasPrefix(path, module+"/")) { + if n.RepoPrefix == repo || !ok || module == "" || (path != module && !strings.HasPrefix(path, module+"/")) { delete(state.imports[repo], id) continue } diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index 2802c02d8..3beb37620 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -122,6 +122,7 @@ func TestGoModuleImportDroppedBatch(t *testing.T) { type goImportCountingStore struct { graph.Store fileScans, hydrated int + edgeLoads int importScans, importRows int afterBatch func() } @@ -155,8 +156,13 @@ func (s *goImportCountingStore) GetFileNodesByPaths(paths []string) map[string][ s.hydrated += len(paths) return s.Store.GetFileNodesByPaths(paths) } +func (s *goImportCountingStore) GetOutEdgesByNodeIDs(ids []string) map[string][]*graph.Edge { + s.edgeLoads += len(ids) + return s.Store.GetOutEdgesByNodeIDs(ids) +} + func TestGoModuleImportLookupBudget(t *testing.T) { - for _, mode := range []string{"cache", "filter", "dedup", "frontier_stats"} { + for _, mode := range []string{"cache", "filter", "dedup", "frontier_stats", "external_stats", "deleted_target"} { t.Run(mode, func(t *testing.T) { g := &goImportCountingStore{Store: graph.New()} g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) @@ -185,14 +191,25 @@ func TestGoModuleImportLookupBudget(t *testing.T) { cr.resolveGoModuleEdgesLocked([]*graph.Edge{e, e}, stats) assert.Equal(t, 1, stats.Resolved) assert.Equal(t, 1, stats.CrossRepoEdges) - case "frontier_stats": + case "frontier_stats", "external_stats": g.RemoveEdge(e.From, e.To, e.Kind) - addDepNode(t, g, "a", "example.com/lib") + if mode == "frontier_stats" { + addDepNode(t, g, "a", "example.com/lib") + } e.To = "unresolved::import::example.com/lib" g.AddEdge(e) stats := cr.ResolveMutationFrontiers([]string{"a/main.go"}, nil, nil) assert.Equal(t, 1, stats.Resolved) assert.Equal(t, 1, stats.CrossRepoEdges) + assert.Zero(t, stats.Unresolved) + case "deleted_target": + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"a/main.go"}, nil, nil)) + g.EvictFile("b/lib.go") + g.AddEdge(&graph.Edge{From: e.From, To: "dep::example.com/lib", Kind: e.Kind, FilePath: e.FilePath, Line: e.Line}) + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"a/main.go"}, nil, nil)) + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + assert.Equal(t, "dep::example.com/lib", edges[0].To, "a stale cached representative must not become a dangling target") } }) } @@ -302,3 +319,89 @@ func TestGoModuleImportBatchUnlock(t *testing.T) { }) } } + +func TestGoModuleImportSameRepoSave(t *testing.T) { + for _, otherRepo := range []bool{false, true} { + t.Run(fmt.Sprint(otherRepo), func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + modules := map[string]string{"a": "example.com/self"} + if otherRepo { + modules["b"] = "example.com/other" + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + } + for i := range 4 { + file := fmt.Sprintf("a/f%d.go", i) + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: file + "::import::example.com/self", Kind: graph.KindImport, FilePath: file, RepoPrefix: "a", Language: "go", StartLine: 2}) + g.AddEdge(&graph.Edge{From: file, To: "a/f0.go", Kind: graph.EdgeImports, FilePath: file, Line: 2}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(modules) + for range 2 { // cold seed, then an ordinary same-repo save + g.hydrated, g.edgeLoads = 0, 0 + cr.ResolveMutationFrontiers(nil, nil, []string{"a/f0.go"}) + assert.Equal(t, 1, g.hydrated, "only the exact definition file may be hydrated") + assert.Zero(t, g.edgeLoads, "same-repo imports must not trigger restore adjacency loads") + assert.Empty(t, cr.goModuleRetry.imports["a"]) + } + }) + } +} + +func TestGoModuleImportFallbackProviderDeletion(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2}) + for _, file := range []string{"b/a.go", "b/z.go"} { + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "b", Language: "go"}) + } + resetImport := func() { + for _, e := range g.GetOutEdges("a/main.go") { + g.RemoveEdge(e.From, e.To, e.Kind) + } + g.AddEdge(&graph.Edge{From: "a/main.go", To: "unresolved::import::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + resetImport() + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + require.Equal(t, "b/a.go", g.GetOutEdges("a/main.go")[0].To) + g.EvictFile("b/a.go") + _, err := cr.ResolveAllContext(t.Context()) // incomplete receipt and watcher fallback + require.NoError(t, err) + var targets []string + for _, e := range g.GetOutEdges("a/main.go") { + if e.Kind == graph.EdgeImports { + targets = append(targets, e.To) + } + } + assert.Equal(t, []string{"b/z.go"}, targets, "fallback must restore the evicted import onto a live representative") + resetImport() + cr.ResolveMutationFrontiers([]string{"a/main.go"}, nil, nil) + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + assert.Equal(t, "b/z.go", edges[0].To, "consumer edit must not reuse the deleted package-cache entry") + require.NotNil(t, g.GetNode(edges[0].To)) + }) + } +} + +func TestGoModuleImportIDsDoesNotSeed(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + cr.mu.Lock() + defer cr.mu.Unlock() + cr.goModuleImportIDs("b") + assert.Zero(t, g.importScans, "the ID getter must not run a lock-yielding seed") + assert.Zero(t, g.hydrated) +} diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index e5f9d3838..f6e91b33b 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -124,10 +124,6 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross if complete { return nil } - err := cr.resolveGoModuleImportSweep(ctx, stats) - if err != nil { - return err - } var batch []graph.EdgeReindex originals := make(map[graph.EdgeIdentity]*graph.Edge) names := make(map[string][]*graph.Node) @@ -502,9 +498,15 @@ func (cr *CrossRepoResolver) resolveGoModuleEdgesLocked(edges []*graph.Edge, sta // RetryGoModuleDeps is the revision-keyed migration used after publishing a module. func (cr *CrossRepoResolver) RetryGoModuleDeps(ctx context.Context) error { - cr.mu.Lock() - defer cr.mu.Unlock() - return cr.retryGoModuleDeps(ctx, &CrossRepoStats{ByRepo: make(map[string]int)}) + stats := &CrossRepoStats{ByRepo: make(map[string]int)} + if err := func() error { + cr.mu.Lock() + defer cr.mu.Unlock() + return cr.retryGoModuleDeps(ctx, stats) + }(); err != nil { + return err + } + return cr.resolveGoModuleImportSweep(ctx, stats) } // ResolveGoModuleIncomingForRepo probes the provider's definitions and retained @@ -514,7 +516,10 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { if len(cr.goModules) == 0 { return nil } - if cr.goModules[repo] != "" { + if err := cr.resolveGoModuleImportSweep(context.Background(), nil); err != nil { + return err + } + if cr.goModules[repo] != "" && len(cr.graph.RepoPrefixes()) >= 2 { var importIDs []string func() { cr.mu.Lock() From 681ef69a1f5a240d81ca6282df7f4c23cdad3b89 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Fri, 23 Oct 2026 07:22:22 -0700 Subject: [PATCH 003/111] fix(resolver): cancel Go import recovery and match imports by path Recovery of retained Go imports now takes the caller's context and stops between batches when it is cancelled. The presence check matched only the statement line, so two imports on one line were treated as one. It now matches the import path. It also accepts an import bound to a longer tracked module (for example example.com/lib/v2 next to example.com/lib), which avoids a duplicate edge on every full pass. File nodes load only when a same-line edge is a dep:: terminal. --- internal/resolver/cross_repo.go | 4 +- internal/resolver/cross_repo_incremental.go | 4 +- internal/resolver/go_module_imports.go | 39 ++++- internal/resolver/go_module_imports_test.go | 184 ++++++++++++++++++++ internal/resolver/go_modules_cross_repo.go | 8 +- 5 files changed, 228 insertions(+), 11 deletions(-) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 8e18a075d..8153eca70 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -407,7 +407,9 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS return stats, err } for repo := range cr.goModules { - cr.restoreGoModuleImports(repo, stats) + if err := cr.restoreGoModuleImports(ctx, repo, stats); err != nil { + return stats, err + } } if err := cr.retryGoModuleDeps(ctx, stats); err != nil { return stats, err diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index df81243af..98570d03c 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -80,7 +80,9 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed for _, file := range definitionFiles { if repo := graph.RepoPrefixOfID(file); cr.goModules[repo] != "" && !repos[repo] { repos[repo] = true - cr.restoreGoModuleImports(repo, stats) + if err := cr.restoreGoModuleImports(context.Background(), repo, stats); err != nil { + cr.logger.Error("cross-repo resolve: restore Go module imports", zap.Error(err)) + } } } } diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index ada330f1d..77cd1f75e 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -288,17 +288,23 @@ func (cr *CrossRepoResolver) goModuleImportIDs(repo string) []string { return ids } -func (cr *CrossRepoResolver) restoreGoModuleImports(repo string, stats *CrossRepoStats) { +func (cr *CrossRepoResolver) restoreGoModuleImports(ctx context.Context, repo string, stats *CrossRepoStats) error { if len(cr.graph.RepoPrefixes()) < 2 { - return + return ctx.Err() } ids := cr.goModuleImportIDs(repo) for start := 0; start < len(ids); start += externalCallMutationChunk { - cr.restoreGoModuleImportBatch(repo, ids[start:min(start+externalCallMutationChunk, len(ids))], stats) + if err := ctx.Err(); err != nil { + return err + } + if err := cr.restoreGoModuleImportBatch(ctx, repo, ids[start:min(start+externalCallMutationChunk, len(ids))], stats); err != nil { + return err + } } + return ctx.Err() } -func (cr *CrossRepoResolver) restoreGoModuleImportBatch(repo string, ids []string, stats *CrossRepoStats) { +func (cr *CrossRepoResolver) restoreGoModuleImportBatch(ctx context.Context, repo string, ids []string, stats *CrossRepoStats) error { state := cr.goModuleImportState() nodes := cr.graph.GetNodesByIDs(ids) paths := make(map[string]bool) @@ -310,6 +316,7 @@ func (cr *CrossRepoResolver) restoreGoModuleImportBatch(repo string, ids []strin } } outgoing := cr.graph.GetOutEdgesByNodeIDs(files) + var fileNodes map[string][]*graph.Node for _, id := range ids { n := nodes[id] if n == nil || n.Kind != graph.KindImport || n.Language != "go" { @@ -324,7 +331,27 @@ func (cr *CrossRepoResolver) restoreGoModuleImportBatch(repo string, ids []strin } present := false for _, e := range outgoing[n.FilePath] { - if e.Kind == graph.EdgeImports && e.Line == n.StartLine { + if e.Kind != graph.EdgeImports || e.Line != n.StartLine { + continue + } + target := graph.UnresolvedName(e.To) + if target == "" { + target = e.To + } + if strings.HasPrefix(strings.TrimPrefix(target, externalCallPrefix), "dep::") && fileNodes == nil { + fileNodes = cr.graph.GetFileNodesByPaths(files) + } + if goModuleImportPath(e, fileNodes[n.FilePath]) == path { + present = true + break + } + // A longer module may own the bound package, even when this + // statement is also registered under the edited provider. + targetRepo := graph.RepoPrefixOfID(e.To) + targetModule := cr.goModules[targetRepo] + if strings.HasSuffix(e.To, ".go") && targetModule != "" && + (path == targetModule || strings.HasPrefix(path, targetModule+"/")) && + filePathDir(e.To) == targetRepo+strings.TrimPrefix(path, targetModule) { present = true break } @@ -333,5 +360,5 @@ func (cr *CrossRepoResolver) restoreGoModuleImportBatch(repo string, ids []strin cr.graph.AddEdge(&graph.Edge{From: n.FilePath, To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: n.FilePath, Line: n.StartLine}) } } - _ = cr.resolveGoModuleImports(context.Background(), files, stats, nil) + return cr.resolveGoModuleImports(ctx, files, stats, nil) } diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index 3beb37620..ec3ccb385 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -405,3 +406,186 @@ func TestGoModuleImportIDsDoesNotSeed(t *testing.T) { assert.Zero(t, g.importScans, "the ID getter must not run a lock-yielding seed") assert.Zero(t, g.hydrated) } + +type goImportCancelStore struct { + graph.Store + cancel context.CancelFunc + batches int +} + +func (s *goImportCancelStore) GetNodesByIDs(ids []string) map[string]*graph.Node { + if s.cancel != nil && len(ids) > 0 && strings.Contains(ids[0], "::import::") { + s.batches++ + s.cancel() + } + return s.Store.GetNodesByIDs(ids) +} + +func TestGoModuleImportRestoreCancellation(t *testing.T) { + g := &goImportCancelStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + for i := range 2*externalCallMutationChunk + 1 { + file := fmt.Sprintf("a/f%d.go", i) + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: file + "::import::example.com/lib", Kind: graph.KindImport, FilePath: file, RepoPrefix: "a", Language: "go", StartLine: 2}) + g.AddEdge(&graph.Edge{From: file, To: "unresolved::import::example.com/lib", Kind: graph.EdgeImports, FilePath: file, Line: 2}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + g.EvictFile("b/lib.go") + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + g.cancel = cancel // cancel when restoration starts its first payload batch + stats, err := cr.ResolveAllContext(ctx) + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, g.batches, "cancellation must stop before the next restoration batch") + assert.Zero(t, stats.Resolved, "the cancelled context must reach the import rebind") +} + +func TestGoModuleImportRestoreSameLine(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"mutation", "fallback", "post_publish"} { + t.Run(backend+"/"+route, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + // import ( b "example.com/lib/sub"; c "example.net/other/pkg" ) + for _, path := range []string{"example.com/lib/sub", "example.net/other/pkg"} { + g.AddNode(&graph.Node{ID: "a/main.go::import::" + path, Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": path}}) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + } + for _, file := range []string{"b/sub/lib.go", "c/pkg/other.go"} { + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: graph.RepoPrefixOfID(file), Language: "go"}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib", "c": "example.net/other"}) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + require.Len(t, g.GetOutEdges("a/main.go"), 2) + g.EvictFile("b/sub/lib.go") + remaining := g.GetOutEdges("a/main.go") + require.Len(t, remaining, 1) + require.Equal(t, "c/pkg/other.go", remaining[0].To) + g.AddNode(&graph.Node{ID: "b/sub/lib.go", Kind: graph.KindFile, FilePath: "b/sub/lib.go", RepoPrefix: "b", Language: "go"}) + resolve := func() *CrossRepoStats { + switch route { + case "mutation": + return cr.ResolveMutationFrontiers(nil, nil, []string{"b/sub/lib.go"}) + case "fallback": + stats, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + return stats + default: + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("b")) + return nil + } + } + for pass := range 2 { // recovery, then an unchanged provider pass + if stats := resolve(); stats != nil { + assert.Equal(t, 1-pass, stats.Resolved, "only the missing import may be rebound") + } + var targets []string + for _, e := range g.GetOutEdges("a/main.go") { + if e.Kind == graph.EdgeImports { + targets = append(targets, e.To) + assert.True(t, e.CrossRepo) + assert.Equal(t, 2, e.Line) + } + } + assert.ElementsMatch(t, []string{"b/sub/lib.go", "c/pkg/other.go"}, targets, "both same-line imports must survive provider recovery without duplicates") + } + }) + } + } +} + +func TestGoModuleImportRestoreNestedProvider(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, module := range []string{"example.com/lib/v2", "example.com/lib/nested"} { + t.Run(backend+"/"+module, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + path := module + "/pkg" + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::" + path, Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": path}}) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + bFile := "b" + strings.TrimPrefix(path, "example.com/lib") + "/lib.go" + for _, file := range []string{bFile, "d/pkg/lib.go"} { + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: graph.RepoPrefixOfID(file), Language: "go"}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib", "d": module}) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + check := func(stats *CrossRepoStats) { + t.Helper() + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1, "an import bound to the longer module must not be restored again") + assert.Equal(t, "d/pkg/lib.go", edges[0].To) + assert.Equal(t, graph.EdgeImports, edges[0].Kind) + assert.Equal(t, 2, edges[0].Line) + assert.True(t, edges[0].CrossRepo) + if stats != nil { + assert.Zero(t, stats.Resolved) + assert.Zero(t, stats.CrossRepoEdges) + assert.Empty(t, stats.ByRepo) + } + } + check(nil) + for range 2 { + stats, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + check(stats) + g.EvictFile(bFile) + g.AddNode(&graph.Node{ID: bFile, Kind: graph.KindFile, FilePath: bFile, RepoPrefix: "b", Language: "go"}) + check(cr.ResolveMutationFrontiers(nil, nil, []string{bFile})) + } + }) + } + } +} + +func TestGoModuleImportRestoreLookupBudget(t *testing.T) { + for _, tc := range []struct { + name, target string + line, want int + }{ + {"bound", "b/lib.go", 2, 1}, + {"missing", "", 2, 1}, + {"unresolved", "unresolved::import::example.com/lib", 2, 1}, + {"external", "external::example.com/lib", 2, 1}, + {"dep", "dep::example.com/lib", 2, 2}, + {"unresolved_dep", "unresolved::dep::example.com/lib", 2, 2}, + {"external_call_dep", "external-call::dep::example.com/lib", 2, 2}, + {"other_line_dep", "dep::example.com/lib", 3, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib"}}) + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "unresolved::import::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + g.RemoveEdge("a/main.go", "b/lib.go", graph.EdgeImports) + if tc.target != "" { + g.AddEdge(&graph.Edge{From: "a/main.go", To: tc.target, Kind: graph.EdgeImports, FilePath: "a/main.go", Line: tc.line}) + } + g.hydrated = 0 + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + assert.LessOrEqual(t, g.hydrated, tc.want, "only a same-line dep terminal may need restore payloads in addition to the rebind's single load") + }) + } +} diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index f6e91b33b..17ddd5728 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -527,11 +527,13 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { importIDs = cr.goModuleImportIDs(repo) }() for start := 0; start < len(importIDs); start += externalCallMutationChunk { - func() { + if err := func() error { cr.mu.Lock() defer cr.mu.Unlock() - cr.restoreGoModuleImportBatch(repo, importIDs[start:min(start+externalCallMutationChunk, len(importIDs))], nil) - }() + return cr.restoreGoModuleImportBatch(context.Background(), repo, importIDs[start:min(start+externalCallMutationChunk, len(importIDs))], nil) + }(); err != nil { + return err + } } } for files := range cr.goModuleImportFileBatches([]string{repo}) { From a4ddd3d5a6a26c00c8a3ae9e83ba95d35f4242c4 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 03:55:05 -0700 Subject: [PATCH 004/111] fix(resolver): keep tracked Go import links across provider changes Use indexed module paths and retain import identity through provider eviction. Recheck file bindings on module changes and bound incoming import recovery. Verify provider lifecycles against independent cold indexes. --- .../go_module_import_lifecycle_test.go | 179 ++++++++++++++++++ internal/indexer/go_module_restub.go | 7 +- internal/indexer/go_module_revision_test.go | 10 +- internal/indexer/incremental_batch.go | 17 +- internal/indexer/indexer.go | 4 +- internal/indexer/workspace_resolve.go | 2 +- internal/resolver/cross_repo.go | 1 + internal/resolver/cross_repo_incremental.go | 35 +++- internal/resolver/go_module_imports.go | 57 ++++-- internal/resolver/go_module_imports_test.go | 30 +++ internal/resolver/go_modules_cross_repo.go | 23 ++- 11 files changed, 329 insertions(+), 36 deletions(-) create mode 100644 internal/indexer/go_module_import_lifecycle_test.go diff --git a/internal/indexer/go_module_import_lifecycle_test.go b/internal/indexer/go_module_import_lifecycle_test.go new file mode 100644 index 000000000..f9aa2917e --- /dev/null +++ b/internal/indexer/go_module_import_lifecycle_test.go @@ -0,0 +1,179 @@ +package indexer + +import ( + "context" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/resolver" + "github.com/zzet/gortex/internal/search" +) + +// Given a tracked provider and an unchanged Go consumer, every provider +// transition must leave the same imports as an independent cold installation. +func TestGoModuleImportLifecycleColdParity(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, reverse := range []bool{false, true} { + order := "consumer_first" + if reverse { + order = "provider_first" + } + for _, mode := range []string{"point", "watcher", "full", "reconcile", "untrack", "rename", "denied"} { + t.Run(backend+"/"+order+"/"+mode, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + writeFile(t, filepath.Join(repos[0].Path, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\n") + writeFile(t, filepath.Join(repos[0].Path, "main.go"), "package caller\nimport library \"example.org/library\"\nimport _ \"example.net/untracked\"\nfunc Caller() { library.Hello() }\n") + if mode == "denied" { + repos[1].Workspace = "other" + } + entries := []config.RepoEntry{repos[0], repos[1]} + if reverse { + entries[0], entries[1] = entries[1], entries[0] + } + track := func(idx *MultiIndexer, entries []config.RepoEntry) { + t.Helper() + for _, entry := range entries { + _, err := idx.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + } + track(mi, entries) + type importRow struct { + To string + CrossRepo bool + Confidence float64 + Origin string + } + imports := func(store graph.Store) []importRow { + t.Helper() + var rows []importRow + for _, e := range store.GetOutEdges("a/main.go") { + if e.Kind != graph.EdgeImports { + continue + } + rows = append(rows, importRow{e.To, e.CrossRepo, e.Confidence, e.Origin}) + if e.CrossRepo { + assert.Equal(t, "example.org/library", e.Meta["go_import"], "retain package identity for eviction and map changes") + } + } + sort.Slice(rows, func(i, j int) bool { return rows[i].To < rows[j].To }) + require.Len(t, rows, 2, "both the tracked and unrelated imports survive") + return rows + } + compareCold := func(active []config.RepoEntry) { + t.Helper() + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + track(cold, active) + cold.RunGlobalResolve() + assert.Equal(t, imports(coldGraph), imports(g), "warm imports must match a fresh cold graph") + } + initial := imports(g) + assert.Contains(t, initial, importRow{To: "external::example.net/untracked"}) + if mode == "denied" { + assert.Contains(t, initial, importRow{To: "dep::example.org/library"}) + } else { + assert.Contains(t, initial, importRow{To: "b/lib.go", CrossRepo: true}) + } + compareCold(entries) + lib := filepath.Join(repos[1].Path, "lib.go") + writeFile(t, lib, "package library\nfunc Hello() { println(1) }\n") + var err error + switch mode { + case "point", "denied": + err = mi.GetIndexer("b").IndexFile(lib) + case "watcher": + _, err = mi.incrementalPointRepoRaw("b", lib) + case "full": + _, err = mi.IndexRepo("b") + case "reconcile": + t.Setenv("GORTEX_WARMUP_FULL_RETRACK", "1") + restored := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), mi.configMgr, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, restored.Close(context.Background())) }) + var result *IndexResult + result, err = restored.ReconcileRepoCtx(t.Context(), repos[1], mi.FileMtimes("b")) + require.NoError(t, err) + require.True(t, result.FullRetrack) + case "untrack": + _, _, err = mi.UntrackRepoChecked(t.Context(), "b") + require.NoError(t, err) + mi.RunGlobalResolve() + compareCold([]config.RepoEntry{repos[0]}) + _, err = mi.TrackRepoCtx(t.Context(), repos[1]) + case "rename": + manifest := filepath.Join(repos[1].Path, "go.mod") + writeFile(t, manifest, "module example.org/renamed\n") + err = mi.GetIndexer("b").IndexFile(manifest) + } + require.NoError(t, err) + compareCold(entries) + mi.RunGlobalResolve() + compareCold(entries) + }) + } + } + } +} + +func TestGoModuleImportParkBeforeEviction(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"repo", "file", "deleted_batch"} { + t.Run(backend+"/"+route, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "park.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + g.AddBatch([]*graph.Node{ + {ID: "a/main.go", Name: "main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, + {ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{{From: "a/main.go", To: "b/lib.go", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/lib"}}}) + if route == "repo" { + restubGoModuleRepoIncoming(g, "b") + } else if route == "deleted_batch" { + provider := g.GetNode("b/lib.go") + carried := restubIncomingRefsFromView(g, + []*incrementalBatchStage{{graphPath: "b/lib.go", priorNodes: []*graph.Node{provider}}}, + incrementalPriorView{inByNode: g.GetInEdgesByNodeIDs([]string{provider.ID})}) + assert.Empty(t, carried, "a deleted file cannot carry a resolved target") + } else { + idx := newTestIndexer(g) + idx.SetRepoPrefix("b") + t.Cleanup(idx.Close) + idx.restubIncomingRefs("b/lib.go") + } + g.EvictFile("b/lib.go") + rows := g.GetOutEdges("a/main.go") + require.Len(t, rows, 1, "the consumer-owned row must survive provider eviction") + assert.Equal(t, "unresolved::import::example.org/lib", rows[0].To) + assert.Equal(t, "example.org/lib", rows[0].Meta["go_import"]) + g.AddNode(&graph.Node{ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + cr := resolver.NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.org/lib"}) + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/lib.go"}) + assert.False(t, admission.Refused) + rows = g.GetOutEdges("a/main.go") + require.Len(t, rows, 1) + assert.Equal(t, "b/lib.go", rows[0].To, "the exact incoming frontier must relink without a retained import statement") + assert.True(t, rows[0].CrossRepo) + }) + } + } +} diff --git a/internal/indexer/go_module_restub.go b/internal/indexer/go_module_restub.go index d0dfeed79..9c65ab770 100644 --- a/internal/indexer/go_module_restub.go +++ b/internal/indexer/go_module_restub.go @@ -8,6 +8,9 @@ import ( func goModuleRestubTarget(e *graph.Edge, fallback string) string { if spec, ok := e.Meta["go_import"].(string); ok && spec != "" && e.CrossRepo && strings.HasSuffix(e.FilePath, ".go") { + if e.Kind == graph.EdgeImports { + return graph.UnresolvedMarker + "import::" + spec + } return graph.UnresolvedMarker + "extern::" + spec } return fallback @@ -25,7 +28,7 @@ func restubGoModuleRepoIncoming(g graph.Store, repo string) []graph.EdgeIdentity var batch []graph.EdgeReindex for _, incoming := range g.GetInEdgesByNodeIDs(ids) { for _, e := range incoming { - if !graph.IsResolvableRefEdge(e.Kind) || graph.RepoPrefixOfID(e.From) == repo { + if !graph.IsResolvableRefEdge(e.Kind) && e.Kind != graph.EdgeImports || graph.RepoPrefixOfID(e.From) == repo { continue } stub := goModuleRestubTarget(e, "") @@ -43,7 +46,7 @@ func restubGoModuleRepoIncoming(g graph.Store, repo string) []graph.EdgeIdentity g.ReindexEdges(batch) ids = ids[:0] } - for n := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFunction, graph.KindType, graph.KindInterface) { + for n := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { ids = append(ids, n.ID) if len(ids) == 512 { flush() diff --git a/internal/indexer/go_module_revision_test.go b/internal/indexer/go_module_revision_test.go index 5b9f1b012..2bbf7a18b 100644 --- a/internal/indexer/go_module_revision_test.go +++ b/internal/indexer/go_module_revision_test.go @@ -244,7 +244,10 @@ func TestGoModuleUntrackRetainsOnlyParkedIdentities(t *testing.T) { require.NoError(t, err) } parked := restubGoModuleRepoIncoming(g, "b") - require.Len(t, parked, 1) + require.ElementsMatch(t, []graph.EdgeIdentity{ + {From: "a/main.go", To: "unresolved::import::example.org/library", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}, + {From: "a/main.go::Caller", To: "unresolved::extern::example.org/library::Hello", Kind: graph.EdgeCalls, FilePath: "a/main.go", Line: 3}, + }, parked, "both caller-owned references must survive and settle") assert.IsType(t, []graph.EdgeIdentity{}, parked, "teardown must not retain full metadata-bearing edge objects") g.EvictFile("b/lib.go") require.NoError(t, mi.finalizeUntrackedGoModuleCalls(parked)) @@ -316,7 +319,10 @@ func TestGoModuleParkedLookupRejectsMismatchedRows(t *testing.T) { require.NoError(t, err) } parked := restubGoModuleRepoIncoming(g, "b") - require.Len(t, parked, 1) + require.ElementsMatch(t, []graph.EdgeIdentity{ + {From: "a/main.go", To: "unresolved::import::example.org/library", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}, + {From: "a/main.go::Caller", To: "unresolved::extern::example.org/library::Hello", Kind: graph.EdgeCalls, FilePath: "a/main.go", Line: 3}, + }, parked, "both caller-owned references must survive and settle") require.NotEmpty(t, g.(graph.EdgeIdentityBatchFinder).FindEdgesByIdentities(parked), "parked row must exist before purge") g.EvictFile("b/lib.go") require.NotEmpty(t, g.(graph.EdgeIdentityBatchFinder).FindEdgesByIdentities(parked), "parked row must survive purge") diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 2779b774d..8edf04be3 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -1393,7 +1393,22 @@ func restubIncomingRefsFromView( // path) survives its own reparse. The eviction deletes the // row and no pass of this save re-derives it — the importer // is not reparsed — so re-state it. - if _, survives := frontier.survivingIDs[node.ID]; survives || frontier.conservative { + if _, survives := frontier.survivingIDs[node.ID]; !survives && (!frontier.conservative || stage.result == nil) { + for _, edge := range view.inByNode[node.ID] { + if edge == nil || edge.Kind != graph.EdgeImports { + continue + } + if _, sourceEvicted := evicted[edge.From]; sourceEvicted { + continue + } + if stub := goModuleRestubTarget(edge, ""); stub != "" { + oldTo := edge.To + graph.StashRestubProvenance(edge) + edge.To, edge.CrossRepo = stub, false + reindexes = append(reindexes, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) + } + } + } else { for _, edge := range view.inByNode[node.ID] { if edge == nil || edge.Kind != graph.EdgeImports { continue diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 64c006ea5..88fb3f339 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -5978,12 +5978,12 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) []string { npmConsumers := restubTrackedNpmImportsFromView(idx.graph, evicted, inEdges) var batch []graph.EdgeReindex for _, n := range nodes { - if n == nil || n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) { + if n == nil || n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) && n.Kind != graph.KindFile { continue } stub := graph.UnresolvedMarker + n.Name for _, e := range inEdges[n.ID] { - if e == nil || !graph.IsResolvableRefEdge(e.Kind) { + if e == nil || !graph.IsResolvableRefEdge(e.Kind) && !(e.Kind == graph.EdgeImports && e.Meta["go_import"] != nil) { continue } if _, fromEvicted := evicted[e.From]; fromEvicted { diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 1a88aac47..191371cfc 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -708,7 +708,7 @@ func (mi *MultiIndexer) resolveDeferredGoModules(idx *Indexer) error { } func (mi *MultiIndexer) resolveIncomingGoModules(repo string) error { - if mi.graph == nil || len(mi.graph.RepoPrefixes()) < 2 || mi.trackedGoModules()[repo] == "" { + if mi.graph == nil || len(mi.graph.RepoPrefixes()) < 2 || len(mi.trackedGoModules()) == 0 { return nil } cr := mi.newCrossRepoResolver() diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 8153eca70..5040d2618 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -135,6 +135,7 @@ type CrossRepoResolver struct { crossWorkspaceLookup CrossWorkspaceDepLookup goModules map[string]string goModulePaths goModulePathIndex + goModuleRepos map[string][]string goModuleReplacements map[string][]GoModuleReplacement goModuleRetry *GoModuleDepRetry goModuleRevision uint64 diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 98570d03c..9f1c7ed78 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -75,22 +75,33 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed cr.mu.Lock() defer cr.mu.Unlock() + budget := &graph.IncomingSourceBudget{} + importsByRepo := make(map[string][]string) if len(definitionFiles) > 0 && len(cr.goModules) > 0 { repos := make(map[string]bool) for _, file := range definitionFiles { if repo := graph.RepoPrefixOfID(file); cr.goModules[repo] != "" && !repos[repo] { repos[repo] = true - if err := cr.restoreGoModuleImports(context.Background(), repo, stats); err != nil { - cr.logger.Error("cross-repo resolve: restore Go module imports", zap.Error(err)) + ids := cr.goModuleImportIDs(repo) + admission.Keys += len(ids) + if err := budget.Charge(len(ids)); err != nil { + admission.Refused, admission.Refusal = true, err + admission.Dropped += len(ids) + break } + admission.Inspected += len(ids) + importsByRepo[repo] = ids } } } if len(resolutionFiles) > 0 { frontier := collectIncrementalFileFrontier(cr.graph, resolutionFiles, nil) - admission = CrossRepoIncomingAdmission{ - IncomingSourceAdmission: frontier.incomingAdmission, - Refusal: frontier.incomingRefusal, + admission.Keys += frontier.incomingAdmission.Keys + admission.Inspected += frontier.incomingAdmission.Inspected + admission.Dropped += frontier.incomingAdmission.Dropped + admission.Limit = frontier.incomingAdmission.Limit + if frontier.incomingAdmission.Refused { + admission.Refused, admission.Refusal = true, frontier.incomingRefusal } logIncomingAdmissionRefusal(cr.logger, "cross_repo_preparation", len(frontier.stubKeys), frontier.incomingAdmission, frontier.incomingRefusal) @@ -138,9 +149,8 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed _ = cr.resolveGoModuleImports(context.Background(), resolutionFiles, stats, counted) } if len(definitionFiles) > 0 && len(cr.goModules) > 0 && !admission.Refused { - budget := &graph.IncomingSourceBudget{} - if admission.Inspected > 0 { - if err := budget.Charge(admission.Inspected); err != nil { + if frontierInspected := admission.Inspected - (graph.MaxIncomingSourceCandidateRows - budget.Remaining()); frontierInspected > 0 { + if err := budget.Charge(frontierInspected); err != nil { admission.Refused, admission.Refusal = true, err } } @@ -153,6 +163,15 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed admission.Dropped += fact.Dropped admission.Refused, admission.Refusal = fact.Refused, err logIncomingAdmissionRefusal(cr.logger, "go_module_incoming", fact.Keys, fact, err) + if !admission.Refused { + for repo, ids := range importsByRepo { + for start := 0; start < len(ids); start += externalCallMutationChunk { + if err := cr.restoreGoModuleImportBatch(context.Background(), repo, ids[start:min(start+externalCallMutationChunk, len(ids))], stats); err != nil { + cr.logger.Error("cross-repo resolve: restore Go module imports", zap.Error(err)) + } + } + } + } cr.resolveGoModuleEdgesLocked(edges, stats) } } diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 77cd1f75e..33188b13b 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -14,9 +14,10 @@ import ( // counted records whether the ordinary frontier pass already counted an import // as resolved (true) or unresolved (false), before this pass retargets it. func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files []string, stats *CrossRepoStats, counted map[graph.EdgeIdentity]bool) error { - if len(cr.goModules) == 0 || len(files) == 0 { + if len(files) == 0 { return nil } + cr.goModuleImportState() flush := func(paths []string) { nodes := cr.graph.GetFileNodesByPaths(paths) outgoing := cr.graph.GetOutEdgesByNodeIDs(paths) @@ -28,7 +29,7 @@ func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files [ continue } importPath := goModuleImportPath(e, nodes[path]) - if importPath == "" || !trackedGoImport(cr.goModules, importPath) { + if importPath == "" || !trackedGoImport(cr.goModulePaths, importPath) && e.Meta["go_import"] == nil { continue } source := cr.graph.GetNode(e.From) @@ -37,13 +38,31 @@ func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files [ caller = source.RepoPrefix } match := cr.goModuleImportCandidate(importPath, caller, candidateWorkspaceID(source)) - if match == nil || match.ID == e.To { + if match != nil && match.ID == e.To || match == nil && e.Meta["go_import"] == nil { continue } copy := *e copy.Meta = maps.Clone(e.Meta) - copy.To = match.ID - copy.CrossRepo = caller != "" && match.RepoPrefix != caller + if copy.Meta == nil { + copy.Meta = make(map[string]any) + } + if copy.Meta["go_import"] == nil { + copy.Meta["go_import"] = importPath + copy.Meta["go_import_fallback"] = e.To + } + if match != nil { + copy.To = match.ID + copy.CrossRepo = caller != "" && match.RepoPrefix != caller + } else { + copy.To, _ = copy.Meta["go_import_fallback"].(string) + if copy.To == "" || graph.IsUnresolvedTarget(copy.To) { + copy.To = "external::" + importPath + } + copy.CrossRepo, copy.Confidence, copy.ConfidenceLabel, copy.Origin, copy.Tier = false, 0, "", "", "" + } + if copy.To == e.To { + continue + } clearEdgeTerminal(©) batch = append(batch, graph.EdgeReindex{Edge: ©, OldTo: e.To, RefreshIdentity: true, OldFilePath: e.FilePath, OldLine: e.Line}) if len(batch) == externalCallMutationChunk { @@ -75,6 +94,9 @@ func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files [ // A dep contract has lost the package suffix. Recover it from the retained // import statement, but never guess when several same-line imports fit. func goModuleImportPath(e *graph.Edge, nodes []*graph.Node) string { + if path, ok := e.Meta["go_import"].(string); ok && path != "" { + return path + } target := graph.UnresolvedName(e.To) if target == "" { target = e.To @@ -110,12 +132,10 @@ func goModuleImportPath(e *graph.Edge, nodes []*graph.Node) string { // Package identities are cached across resolver objects by module revision. // A provider mutation invalidates only that provider's projection. func (cr *CrossRepoResolver) goModuleImportCandidate(importPath, caller, workspace string) *graph.Node { - state := cr.goModuleImportState() + state := cr.goModuleRetry var candidates []*graph.Node - for repo, module := range cr.goModules { - if module == "" || (importPath != module && !strings.HasPrefix(importPath, module+"/")) { - continue - } + module := cr.goModulePaths.longest(importPath) + for _, repo := range cr.goModuleRepos[module] { packages, cached := state.packages[repo] if !cached { packages = make(map[string]graph.FileNodeIdentity) @@ -134,6 +154,9 @@ func (cr *CrossRepoResolver) goModuleImportCandidate(importPath, caller, workspa candidates = append(candidates, &graph.Node{ID: file.ID, Kind: graph.KindFile, FilePath: file.FilePath, RepoPrefix: file.RepoPrefix, WorkspaceID: file.WorkspaceID, Language: "go"}) } } + if _, replaced := goModuleReplacementFor(cr.goModuleReplacements, caller, importPath); replaced { + return nil + } return goModuleCandidate(cr.goModules, importPath, candidates, caller, workspace, cr.crossWorkspaceLookup) } @@ -180,7 +203,7 @@ func (cr *CrossRepoResolver) goModuleImportFileBatches(repos []string) iter.Seq[ var files []string for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, graph.KindImport) { file, path, ok := strings.Cut(n.ID, "::import::") - if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModules, path) || seen[file] { + if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModulePaths, path) || seen[file] { continue } if module := cr.goModules[n.RepoPrefix]; module != "" && (path == module || strings.HasPrefix(path, module+"/")) { @@ -208,7 +231,11 @@ func (cr *CrossRepoResolver) goModuleImportState() *GoModuleDepRetry { cr.goModuleRetry = &GoModuleDepRetry{} } state := cr.goModuleRetry - if state.packageRevision != cr.goModuleRevision || !maps.Equal(state.packageModules, cr.goModules) { + if state.packageRevision != cr.goModuleRevision { + state.packageRevision = cr.goModuleRevision + state.packages = nil + } + if !maps.Equal(state.packageModules, cr.goModules) { state.packageModules = maps.Clone(cr.goModules) state.packageRevision = cr.goModuleRevision state.packages = nil @@ -255,7 +282,7 @@ func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, sta } func (cr *CrossRepoResolver) rememberGoModuleImports(nodes []*graph.Node) { - state := cr.goModuleImportState() + state := cr.goModuleRetry for _, n := range nodes { if n.Kind != graph.KindImport || n.Language != "go" { continue @@ -264,8 +291,8 @@ func (cr *CrossRepoResolver) rememberGoModuleImports(nodes []*graph.Node) { if !ok { continue } - for repo, module := range cr.goModules { - if repo == n.RepoPrefix || module == "" || (path != module && !strings.HasPrefix(path, module+"/")) { + for _, repo := range cr.goModuleRepos[cr.goModulePaths.longest(path)] { + if repo == n.RepoPrefix { continue } if state.imports[repo] == nil { diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index ec3ccb385..305b54623 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -589,3 +589,33 @@ func TestGoModuleImportRestoreLookupBudget(t *testing.T) { }) } } + +// Given import rows above the shared admission ceiling, the incoming leg +// refuses before either parked-edge binding or legacy recovery changes rows. +func TestGoModuleImportIncomingAdmissionRefused(t *testing.T) { + for _, recovery := range []bool{false, true} { + t.Run(fmt.Sprint(recovery), func(t *testing.T) { + source := &graph.Node{ID: "a/main.go", Name: "main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"} + provider := &graph.Node{ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"} + edge := &graph.Edge{From: source.ID, To: "unresolved::import::example.org/lib", Kind: graph.EdgeImports, FilePath: source.FilePath, Meta: map[string]any{"go_import": "example.org/lib"}} + g := &goModuleAdmissionStore{Store: graph.New(), edge: edge} + g.AddBatch([]*graph.Node{source, provider}, []*graph.Edge{edge}) + modules := map[string]string{"a": "example.org/app", "b": "example.org/lib"} + state := &GoModuleDepRetry{modules: modules, complete: true, revision: 1, packageModules: modules, packageRevision: 1, importsSeeded: true} + if recovery { + state.imports = map[string]map[string]struct{}{"b": {}} + for i := 0; i <= graph.MaxIncomingSourceCandidateRows; i++ { + state.imports["b"][fmt.Sprint(i)] = struct{}{} + } + } + cr := NewCrossRepo(g) + cr.SetGoModules(modules) + cr.SetGoModuleDepRetry(state, 1) + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{provider.FilePath}) + require.True(t, admission.Refused) + var limit *graph.BoundedLocalizationLimitError + require.ErrorAs(t, admission.Refusal, &limit) + assert.Equal(t, "unresolved::import::example.org/lib", g.GetOutEdges(source.ID)[0].To) + }) + } +} diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index 17ddd5728..ede1ff62a 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -13,6 +13,10 @@ import ( func (cr *CrossRepoResolver) SetGoModules(modules map[string]string) { cr.goModules = maps.Clone(modules) cr.goModulePaths = indexGoModulePaths(modules) + cr.goModuleRepos = make(map[string][]string) + for repo, module := range modules { + cr.goModuleRepos[module] = append(cr.goModuleRepos[module], repo) + } } func (cr *CrossRepoResolver) resolveGoExtern(e *graph.Edge, spec string, stats *CrossRepoStats) { @@ -241,7 +245,7 @@ func (cr *CrossRepoResolver) goModuleIncomingEdgesBounded(nodesByFile map[string synthetic := make(map[string][]*graph.Node) for _, file := range definitionFiles { for _, n := range nodesByFile[file] { - if n == nil || n.Language != "go" || n.Name == "" || (n.Kind != graph.KindFunction && n.Kind != graph.KindType && n.Kind != graph.KindInterface) { + if n == nil || n.Language != "go" || n.Name == "" || (n.Kind != graph.KindFunction && n.Kind != graph.KindType && n.Kind != graph.KindInterface && n.Kind != graph.KindFile) { continue } module := cr.goModules[n.RepoPrefix] @@ -250,6 +254,12 @@ func (cr *CrossRepoResolver) goModuleIncomingEdgesBounded(nodesByFile map[string continue } spec := module + strings.TrimPrefix(dir, n.RepoPrefix) + "::" + n.Name + if n.Kind == graph.KindFile { + spec = module + strings.TrimPrefix(dir, n.RepoPrefix) + providers[spec] = append(providers[spec], n) + keys[graph.UnresolvedMarker+"import::"+spec] = struct{}{} + continue + } providers[spec] = append(providers[spec], n) keys[graph.UnresolvedMarker+"extern::"+spec] = struct{}{} keys["dep::"+spec] = struct{}{} @@ -299,12 +309,15 @@ func (cr *CrossRepoResolver) goModuleIncomingEdgesBounded(nodesByFile map[string } spec := goModuleEdgeSpec(e) sep := strings.LastIndex(spec, "::") - if sep < 0 { + importPath := spec + if sep >= 0 { + importPath = spec[:sep] + } else if e.Kind != graph.EdgeImports { continue } source := sourceOf(e.From) for _, provider := range providers[spec] { - if crossWorkspaceEligible(cr.crossWorkspaceLookup, candidateWorkspaceID(source), candidateWorkspaceID(provider), spec[:sep]) { + if crossWorkspaceEligible(cr.crossWorkspaceLookup, candidateWorkspaceID(source), candidateWorkspaceID(provider), importPath) { edges = append(edges, e) break } @@ -564,7 +577,7 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { ids = ids[:0] return cr.ResolveDeferredGoModuleEdges(edges) } - for n := range graph.RepoNodeIdentitiesSeq(cr.graph, []string{repo}, graph.KindFunction, graph.KindType, graph.KindInterface) { + for n := range graph.RepoNodeIdentitiesSeq(cr.graph, []string{repo}, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { ids = append(ids, n.ID) if len(ids) == externalCallMutationChunk { if err := flush(); err != nil { @@ -627,7 +640,7 @@ func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previo cr.resolveGoModuleEdgesLocked(edges, stats) ids = ids[:0] } - for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, graph.KindFunction, graph.KindType, graph.KindInterface) { + for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { if err := ctx.Err(); err != nil { return err } From b66bd875c0711a994cb8d18eb65cbba1451146bd Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 05:02:46 -0700 Subject: [PATCH 005/111] fix(resolver): keep Go import recovery idempotent and within its own budget Recognize caller-local replace targets before restoring import rows. Keep import recovery and restart seeding on a separate bounded budget so incoming call binding and terminal retries still run. --- .../indexer/go_module_import_recovery_test.go | 165 +++++++++++++++++ internal/resolver/cross_repo.go | 2 +- internal/resolver/cross_repo_incremental.go | 55 +++--- .../go_module_import_recovery_test.go | 170 ++++++++++++++++++ internal/resolver/go_module_imports.go | 33 +++- internal/resolver/go_modules_cross_repo.go | 9 +- 6 files changed, 394 insertions(+), 40 deletions(-) create mode 100644 internal/indexer/go_module_import_recovery_test.go create mode 100644 internal/resolver/go_module_import_recovery_test.go diff --git a/internal/indexer/go_module_import_recovery_test.go b/internal/indexer/go_module_import_recovery_test.go new file mode 100644 index 000000000..8baaccf10 --- /dev/null +++ b/internal/indexer/go_module_import_recovery_test.go @@ -0,0 +1,165 @@ +package indexer + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/search" +) + +func TestGoModuleImportReplaceRecoveryColdParity(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, reverse := range []bool{false, true} { + for _, scen := range []string{"inreplace", "samedir"} { + t.Run(fmt.Sprintf("%s/%s/reverse=%v", backend, scen, reverse), func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a, b := repos[0].Path, repos[1].Path + gomod := "module example.org/caller\nrequire example.org/library v1.0.0\n" + if scen == "inreplace" { + gomod += "replace example.org/library => ./third_party/library\n" + require.NoError(t, os.MkdirAll(filepath.Join(a, "third_party", "library"), 0o755)) + writeFile(t, filepath.Join(a, "third_party", "library", "lib.go"), "package library\nfunc Hello() {}\n") + } else { + require.NoError(t, os.MkdirAll(filepath.Join(a, "pkg", "library"), 0o755)) + writeFile(t, filepath.Join(a, "pkg", "library", "l.go"), "package library\nfunc Hello() {}\n") + } + writeFile(t, filepath.Join(a, "go.mod"), gomod) + writeFile(t, filepath.Join(a, "main.go"), "package caller\nimport library \"example.org/library\"\nfunc Caller() { library.Hello() }\n") + entries := []config.RepoEntry{repos[0], repos[1]} + if reverse { + entries[0], entries[1] = entries[1], entries[0] + } + for _, e := range entries { + _, err := mi.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + compareCold := func() { + t.Helper() + var cg graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + defer func() { require.NoError(t, db.Close()) }() + cg = db + } + cold := NewMultiIndexer(cg, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + defer func() { require.NoError(t, cold.Close(context.Background())) }() + for _, e := range entries { + _, err := cold.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + cold.RunGlobalResolve() + require.Len(t, goImportRecoveryRows(cg, "a/main.go"), 1) + assert.Equal(t, goImportRecoveryRows(cg, "a/main.go"), goImportRecoveryRows(g, "a/main.go")) + } + compareCold() + lib := filepath.Join(b, "lib.go") + for i := 0; i < 3; i++ { + writeFile(t, lib, "package library\nfunc Hello() { println("+string(rune('1'+i))+") }\n") + _, err := mi.IncrementalReindexRepo("b", []string{lib}) + require.NoError(t, err) + compareCold() + } + for i := 0; i < 3; i++ { + mi.RunGlobalResolve() + compareCold() + } + assert.Len(t, goImportRecoveryRows(g, "a/main.go"), 1, "one import statement must keep one IMPORT edge") + }) + } + } + } +} + +func goImportRecoveryRows(g graph.Store, file string) []string { + var rows []string + for _, e := range g.GetOutEdges(file) { + if e.Kind != graph.EdgeImports { + continue + } + rows = append(rows, fmt.Sprintf("L%d %s cross=%v conf=%v origin=%s gi=%v fb=%v", e.Line, e.To, e.CrossRepo, e.Confidence, e.Origin, e.Meta["go_import"], e.Meta["go_import_fallback"])) + } + sort.Strings(rows) + return rows +} + +func TestGoModuleImportPopularProviderSave(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, reverse := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/reverse=%v", backend, reverse), func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a, b := repos[0].Path, repos[1].Path + writeFile(t, filepath.Join(a, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\n") + writeFile(t, filepath.Join(a, "main.go"), "package caller\nimport library \"example.org/library\"\nfunc Caller() { library.New() }\n") + writeFile(t, filepath.Join(a, "newpkg.go"), "package caller\nimport _ \"example.org/library/new\"\n") + entries := []config.RepoEntry{repos[0], repos[1]} + if reverse { + entries[0], entries[1] = entries[1], entries[0] + } + track := func(idx *MultiIndexer) { + t.Helper() + for _, entry := range entries { + _, err := idx.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + } + track(mi) + populate := func(store graph.Store) { + var nodes []*graph.Node + var edges []*graph.Edge + for i := 0; i < graph.MaxIncomingSourceCandidateRows+1; i++ { + f := fmt.Sprintf("a/filler%d.go", i) + nodes = append(nodes, &graph.Node{ID: f, Kind: graph.KindFile, FilePath: f, RepoPrefix: "a", Language: "go"}, &graph.Node{ID: f + "::import::example.org/library", Kind: graph.KindImport, FilePath: f, RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/library"}}) + edges = append(edges, &graph.Edge{From: f, To: "b/lib.go", Kind: graph.EdgeImports, FilePath: f, Line: 2, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/library", "go_import_fallback": "dep::example.org/library"}}) + } + store.AddBatch(nodes, edges) + } + populate(g) + // Reconcile the unchanged persisted graph into a new process, then seed on a full pass. + restored := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), mi.configMgr, zap.NewNop()) + defer func() { require.NoError(t, restored.Close(context.Background())) }() + for _, entry := range entries { + _, err := restored.ReconcileRepoCtx(t.Context(), entry, mi.FileMtimes(entry.Name)) + require.NoError(t, err) + } + restored.RunGlobalResolve() + f := filepath.Join(b, "new.go") + writeFile(t, f, "package library\nfunc New() {}\n") + require.NoError(t, restored.GetIndexer("b").IndexFile(f)) + assert.Equal(t, "b/new.go::New", goModuleLifecycleCall(t, g).To, "the actual provider save must bind the unchanged caller") + require.NoError(t, os.MkdirAll(filepath.Join(b, "new"), 0o755)) + pkg := filepath.Join(b, "new", "p.go") + writeFile(t, pkg, "package newpkg\n") + require.NoError(t, restored.GetIndexer("b").IndexFile(pkg)) + restored.RunGlobalResolve() + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + defer func() { require.NoError(t, db.Close()) }() + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + defer func() { require.NoError(t, cold.Close(context.Background())) }() + track(cold) + populate(coldGraph) + cold.RunGlobalResolve() + for _, file := range []string{"a/main.go", "a/newpkg.go"} { + assert.Equal(t, goImportRecoveryRows(coldGraph, file), goImportRecoveryRows(g, file)) + } + assert.Equal(t, goModuleLifecycleCall(t, coldGraph).To, goModuleLifecycleCall(t, g).To) + }) + } + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 5040d2618..5f43d2256 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -399,7 +399,7 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS cr.mu.Lock() clear(cr.goModuleImportState().packages) cr.mu.Unlock() - if err := cr.resolveGoModuleImportSweep(ctx, stats); err != nil { + if err := cr.resolveGoModuleImportSweep(ctx, stats, nil); err != nil { return stats, err } cr.mu.Lock() diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 9f1c7ed78..d96fe7176 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -65,35 +65,18 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed return stats, admission } + importBudget := &graph.IncomingSourceBudget{} // Provider recovery needs the lazy seed before the pass lock is taken. // Ordinary consumer frontiers keep local resolution ahead of the bridge. if len(definitionFiles) > 0 { - if err := cr.resolveGoModuleImportSweep(context.Background(), stats); err != nil { - cr.logger.Error("cross-repo resolve: seed Go module imports", zap.Error(err)) + if err := cr.resolveGoModuleImportSweep(context.Background(), stats, importBudget); err != nil { + cr.logger.Warn("cross-repo resolve: seed Go module imports deferred", zap.Error(err)) } } cr.mu.Lock() defer cr.mu.Unlock() budget := &graph.IncomingSourceBudget{} - importsByRepo := make(map[string][]string) - if len(definitionFiles) > 0 && len(cr.goModules) > 0 { - repos := make(map[string]bool) - for _, file := range definitionFiles { - if repo := graph.RepoPrefixOfID(file); cr.goModules[repo] != "" && !repos[repo] { - repos[repo] = true - ids := cr.goModuleImportIDs(repo) - admission.Keys += len(ids) - if err := budget.Charge(len(ids)); err != nil { - admission.Refused, admission.Refusal = true, err - admission.Dropped += len(ids) - break - } - admission.Inspected += len(ids) - importsByRepo[repo] = ids - } - } - } if len(resolutionFiles) > 0 { frontier := collectIncrementalFileFrontier(cr.graph, resolutionFiles, nil) admission.Keys += frontier.incomingAdmission.Keys @@ -149,7 +132,7 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed _ = cr.resolveGoModuleImports(context.Background(), resolutionFiles, stats, counted) } if len(definitionFiles) > 0 && len(cr.goModules) > 0 && !admission.Refused { - if frontierInspected := admission.Inspected - (graph.MaxIncomingSourceCandidateRows - budget.Remaining()); frontierInspected > 0 { + if frontierInspected := admission.Inspected; frontierInspected > 0 { if err := budget.Charge(frontierInspected); err != nil { admission.Refused, admission.Refusal = true, err } @@ -163,15 +146,6 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed admission.Dropped += fact.Dropped admission.Refused, admission.Refusal = fact.Refused, err logIncomingAdmissionRefusal(cr.logger, "go_module_incoming", fact.Keys, fact, err) - if !admission.Refused { - for repo, ids := range importsByRepo { - for start := 0; start < len(ids); start += externalCallMutationChunk { - if err := cr.restoreGoModuleImportBatch(context.Background(), repo, ids[start:min(start+externalCallMutationChunk, len(ids))], stats); err != nil { - cr.logger.Error("cross-repo resolve: restore Go module imports", zap.Error(err)) - } - } - } - } cr.resolveGoModuleEdgesLocked(edges, stats) } } @@ -182,6 +156,27 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed cr.logger.Error("cross-repo resolve: retry Go module terminals", zap.Error(err)) } } + // Import recovery has its own ceiling: a popular provider must not starve + // incoming calls or the revision-keyed call-terminal retry above. + repos := make(map[string]bool) + for _, file := range definitionFiles { + repo := graph.RepoPrefixOfID(file) + if cr.goModules[repo] == "" || repos[repo] { + continue + } + repos[repo] = true + ids := cr.goModuleImportIDs(repo) + if err := importBudget.Charge(len(ids)); err != nil { + cr.logger.Warn("cross-repo resolve: Go module imports deferred to full pass", zap.Error(err)) + continue + } + for start := 0; start < len(ids); start += externalCallMutationChunk { + if err := cr.restoreGoModuleImportBatch(context.Background(), repo, ids[start:min(start+externalCallMutationChunk, len(ids))], stats); err != nil { + cr.logger.Error("cross-repo resolve: restore Go module imports", zap.Error(err)) + } + } + } + DetectCrossRepoEdgesForMutationFiles(cr.graph, edgeSourceFiles, definitionFiles) return stats, admission } diff --git a/internal/resolver/go_module_import_recovery_test.go b/internal/resolver/go_module_import_recovery_test.go new file mode 100644 index 000000000..16acd7972 --- /dev/null +++ b/internal/resolver/go_module_import_recovery_test.go @@ -0,0 +1,170 @@ +package resolver + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestGoModuleImportBudgetDoesNotStarveCalls(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, restart := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/providerFirst=%v/restart=%v", backend, providerFirst, restart), func(t *testing.T) { + newStore := func() graph.Store { + t.Helper() + if backend == "memory" { + return graph.New() + } + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db + } + g := newStore() + modules := map[string]string{"a": "example.org/app", "b": "example.org/lib"} + state := &GoModuleDepRetry{} + resolver := func(store graph.Store, retry *GoModuleDepRetry) *CrossRepoResolver { + cr := NewCrossRepo(store) + cr.SetGoModules(modules) + cr.SetGoModuleDepRetry(retry, 1) + return cr + } + seed := func(store graph.Store) { + t.Helper() + if bulk, ok := store.(graph.BulkLoader); ok { + bulk.BeginBulkLoad() + defer func() { require.NoError(t, bulk.FlushBulk()) }() + } + var consumers []*graph.Node + var edges []*graph.Edge + for i := 0; i < graph.MaxIncomingSourceCandidateRows+1; i++ { + f := fmt.Sprintf("a/f%d.go", i) + consumers = append(consumers, &graph.Node{ID: f, Kind: graph.KindFile, FilePath: f, RepoPrefix: "a", Language: "go"}, &graph.Node{ID: f + "::import::example.org/lib", Kind: graph.KindImport, FilePath: f, RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/lib"}}) + edges = append(edges, &graph.Edge{From: f, To: "b/lib.go", Kind: graph.EdgeImports, FilePath: f, Line: 2, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/lib", "go_import_fallback": "dep::example.org/lib"}}) + } + consumers = append(consumers, &graph.Node{ID: "a/main.go::Caller", Name: "Caller", Kind: graph.KindFunction, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + consumers = append(consumers, + &graph.Node{ID: "a/new.go", Kind: graph.KindFile, FilePath: "a/new.go", RepoPrefix: "a", Language: "go"}, + &graph.Node{ID: "a/new.go::import::example.org/lib/new", Kind: graph.KindImport, FilePath: "a/new.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/lib/new"}}) + edges = append(edges, &graph.Edge{From: "a/new.go", To: "dep::example.org/lib", Kind: graph.EdgeImports, FilePath: "a/new.go", Line: 2}) + provider := &graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"} + if providerFirst { + store.AddNode(provider) + } + store.AddBatch(consumers, edges) + if !providerFirst { + store.AddNode(provider) + } + } + seed(g) + cr := resolver(g, state) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + // The provider now adds the missing package and function. + addNew := func(store graph.Store) { + store.AddBatch([]*graph.Node{ + {ID: "b/new/new.go", Kind: graph.KindFile, FilePath: "b/new/new.go", RepoPrefix: "b", Language: "go"}, + {ID: "b/new/new.go::New", Name: "New", Kind: graph.KindFunction, FilePath: "b/new/new.go", RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{ + {From: "a/main.go::Caller", To: "unresolved::extern::example.org/lib/new::New", Kind: graph.EdgeCalls, FilePath: "a/main.go", Line: 3}, + }) + } + addNew(g) + if restart { + state = &GoModuleDepRetry{} + } + cr = resolver(g, state) + // When the import recovery exceeds its own ceiling, the call leg must still bind. + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/new/new.go"}) + assert.False(t, admission.Refused, "import refusal is not a call admission refusal") + var calls []*graph.Edge + for _, e := range g.GetOutEdges("a/main.go::Caller") { + if e.Kind == graph.EdgeCalls { + calls = append(calls, e) + } + } + require.Len(t, calls, 1) + assert.Equal(t, "b/new/new.go::New", calls[0].To) + // Refused imports remain eligible for the next full pass, not lost or marked complete. + imports := g.GetOutEdges("a/f0.go") + require.Len(t, imports, 1) + assert.Equal(t, "b/lib.go", imports[0].To) + if !restart { + rows := g.GetOutEdges("a/new.go") + require.Len(t, rows, 1) + assert.Equal(t, "dep::example.org/lib", rows[0].To) + } + _, err := cr.ResolveAllContext(context.Background()) + require.NoError(t, err) + cold := newStore() + seed(cold) + addNew(cold) + _, err = resolver(cold, &GoModuleDepRetry{}).ResolveAllContext(t.Context()) + require.NoError(t, err) + files := []string{"a/new.go", "a/main.go::Caller"} + for i := 0; i < graph.MaxIncomingSourceCandidateRows+1; i++ { + files = append(files, fmt.Sprintf("a/f%d.go", i)) + } + project := func(store graph.Store) map[string][]string { + rows := make(map[string][]string) + for file, edges := range store.GetOutEdgesByNodeIDs(files) { + for _, e := range edges { + rows[file] = append(rows[file], fmt.Sprintf("%s %s cross=%v", e.Kind, e.To, e.CrossRepo)) + } + sort.Strings(rows[file]) + } + return rows + } + assert.Equal(t, project(cold), project(g), "all imports and the new call must match cold, including multiplicity") + freshImports := g.GetOutEdges("a/new.go") + require.Len(t, freshImports, 1, "one source import must bind exactly once") + for _, e := range freshImports { + assert.Equal(t, "b/new/new.go", e.To) + } + }) + } + } + } +} + +// A lazy restart seed must stop even when all import identities are filtered out. +func TestGoModuleImportSeedBoundAndCancellation(t *testing.T) { + for _, mode := range []string{"budget", "cancel"} { + t.Run(mode, func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + count := graph.MaxIncomingSourceCandidateRows + 10 + if mode == "cancel" { + count = externalCallMutationChunk*2 + 1 + } + var nodes []*graph.Node + for i := 0; i < count; i++ { + nodes = append(nodes, &graph.Node{ID: fmt.Sprintf("a/f%d.go::import::untracked.org/lib", i), Kind: graph.KindImport, FilePath: fmt.Sprintf("a/f%d.go", i), RepoPrefix: "a", Language: "go"}) + } + nodes = append(nodes, &graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + g.AddBatch(nodes, nil) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.org/lib"}) + if mode == "cancel" { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + g.afterBatch = cancel + require.ErrorIs(t, cr.RetryGoModuleDeps(ctx), context.Canceled) + assert.LessOrEqual(t, g.importRows, externalCallMutationChunk+1) + } else { + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/lib.go"}) + assert.False(t, admission.Refused) + assert.LessOrEqual(t, g.importRows, graph.MaxIncomingSourceCandidateRows+1) + } + require.NotNil(t, cr.goModuleRetry) + assert.False(t, cr.goModuleRetry.importsSeeded, "a partial seed must remain eligible for the full pass") + }) + } +} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 33188b13b..97610c4ed 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -197,11 +197,22 @@ func (cr *CrossRepoResolver) applyGoModuleImportReindexes(batch []graph.EdgeRein } // Filter identity-only rows before loading any file payload or adjacency. -func (cr *CrossRepoResolver) goModuleImportFileBatches(repos []string) iter.Seq[[]string] { - return func(yield func([]string) bool) { +// Charge every inspected identity, including unrelated and duplicate imports. +func (cr *CrossRepoResolver) goModuleImportFileBatches(ctx context.Context, repos []string, budget *graph.IncomingSourceBudget) iter.Seq2[[]string, error] { + return func(yield func([]string, error) bool) { seen := make(map[string]bool) var files []string for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, graph.KindImport) { + if err := ctx.Err(); err != nil { + yield(nil, err) + return + } + if budget != nil { + if err := budget.Charge(1); err != nil { + yield(nil, err) + return + } + } file, path, ok := strings.Cut(n.ID, "::import::") if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModulePaths, path) || seen[file] { continue @@ -212,14 +223,14 @@ func (cr *CrossRepoResolver) goModuleImportFileBatches(repos []string) iter.Seq[ seen[file] = true files = append(files, file) if len(files) == externalCallMutationChunk { - if !yield(files) { + if !yield(files, nil) { return } files = nil } } if len(files) > 0 { - yield(files) + yield(files, nil) } } } @@ -253,7 +264,8 @@ func (cr *CrossRepoResolver) goModuleImportState() *GoModuleDepRetry { // An unchanged warm restart skips global resolution. Public entry points seed // once before taking the pass lock; each candidate batch takes its own lock. -func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, stats *CrossRepoStats) error { +// Mutation passes supply their import budget; full passes supply nil. +func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, stats *CrossRepoStats, budget *graph.IncomingSourceBudget) error { cr.mu.Lock() state := cr.goModuleImportState() seeded := state.importsSeeded @@ -261,7 +273,10 @@ func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, sta if seeded || len(cr.goModules) == 0 { return ctx.Err() } - for files := range cr.goModuleImportFileBatches(nil) { + for files, err := range cr.goModuleImportFileBatches(ctx, nil, budget) { + if err != nil { + return err + } if err := ctx.Err(); err != nil { return err } @@ -372,6 +387,12 @@ func (cr *CrossRepoResolver) restoreGoModuleImportBatch(ctx context.Context, rep present = true break } + replacement, _ := goModuleReplacementFor(cr.goModuleReplacements, n.RepoPrefix, path) + if replacement.Directory != "" && graph.RepoPrefixOfID(e.To) == n.RepoPrefix && + strings.HasSuffix(e.To, ".go") && filePathDir(e.To) == replacement.Directory { + present = true + break + } // A longer module may own the bound package, even when this // statement is also registered under the edited provider. targetRepo := graph.RepoPrefixOfID(e.To) diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index ede1ff62a..a469a51b8 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -519,7 +519,7 @@ func (cr *CrossRepoResolver) RetryGoModuleDeps(ctx context.Context) error { }(); err != nil { return err } - return cr.resolveGoModuleImportSweep(ctx, stats) + return cr.resolveGoModuleImportSweep(ctx, stats, nil) } // ResolveGoModuleIncomingForRepo probes the provider's definitions and retained @@ -529,7 +529,7 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { if len(cr.goModules) == 0 { return nil } - if err := cr.resolveGoModuleImportSweep(context.Background(), nil); err != nil { + if err := cr.resolveGoModuleImportSweep(context.Background(), nil, nil); err != nil { return err } if cr.goModules[repo] != "" && len(cr.graph.RepoPrefixes()) >= 2 { @@ -549,7 +549,10 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { } } } - for files := range cr.goModuleImportFileBatches([]string{repo}) { + for files, err := range cr.goModuleImportFileBatches(context.Background(), []string{repo}, nil) { + if err != nil { + return err + } func() { cr.mu.Lock() defer cr.mu.Unlock() From 91d0312cc80782e134bdc8726c8bb0b810ec16d8 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 05:22:33 -0700 Subject: [PATCH 006/111] fix(resolver): finish Go import registry seeding before provider recovery Seed retained import identities once without hydrating files or rewriting edges. Keep provider import recovery on its own bounded budget so a large workspace cannot disable recovery after a warm restart. --- internal/resolver/cross_repo.go | 2 +- internal/resolver/cross_repo_incremental.go | 2 +- .../go_module_import_recovery_test.go | 15 ++-- .../resolver/go_module_import_seed_test.go | 89 +++++++++++++++++++ internal/resolver/go_module_imports.go | 40 ++++++--- internal/resolver/go_modules_cross_repo.go | 6 +- 6 files changed, 133 insertions(+), 21 deletions(-) create mode 100644 internal/resolver/go_module_import_seed_test.go diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 5f43d2256..5040d2618 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -399,7 +399,7 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS cr.mu.Lock() clear(cr.goModuleImportState().packages) cr.mu.Unlock() - if err := cr.resolveGoModuleImportSweep(ctx, stats, nil); err != nil { + if err := cr.resolveGoModuleImportSweep(ctx, stats); err != nil { return stats, err } cr.mu.Lock() diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index d96fe7176..04f52d5b4 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -69,7 +69,7 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed // Provider recovery needs the lazy seed before the pass lock is taken. // Ordinary consumer frontiers keep local resolution ahead of the bridge. if len(definitionFiles) > 0 { - if err := cr.resolveGoModuleImportSweep(context.Background(), stats, importBudget); err != nil { + if err := cr.seedGoModuleImportRegistry(context.Background()); err != nil { cr.logger.Warn("cross-repo resolve: seed Go module imports deferred", zap.Error(err)) } } diff --git a/internal/resolver/go_module_import_recovery_test.go b/internal/resolver/go_module_import_recovery_test.go index 16acd7972..92286fe8a 100644 --- a/internal/resolver/go_module_import_recovery_test.go +++ b/internal/resolver/go_module_import_recovery_test.go @@ -135,9 +135,9 @@ func TestGoModuleImportBudgetDoesNotStarveCalls(t *testing.T) { } } -// A lazy restart seed must stop even when all import identities are filtered out. -func TestGoModuleImportSeedBoundAndCancellation(t *testing.T) { - for _, mode := range []string{"budget", "cancel"} { +// A restart seeds all identities once; cancellation still stops filtered rows. +func TestGoModuleImportSeedCompletionAndCancellation(t *testing.T) { + for _, mode := range []string{"complete", "cancel"} { t.Run(mode, func(t *testing.T) { g := &goImportCountingStore{Store: graph.New()} count := graph.MaxIncomingSourceCandidateRows + 10 @@ -161,10 +161,15 @@ func TestGoModuleImportSeedBoundAndCancellation(t *testing.T) { } else { _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/lib.go"}) assert.False(t, admission.Refused) - assert.LessOrEqual(t, g.importRows, graph.MaxIncomingSourceCandidateRows+1) + assert.Equal(t, count, g.importRows) + assert.True(t, cr.goModuleRetry.importsSeeded) + cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/lib.go"}) + assert.Equal(t, 1, g.importScans) } require.NotNil(t, cr.goModuleRetry) - assert.False(t, cr.goModuleRetry.importsSeeded, "a partial seed must remain eligible for the full pass") + if mode == "cancel" { + assert.False(t, cr.goModuleRetry.importsSeeded, "a cancelled seed remains eligible for retry") + } }) } } diff --git a/internal/resolver/go_module_import_seed_test.go b/internal/resolver/go_module_import_seed_test.go new file mode 100644 index 000000000..c3606c377 --- /dev/null +++ b/internal/resolver/go_module_import_seed_test.go @@ -0,0 +1,89 @@ +package resolver + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +// Given more unrelated import identities than a save's recovery ceiling, the +// first save after restart must still discover and recover a small provider. +func TestGoModuleImportLargeRestartSeed(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"new_package", "representative"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + newStore := func() graph.Store { + t.Helper() + if backend == "memory" { + return graph.New() + } + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db + } + var nodes []*graph.Node + for i := 0; i < graph.MaxIncomingSourceCandidateRows+1; i++ { + file := fmt.Sprintf("noise/f%d.go", i) + nodes = append(nodes, &graph.Node{ID: file + "::import::untracked.org/library", Kind: graph.KindImport, FilePath: file, RepoPrefix: "noise", Language: "go"}) + } + path, target := "example.org/lib/sub", "dep::example.org/lib" + first, second := "b/sub/z.go", "b/sub/a.go" + if mode == "representative" { + path, target = "example.org/lib", "b/z.go" + first, second = "b/a.go", "b/0.go" + } + nodes = append(nodes, + &graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, + &graph.Node{ID: "a/main.go::import::" + path, Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": path}}, + &graph.Node{ID: "b/z.go", Kind: graph.KindFile, FilePath: "b/z.go", RepoPrefix: "b", Language: "go"}) + edge := &graph.Edge{From: "a/main.go", To: target, Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2} + if mode == "representative" { + edge.CrossRepo = true + edge.Meta = map[string]any{"go_import": path, "go_import_fallback": "dep::example.org/lib"} + } + g := &goImportCountingStore{Store: newStore()} + g.AddBatch(nodes, []*graph.Edge{edge}) + state := &GoModuleDepRetry{} // unchanged warm restart has no process-local registry + resolver := func(store graph.Store, retry *GoModuleDepRetry) *CrossRepoResolver { + cr := NewCrossRepo(store) + cr.SetGoModules(map[string]string{"a": "example.org/app", "b": "example.org/lib"}) + cr.SetGoModuleDepRetry(retry, 1) + return cr + } + for _, file := range []string{first, second} { + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "b", Language: "go"}) + cr := resolver(g, state) + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{file}) + assert.False(t, admission.Refused) + rows := g.GetOutEdges("a/main.go") + require.Len(t, rows, 1) + assert.Equal(t, file, rows[0].To, "save must recover the new package or representative without a full pass") + assert.True(t, rows[0].CrossRepo) + assert.True(t, state.importsSeeded, "first save must complete the registry even above the recovery ceiling") + } + assert.Equal(t, 1, g.importScans, "later saves must not repeat the global identity scan") + assert.Equal(t, len(nodes)-2, g.importRows, "visit all import identities once, not only the first budget-sized prefix") + cold := newStore() + cold.AddBatch(nodes, []*graph.Edge{edge}) + for _, file := range []string{first, second} { + cold.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "b", Language: "go"}) + } + _, err := resolver(cold, &GoModuleDepRetry{}).ResolveAllContext(context.Background()) + require.NoError(t, err) + warmRows, coldRows := g.GetOutEdges("a/main.go"), cold.GetOutEdges("a/main.go") + require.Len(t, coldRows, 1) + require.Len(t, warmRows, 1) + assert.Equal(t, coldRows[0].To, warmRows[0].To) + assert.Equal(t, coldRows[0].CrossRepo, warmRows[0].CrossRepo) + }) + } + } +} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 97610c4ed..ecad9bdf2 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -197,8 +197,7 @@ func (cr *CrossRepoResolver) applyGoModuleImportReindexes(batch []graph.EdgeRein } // Filter identity-only rows before loading any file payload or adjacency. -// Charge every inspected identity, including unrelated and duplicate imports. -func (cr *CrossRepoResolver) goModuleImportFileBatches(ctx context.Context, repos []string, budget *graph.IncomingSourceBudget) iter.Seq2[[]string, error] { +func (cr *CrossRepoResolver) goModuleImportFileBatches(ctx context.Context, repos []string) iter.Seq2[[]string, error] { return func(yield func([]string, error) bool) { seen := make(map[string]bool) var files []string @@ -207,12 +206,6 @@ func (cr *CrossRepoResolver) goModuleImportFileBatches(ctx context.Context, repo yield(nil, err) return } - if budget != nil { - if err := budget.Charge(1); err != nil { - yield(nil, err) - return - } - } file, path, ok := strings.Cut(n.ID, "::import::") if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModulePaths, path) || seen[file] { continue @@ -262,10 +255,35 @@ func (cr *CrossRepoResolver) goModuleImportState() *GoModuleDepRetry { return state } +// Seed the save-time registry once, without hydrating files or mutating edges. +// A workspace-sized identity scan must not consume a provider's recovery budget. +func (cr *CrossRepoResolver) seedGoModuleImportRegistry(ctx context.Context) error { + cr.mu.Lock() + defer cr.mu.Unlock() + state := cr.goModuleImportState() + if state.importsSeeded || len(cr.goModules) == 0 { + return ctx.Err() + } + for n := range graph.RepoNodeIdentitiesSeq(cr.graph, nil, graph.KindImport) { + if err := ctx.Err(); err != nil { + return err + } + file, path, ok := strings.Cut(n.ID, "::import::") + if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModulePaths, path) { + continue + } + if module := cr.goModules[n.RepoPrefix]; module != "" && (path == module || strings.HasPrefix(path, module+"/")) { + continue + } + cr.rememberGoModuleImports([]*graph.Node{{ID: n.ID, RepoPrefix: n.RepoPrefix, Kind: graph.KindImport, Language: "go"}}) + } + state.importsSeeded = ctx.Err() == nil + return ctx.Err() +} + // An unchanged warm restart skips global resolution. Public entry points seed // once before taking the pass lock; each candidate batch takes its own lock. -// Mutation passes supply their import budget; full passes supply nil. -func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, stats *CrossRepoStats, budget *graph.IncomingSourceBudget) error { +func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, stats *CrossRepoStats) error { cr.mu.Lock() state := cr.goModuleImportState() seeded := state.importsSeeded @@ -273,7 +291,7 @@ func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, sta if seeded || len(cr.goModules) == 0 { return ctx.Err() } - for files, err := range cr.goModuleImportFileBatches(ctx, nil, budget) { + for files, err := range cr.goModuleImportFileBatches(ctx, nil) { if err != nil { return err } diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index a469a51b8..d1ba98b3f 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -519,7 +519,7 @@ func (cr *CrossRepoResolver) RetryGoModuleDeps(ctx context.Context) error { }(); err != nil { return err } - return cr.resolveGoModuleImportSweep(ctx, stats, nil) + return cr.resolveGoModuleImportSweep(ctx, stats) } // ResolveGoModuleIncomingForRepo probes the provider's definitions and retained @@ -529,7 +529,7 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { if len(cr.goModules) == 0 { return nil } - if err := cr.resolveGoModuleImportSweep(context.Background(), nil, nil); err != nil { + if err := cr.resolveGoModuleImportSweep(context.Background(), nil); err != nil { return err } if cr.goModules[repo] != "" && len(cr.graph.RepoPrefixes()) >= 2 { @@ -549,7 +549,7 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { } } } - for files, err := range cr.goModuleImportFileBatches(context.Background(), []string{repo}, nil) { + for files, err := range cr.goModuleImportFileBatches(context.Background(), []string{repo}) { if err != nil { return err } From 1376ff3581a93ae163ee27b96a6343e2be514e86 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 05:47:47 -0700 Subject: [PATCH 007/111] fix(resolver): recover Go imports when module identity ties change Recover retained imports for the existing module-change repository set, not only the saved repository. Bound this recovery independently so an import refusal cannot suppress incoming call binding or terminal retry. --- internal/indexer/go_module_import_tie_test.go | 53 +++++++++++++++++++ internal/resolver/go_modules_cross_repo.go | 20 +++++++ 2 files changed, 73 insertions(+) create mode 100644 internal/indexer/go_module_import_tie_test.go diff --git a/internal/indexer/go_module_import_tie_test.go b/internal/indexer/go_module_import_tie_test.go new file mode 100644 index 000000000..4020d7e49 --- /dev/null +++ b/internal/indexer/go_module_import_tie_test.go @@ -0,0 +1,53 @@ +package indexer + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/search" +) + +func TestGoModuleImportTieBrokenByRename(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a := repos[0].Path + writeFile(t, filepath.Join(a, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\n") + entries := []config.RepoEntry{repos[0], repos[1], repos[2]} + for _, e := range entries { + _, err := mi.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + mi.RunGlobalResolve() + t.Logf("tie: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + manifest := filepath.Join(repos[2].Path, "go.mod") + writeFile(t, manifest, "module example.org/forked\n") + require.NoError(t, mi.GetIndexer("fork").IndexFile(manifest)) + t.Logf("after rename: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + for _, e := range entries { + _, err := cold.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + cold.RunGlobalResolve() + t.Logf("cold: %v call=%s", goImportRecoveryRows(coldGraph, "a/main.go"), goModuleLifecycleCall(t, coldGraph).To) + assert.Equal(t, goImportRecoveryRows(coldGraph, "a/main.go"), goImportRecoveryRows(g, "a/main.go")) + }) + } +} diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index d1ba98b3f..b69febe17 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -7,6 +7,8 @@ import ( "strings" "sync" + "go.uber.org/zap" + "github.com/zzet/gortex/internal/graph" ) @@ -655,6 +657,24 @@ func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previo if len(ids) > 0 { flush() } + // A changed module can make another provider's fallback imports bindable, + // even when that provider is outside the saved-file frontier. + importBudget := &graph.IncomingSourceBudget{} + for _, repo := range repos { + if cr.goModules[repo] == "" { + continue + } + importIDs := cr.goModuleImportIDs(repo) + if err := importBudget.Charge(len(importIDs)); err != nil { + cr.logger.Warn("Go module import identity recheck deferred to full pass", zap.Error(err)) + continue + } + for start := 0; start < len(importIDs); start += externalCallMutationChunk { + if err := cr.restoreGoModuleImportBatch(ctx, repo, importIDs[start:min(start+externalCallMutationChunk, len(importIDs))], stats); err != nil { + return err + } + } + } return ctx.Err() } From 3fdf71537d1728622ffff27417a9174f759e2702 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 06:31:52 -0700 Subject: [PATCH 008/111] fix(resolver): preserve Go import state during full resolution and restubbing --- .../go_module_import_lifecycle_test.go | 6 +- internal/indexer/incremental_batch.go | 2 +- internal/indexer/indexer.go | 3 + internal/resolver/cross_repo.go | 8 +- .../resolver/go_module_import_review_test.go | 108 ++++++++++++++++++ internal/resolver/go_module_imports.go | 17 ++- 6 files changed, 135 insertions(+), 9 deletions(-) create mode 100644 internal/resolver/go_module_import_review_test.go diff --git a/internal/indexer/go_module_import_lifecycle_test.go b/internal/indexer/go_module_import_lifecycle_test.go index f9aa2917e..6cde7050a 100644 --- a/internal/indexer/go_module_import_lifecycle_test.go +++ b/internal/indexer/go_module_import_lifecycle_test.go @@ -144,7 +144,7 @@ func TestGoModuleImportParkBeforeEviction(t *testing.T) { g.AddBatch([]*graph.Node{ {ID: "a/main.go", Name: "main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, {ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}, - }, []*graph.Edge{{From: "a/main.go", To: "b/lib.go", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/lib"}}}) + }, []*graph.Edge{{From: "a/main.go", To: "b/lib.go", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2, CrossRepo: true, Confidence: 0.6, ConfidenceLabel: "EXTRACTED", Origin: graph.OriginASTInferred, Meta: map[string]any{"go_import": "example.org/lib"}}}) if route == "repo" { restubGoModuleRepoIncoming(g, "b") } else if route == "deleted_batch" { @@ -163,6 +163,10 @@ func TestGoModuleImportParkBeforeEviction(t *testing.T) { rows := g.GetOutEdges("a/main.go") require.Len(t, rows, 1, "the consumer-owned row must survive provider eviction") assert.Equal(t, "unresolved::import::example.org/lib", rows[0].To) + assert.False(t, rows[0].CrossRepo) + assert.Empty(t, rows[0].ConfidenceLabel) + assert.Empty(t, rows[0].Origin) + assert.Zero(t, rows[0].Confidence) assert.Equal(t, "example.org/lib", rows[0].Meta["go_import"]) g.AddNode(&graph.Node{ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) cr := resolver.NewCrossRepo(g) diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 8edf04be3..dfe8b67dd 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -1404,7 +1404,7 @@ func restubIncomingRefsFromView( if stub := goModuleRestubTarget(edge, ""); stub != "" { oldTo := edge.To graph.StashRestubProvenance(edge) - edge.To, edge.CrossRepo = stub, false + edge.To, edge.CrossRepo, edge.ConfidenceLabel = stub, false, "" reindexes = append(reindexes, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) } } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 88fb3f339..70d8f8fde 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -6000,6 +6000,9 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) []string { // moved target leaves the stub honestly unresolved. graph.StashRestubProvenance(e) e.To = goModuleRestubTarget(e, stub) + if e.Kind == graph.EdgeImports { + e.CrossRepo, e.ConfidenceLabel = false, "" + } batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) } } diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 5040d2618..5c006047b 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -397,9 +397,13 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS // Whole-graph fallback has no exact provider frontier. Discard package // projections even if the module revision's terminal retry is complete. cr.mu.Lock() - clear(cr.goModuleImportState().packages) + if len(cr.goModules) > 0 || cr.goModuleRetry != nil { + state := cr.goModuleImportState() + clear(state.packages) + state.importsSeeded = false // A full pass must include newly published consumers. + } cr.mu.Unlock() - if err := cr.resolveGoModuleImportSweep(ctx, stats); err != nil { + if err := cr.seedGoModuleImportRegistry(ctx); err != nil { return stats, err } cr.mu.Lock() diff --git a/internal/resolver/go_module_import_review_test.go b/internal/resolver/go_module_import_review_test.go new file mode 100644 index 000000000..b9072e3a8 --- /dev/null +++ b/internal/resolver/go_module_import_review_test.go @@ -0,0 +1,108 @@ +package resolver + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func goImportReviewStore(t *testing.T, backend string) graph.Store { + t.Helper() + if backend == "memory" { + return graph.New() + } + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "review.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + return s +} + +func TestGoModuleImportFullPassNewConsumer(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + g.AddNode(&graph.Node{ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + state := &GoModuleDepRetry{} + resolve := func() { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.org/lib"}) + cr.SetGoModuleDepRetry(state, 1) + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + } + for _, repo := range []string{"a", "c"} { + file := repo + "/main.go" + g.AddBatch([]*graph.Node{ + {ID: file, Name: "main.go", Kind: graph.KindFile, FilePath: file, RepoPrefix: repo, Language: "go"}, + {ID: file + "::import::example.org/lib", Name: "lib", Kind: graph.KindImport, FilePath: file, RepoPrefix: repo, Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/lib"}}, + }, []*graph.Edge{{From: file, To: "dep::example.org/lib", Kind: graph.EdgeImports, FilePath: file, Line: 2}}) + resolve() + rows := g.GetOutEdges(file) + require.Len(t, rows, 1) + assert.Equal(t, "b/lib.go", rows[0].To, "a full pass must include current import identities even with shared retry state") + assert.True(t, rows[0].CrossRepo) + } + }) + } +} + +func TestGoModuleImportUnconfiguredRetryState(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"full", "consumer", "provider", "retry"} { + for _, configured := range []bool{false, true} { + name := backend + "/" + route + "/unconfigured" + if configured { + name = backend + "/" + route + "/removed_modules" + } + t.Run(name, func(t *testing.T) { + g := goImportReviewStore(t, backend) + g.AddBatch([]*graph.Node{ + {ID: "a/main.go", Name: "main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, + {ID: "a/main.go::Caller", Name: "Caller", Kind: graph.KindFunction, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, + {ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}, + {ID: "b/lib.go::Hello", Name: "Hello", Kind: graph.KindFunction, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{{From: "a/main.go::Caller", To: "dep::example.org/lib::Hello", Kind: graph.EdgeCalls, FilePath: "a/main.go", Line: 3}}) + bound := NewCrossRepo(g) + bound.SetGoModules(map[string]string{"b": "example.org/lib"}) + _, err := bound.ResolveAllContext(t.Context()) + require.NoError(t, err) + require.Equal(t, "b/lib.go::Hello", g.GetOutEdges("a/main.go::Caller")[0].To) + cr := NewCrossRepo(g) + if configured { + cr.SetGoModules(map[string]string{}) + cr.SetGoModuleDepRetry(&GoModuleDepRetry{}, 1) + } + switch route { + case "consumer": + cr.ResolveMutationFrontiers([]string{"a/main.go"}, nil, nil) + case "provider": + cr.ResolveMutationFrontiers(nil, nil, []string{"b/lib.go"}) + case "retry": + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + } + _, err = cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + var rows []*graph.Edge + for _, edge := range g.GetOutEdges("a/main.go::Caller") { + if edge.Kind == graph.EdgeCalls { + rows = append(rows, edge) + } + } + require.Len(t, rows, 1) + if configured { + assert.Equal(t, "dep::example.org/lib::Hello", rows[0].To) + assert.False(t, rows[0].CrossRepo) + } else { + assert.Equal(t, "b/lib.go::Hello", rows[0].To) + assert.True(t, rows[0].CrossRepo) + assert.Nil(t, cr.goModuleRetry, "no module context must not allocate retry state") + } + }) + } + } + } +} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index ecad9bdf2..9dd157a17 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -14,7 +14,7 @@ import ( // counted records whether the ordinary frontier pass already counted an import // as resolved (true) or unresolved (false), before this pass retargets it. func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files []string, stats *CrossRepoStats, counted map[graph.EdgeIdentity]bool) error { - if len(files) == 0 { + if len(files) == 0 || len(cr.goModules) == 0 && cr.goModuleRetry == nil { return nil } cr.goModuleImportState() @@ -255,13 +255,17 @@ func (cr *CrossRepoResolver) goModuleImportState() *GoModuleDepRetry { return state } -// Seed the save-time registry once, without hydrating files or mutating edges. -// A workspace-sized identity scan must not consume a provider's recovery budget. +// Seed import identities without hydrating files or mutating edges. Saves seed +// once; whole-graph passes invalidate the seed. This scan must not consume a +// provider's recovery budget. func (cr *CrossRepoResolver) seedGoModuleImportRegistry(ctx context.Context) error { + if len(cr.goModules) == 0 { + return ctx.Err() + } cr.mu.Lock() defer cr.mu.Unlock() state := cr.goModuleImportState() - if state.importsSeeded || len(cr.goModules) == 0 { + if state.importsSeeded { return ctx.Err() } for n := range graph.RepoNodeIdentitiesSeq(cr.graph, nil, graph.KindImport) { @@ -284,11 +288,14 @@ func (cr *CrossRepoResolver) seedGoModuleImportRegistry(ctx context.Context) err // An unchanged warm restart skips global resolution. Public entry points seed // once before taking the pass lock; each candidate batch takes its own lock. func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, stats *CrossRepoStats) error { + if len(cr.goModules) == 0 { + return ctx.Err() + } cr.mu.Lock() state := cr.goModuleImportState() seeded := state.importsSeeded cr.mu.Unlock() - if seeded || len(cr.goModules) == 0 { + if seeded { return ctx.Err() } for files, err := range cr.goModuleImportFileBatches(ctx, nil) { From b493d9b9e0cc2e05e03f47cccecf5ddfe1f558c7 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 06:51:10 -0700 Subject: [PATCH 009/111] fix(resolver): rebind Go imports when local replacements are added --- .../indexer/go_module_import_recovery_test.go | 43 +++++++++++++++++++ .../resolver/go_module_import_review_test.go | 29 +++++++++++++ internal/resolver/go_module_imports.go | 28 ++++++++++-- 3 files changed, 96 insertions(+), 4 deletions(-) diff --git a/internal/indexer/go_module_import_recovery_test.go b/internal/indexer/go_module_import_recovery_test.go index 8baaccf10..f5972cc6f 100644 --- a/internal/indexer/go_module_import_recovery_test.go +++ b/internal/indexer/go_module_import_recovery_test.go @@ -163,3 +163,46 @@ func TestGoModuleImportPopularProviderSave(t *testing.T) { } } } + +func TestGoModuleImportReplaceAddedLater(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a := repos[0].Path + manifest := filepath.Join(a, "go.mod") + writeFile(t, manifest, "module example.org/caller\nrequire example.org/library v1.0.0\n") + entries := []config.RepoEntry{repos[0], repos[1]} + for _, e := range entries { + _, err := mi.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + t.Logf("linked: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + require.NoError(t, os.MkdirAll(filepath.Join(a, "third_party", "library"), 0o755)) + vendored := filepath.Join(a, "third_party", "library", "lib.go") + writeFile(t, vendored, "package library\nfunc Hello() {}\n") + writeFile(t, manifest, "module example.org/caller\nrequire example.org/library v1.0.0\nreplace example.org/library => ./third_party/library\n") + _, err := mi.IncrementalReindexRepo("a", []string{manifest, vendored}) + require.NoError(t, err) + t.Logf("after replace save: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + mi.RunGlobalResolve() + t.Logf("after global: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + for _, e := range entries { + _, err := cold.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + cold.RunGlobalResolve() + require.Len(t, goImportRecoveryRows(coldGraph, "a/main.go"), 1) + t.Logf("cold: %v call=%s", goImportRecoveryRows(coldGraph, "a/main.go"), goModuleLifecycleCall(t, coldGraph).To) + assert.Equal(t, goImportRecoveryRows(coldGraph, "a/main.go"), goImportRecoveryRows(g, "a/main.go")) + }) + } +} diff --git a/internal/resolver/go_module_import_review_test.go b/internal/resolver/go_module_import_review_test.go index b9072e3a8..7f98db968 100644 --- a/internal/resolver/go_module_import_review_test.go +++ b/internal/resolver/go_module_import_review_test.go @@ -106,3 +106,32 @@ func TestGoModuleImportUnconfiguredRetryState(t *testing.T) { } } } + +func TestGoModuleImportReplacementDirectory(t *testing.T) { + for _, tc := range []struct{ name, directory, path, want string }{ + {"local", "a/third_party/library", "example.org/library", "a/third_party/library/lib.go"}, + {"sub_package", "a/third_party/library", "example.org/library/sub", "a/third_party/library/sub/a.go"}, + {"denied", "", "example.org/library", ""}, + {"missing", "a/missing", "example.org/library", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + g := graph.New() + for _, file := range []string{"a/third_party/library/lib.go", "a/third_party/library/sub/a.go", "b/lib.go", "b/sub/a.go"} { + repo := file[:1] + g.AddNode(&graph.Node{ID: file, Name: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: repo, Language: "go"}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"a": "example.org/caller", "b": "example.org/library"}) + cr.SetGoModuleReplacements(map[string][]GoModuleReplacement{"a": {{Module: "example.org/library", Directory: tc.directory}}}) + cr.goModuleImportState() + match := cr.goModuleImportCandidate(tc.path, "a", "") + if tc.want == "" { + assert.Nil(t, match, "a replacement must never fall through to the remote module") + } else { + require.NotNil(t, match) + assert.Equal(t, tc.want, match.ID) + assert.Equal(t, "a", match.RepoPrefix) + } + }) + } +} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 9dd157a17..6576c5c3d 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -53,6 +53,10 @@ func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files [ if match != nil { copy.To = match.ID copy.CrossRepo = caller != "" && match.RepoPrefix != caller + if match.RepoPrefix == caller { + delete(copy.Meta, "go_import") + delete(copy.Meta, "go_import_fallback") + } } else { copy.To, _ = copy.Meta["go_import_fallback"].(string) if copy.To == "" || graph.IsUnresolvedTarget(copy.To) { @@ -135,7 +139,15 @@ func (cr *CrossRepoResolver) goModuleImportCandidate(importPath, caller, workspa state := cr.goModuleRetry var candidates []*graph.Node module := cr.goModulePaths.longest(importPath) - for _, repo := range cr.goModuleRepos[module] { + replacement, replaced := goModuleReplacementFor(cr.goModuleReplacements, caller, importPath) + repos := cr.goModuleRepos[module] + if replaced { + if replacement.Directory == "" { + return nil + } + repos = []string{caller} + } + for _, repo := range repos { packages, cached := state.packages[repo] if !cached { packages = make(map[string]graph.FileNodeIdentity) @@ -150,11 +162,19 @@ func (cr *CrossRepoResolver) goModuleImportCandidate(importPath, caller, workspa } state.packages[repo] = packages } - if file, ok := packages[repo+strings.TrimPrefix(importPath, module)]; ok { - candidates = append(candidates, &graph.Node{ID: file.ID, Kind: graph.KindFile, FilePath: file.FilePath, RepoPrefix: file.RepoPrefix, WorkspaceID: file.WorkspaceID, Language: "go"}) + dir := repo + strings.TrimPrefix(importPath, module) + if replaced { + dir = replacement.Directory + } + if file, ok := packages[dir]; ok { + candidate := &graph.Node{ID: file.ID, Kind: graph.KindFile, FilePath: file.FilePath, RepoPrefix: file.RepoPrefix, WorkspaceID: file.WorkspaceID, Language: "go"} + if replaced { + return candidate + } + candidates = append(candidates, candidate) } } - if _, replaced := goModuleReplacementFor(cr.goModuleReplacements, caller, importPath); replaced { + if replaced { return nil } return goModuleCandidate(cr.goModules, importPath, candidates, caller, workspace, cr.crossWorkspaceLookup) From b0b30ee6634e93acafdaa7e9df00150589f2edc5 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 07:00:30 -0700 Subject: [PATCH 010/111] fix(resolver): retain Go import retry metadata across replacement cycles --- .../indexer/go_module_import_recovery_test.go | 59 +++++++++++++++++++ internal/resolver/go_module_imports.go | 4 -- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/internal/indexer/go_module_import_recovery_test.go b/internal/indexer/go_module_import_recovery_test.go index f5972cc6f..3366643fe 100644 --- a/internal/indexer/go_module_import_recovery_test.go +++ b/internal/indexer/go_module_import_recovery_test.go @@ -202,7 +202,66 @@ func TestGoModuleImportReplaceAddedLater(t *testing.T) { cold.RunGlobalResolve() require.Len(t, goImportRecoveryRows(coldGraph, "a/main.go"), 1) t.Logf("cold: %v call=%s", goImportRecoveryRows(coldGraph, "a/main.go"), goModuleLifecycleCall(t, coldGraph).To) + // Retain retry metadata on the local row so removing replace can rebind it. + assert.Equal(t, goImportBindingRows(coldGraph, "a/main.go"), goImportBindingRows(g, "a/main.go")) + }) + } +} + +func TestGoModuleImportReplaceAddedThenRemoved(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a := repos[0].Path + manifest := filepath.Join(a, "go.mod") + plain := "module example.org/caller\nrequire example.org/library v1.0.0\n" + writeFile(t, manifest, plain) + entries := []config.RepoEntry{repos[0], repos[1]} + for _, e := range entries { + _, err := mi.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + t.Logf("linked: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + require.NoError(t, os.MkdirAll(filepath.Join(a, "third_party", "library"), 0o755)) + vendored := filepath.Join(a, "third_party", "library", "lib.go") + writeFile(t, vendored, "package library\nfunc Hello() {}\n") + writeFile(t, manifest, plain+"replace example.org/library => ./third_party/library\n") + _, err := mi.IncrementalReindexRepo("a", []string{manifest, vendored}) + require.NoError(t, err) + t.Logf("replace added: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + writeFile(t, manifest, plain) + _, err = mi.IncrementalReindexRepo("a", []string{manifest}) + require.NoError(t, err) + t.Logf("replace removed: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + mi.RunGlobalResolve() + t.Logf("after global: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + for _, e := range entries { + _, err := cold.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + cold.RunGlobalResolve() + t.Logf("cold: %v call=%s", goImportRecoveryRows(coldGraph, "a/main.go"), goModuleLifecycleCall(t, coldGraph).To) assert.Equal(t, goImportRecoveryRows(coldGraph, "a/main.go"), goImportRecoveryRows(g, "a/main.go")) }) } } + +func goImportBindingRows(g graph.Store, file string) []string { + var rows []string + for _, e := range g.GetOutEdges(file) { + if e.Kind == graph.EdgeImports { + rows = append(rows, fmt.Sprintf("L%d %s cross=%v conf=%v origin=%s", e.Line, e.To, e.CrossRepo, e.Confidence, e.Origin)) + } + } + sort.Strings(rows) + return rows +} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 6576c5c3d..d1ec2305a 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -53,10 +53,6 @@ func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files [ if match != nil { copy.To = match.ID copy.CrossRepo = caller != "" && match.RepoPrefix != caller - if match.RepoPrefix == caller { - delete(copy.Meta, "go_import") - delete(copy.Meta, "go_import_fallback") - } } else { copy.To, _ = copy.Meta["go_import_fallback"].(string) if copy.To == "" || graph.IsUnresolvedTarget(copy.To) { From 76c815258f1150082935477a24a2f4876260c7fd Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 07:24:12 -0700 Subject: [PATCH 011/111] test(indexer): adapt Go import restub test to the new return values --- internal/indexer/go_module_import_lifecycle_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/indexer/go_module_import_lifecycle_test.go b/internal/indexer/go_module_import_lifecycle_test.go index 6cde7050a..85b8a8f4d 100644 --- a/internal/indexer/go_module_import_lifecycle_test.go +++ b/internal/indexer/go_module_import_lifecycle_test.go @@ -149,7 +149,7 @@ func TestGoModuleImportParkBeforeEviction(t *testing.T) { restubGoModuleRepoIncoming(g, "b") } else if route == "deleted_batch" { provider := g.GetNode("b/lib.go") - carried := restubIncomingRefsFromView(g, + carried, _ := restubIncomingRefsFromView(g, []*incrementalBatchStage{{graphPath: "b/lib.go", priorNodes: []*graph.Node{provider}}}, incrementalPriorView{inByNode: g.GetInEdgesByNodeIDs([]string{provider.ID})}) assert.Empty(t, carried, "a deleted file cannot carry a resolved target") From a4769d07c0a05363695470aaf204b1e470adea53 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 11:42:33 -0700 Subject: [PATCH 012/111] fix(resolver): bind duplicate Go import aliases --- .../indexer/go_module_import_recovery_test.go | 48 +++++++++++++++++++ internal/resolver/go_module_imports.go | 16 +++++-- internal/resolver/go_module_imports_test.go | 38 ++++++++++++++- 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/internal/indexer/go_module_import_recovery_test.go b/internal/indexer/go_module_import_recovery_test.go index 3366643fe..381f3e336 100644 --- a/internal/indexer/go_module_import_recovery_test.go +++ b/internal/indexer/go_module_import_recovery_test.go @@ -265,3 +265,51 @@ func goImportBindingRows(g graph.Store, file string) []string { sort.Strings(rows) return rows } + +func TestGoModuleImportDuplicateAliases(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scope := range []string{"same_repo", "cross_repo"} { + t.Run(backend+"/"+scope, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + writeFile(t, filepath.Join(repos[0].Path, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\n") + provider, target, importPath := repos[1].Path, "b/sub/lib.go", "example.org/library/sub" + if scope == "same_repo" { + provider, target, importPath = repos[0].Path, "a/sub/lib.go", "example.org/caller/sub" + } + require.NoError(t, os.MkdirAll(filepath.Join(provider, "sub"), 0o755)) + writeFile(t, filepath.Join(provider, "sub", "lib.go"), "package sub\nfunc Hello() {}\n") + writeFile(t, filepath.Join(repos[0].Path, "main.go"), "package caller\nimport (\n first \""+importPath+"\"\n second \""+importPath+"\"\n)\nfunc Caller() {\n first.Hello()\n second.Hello()\n}\n") + for _, entry := range []config.RepoEntry{repos[1], repos[0]} { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + check := func() { + t.Helper() + var imports, calls []*graph.Edge + for _, edge := range g.GetOutEdges("a/main.go") { + if edge.Kind == graph.EdgeImports { + imports = append(imports, edge) + } + } + for _, edge := range g.GetOutEdges("a/main.go::Caller") { + if edge.Kind == graph.EdgeCalls { + calls = append(calls, edge) + } + } + require.Len(t, imports, 2, "line identity preserves both imports, as for same-repo aliases") + require.Len(t, calls, 2, "calls through both aliases must survive") + for _, edge := range imports { + assert.Equal(t, target, edge.To, "import line %d", edge.Line) + assert.Equal(t, scope == "cross_repo", edge.CrossRepo) + } + for _, edge := range calls { + assert.Equal(t, target+"::Hello", edge.To, "call line %d", edge.Line) + } + } + check() + mi.RunGlobalResolve() + check() + }) + } + } +} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index d1ec2305a..01aa77bec 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -28,7 +28,7 @@ func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files [ if e.Kind != graph.EdgeImports { continue } - importPath := goModuleImportPath(e, nodes[path]) + importPath := goModuleImportPath(e, nodes[path], outgoing[path]) if importPath == "" || !trackedGoImport(cr.goModulePaths, importPath) && e.Meta["go_import"] == nil { continue } @@ -93,7 +93,7 @@ func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files [ // A dep contract has lost the package suffix. Recover it from the retained // import statement, but never guess when several same-line imports fit. -func goModuleImportPath(e *graph.Edge, nodes []*graph.Node) string { +func goModuleImportPath(e *graph.Edge, nodes []*graph.Node, edges []*graph.Edge) string { if path, ok := e.Meta["go_import"].(string); ok && path != "" { return path } @@ -112,9 +112,17 @@ func goModuleImportPath(e *graph.Edge, nodes []*graph.Node) string { if !dep || strings.Contains(module, "::") { return "" } + // Duplicate aliases share an import node, whose retained line is the last + // occurrence. Containment edges preserve every statement's source line. + contains := make(map[string]bool) + for _, link := range edges { + if link.Kind == graph.EdgeContains && link.From == e.From && link.Line == e.Line { + contains[link.To] = true + } + } path := "" for _, n := range nodes { - if n.Kind != graph.KindImport || n.Language != "go" || n.StartLine != e.Line { + if n.Kind != graph.KindImport || n.Language != "go" || (n.StartLine != e.Line && !contains[n.ID]) { continue } candidate, _ := n.Meta["path"].(string) @@ -424,7 +432,7 @@ func (cr *CrossRepoResolver) restoreGoModuleImportBatch(ctx context.Context, rep if strings.HasPrefix(strings.TrimPrefix(target, externalCallPrefix), "dep::") && fileNodes == nil { fileNodes = cr.graph.GetFileNodesByPaths(files) } - if goModuleImportPath(e, fileNodes[n.FilePath]) == path { + if goModuleImportPath(e, fileNodes[n.FilePath], outgoing[n.FilePath]) == path { present = true break } diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index 305b54623..a67cdfee8 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -92,7 +92,43 @@ func TestGoModuleImportPathAmbiguousLine(t *testing.T) { {Kind: graph.KindImport, Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib/a"}}, {Kind: graph.KindImport, Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib/b"}}, } - assert.Empty(t, goModuleImportPath(edge, nodes), "a legacy module-only target is not enough evidence for either package") + assert.Empty(t, goModuleImportPath(edge, nodes, nil), "a legacy module-only target is not enough evidence for either package") +} + +func TestGoModuleImportPathContainment(t *testing.T) { + const file = "a/main.go" + const path = "example.com/lib/sub" + const id = file + "::import::" + path + edge := &graph.Edge{From: file, To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: file, Line: 3} + nodes := []*graph.Node{ + {ID: id, Kind: graph.KindImport, Language: "go", StartLine: 4, Meta: map[string]any{"path": path}}, + } + for _, tc := range []struct { + name string + link *graph.Edge + want string + }{ + {"exact_line", &graph.Edge{From: file, To: id, Kind: graph.EdgeContains, Line: 3}, path}, + {"wrong_owner", &graph.Edge{From: "a/other.go", To: id, Kind: graph.EdgeContains, Line: 3}, ""}, + {"wrong_line", &graph.Edge{From: file, To: id, Kind: graph.EdgeContains, Line: 4}, ""}, + {"wrong_kind", &graph.Edge{From: file, To: id, Kind: graph.EdgeImports, Line: 3}, ""}, + {"wrong_target", &graph.Edge{From: file, To: "other", Kind: graph.EdgeContains, Line: 3}, ""}, + {"missing", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + var links []*graph.Edge + if tc.link != nil { + links = append(links, tc.link) + } + assert.Equal(t, tc.want, goModuleImportPath(edge, nodes, links)) + }) + } + nodes = append(nodes, &graph.Node{ID: "other", Kind: graph.KindImport, Language: "go", StartLine: 4, Meta: map[string]any{"path": "example.com/lib/other"}}) + links := []*graph.Edge{ + {From: file, To: id, Kind: graph.EdgeContains, Line: 3}, + {From: file, To: "other", Kind: graph.EdgeContains, Line: 3}, + } + assert.Empty(t, goModuleImportPath(edge, nodes, links), "containment must not bypass same-line package ambiguity") } func TestGoModuleImportDroppedBatch(t *testing.T) { From a465476d55c84c02b857233b155c042bcaff47b9 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Fri, 23 Oct 2026 00:16:29 -0700 Subject: [PATCH 013/111] feat(resolver): link Rust imports to a declared tracked crate A Rust `use other_crate::x` import of a crate whose source is another tracked repository stayed external. Directory matching does not know crate roots, Cargo package names or item paths, and Gortex had no map from crate names to repositories. Map each tracked repository's Cargo crates (root package and listed, non-excluded workspace members) to their exposed name ([lib].name, else the package name with dashes as underscores) and root file ([lib].path, else src/lib.rs). An external Rust import links only when the crate that owns the importing file declares the dependency in its own Cargo.toml ([dependencies], dev, build and target tables, and workspace-inherited entries), including `alias = { package = ... }` renames, and exactly one eligible tracked crate provides it. The rest of the path resolves to a module file in that crate, falling back to the crate root for items. Workspace policy is checked against the provider crate, not the alias. The retry runs in the full cross-repo pass and in the mutation-frontier pass, so a link survives an edit to the importing file. It is skipped when no tracked repository has a Cargo.toml. Generic directory matching and other languages are unchanged. Limit: a link stays after the consumer removes the dependency until the importing file is re-indexed. Tests index two tracked repositories on memory and SQLite stores and cover declared, renamed, member, lib-renamed and edited-consumer cases, plus refusals for undeclared, ambiguous, excluded-member and cross-workspace imports. The positive cases fail before this change. --- internal/indexer/cargo_crate.go | 186 ++++++++++++++++++++ internal/indexer/cargo_crate_test.go | 171 ++++++++++++++++++ internal/indexer/workspace_resolve.go | 1 + internal/resolver/cargo_crate.go | 160 +++++++++++++++++ internal/resolver/cargo_crate_test.go | 106 +++++++++++ internal/resolver/cross_repo.go | 12 ++ internal/resolver/cross_repo_incremental.go | 13 +- 7 files changed, 648 insertions(+), 1 deletion(-) create mode 100644 internal/indexer/cargo_crate.go create mode 100644 internal/indexer/cargo_crate_test.go create mode 100644 internal/resolver/cargo_crate.go create mode 100644 internal/resolver/cargo_crate_test.go diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go new file mode 100644 index 000000000..bf755a82a --- /dev/null +++ b/internal/indexer/cargo_crate.go @@ -0,0 +1,186 @@ +package indexer + +import ( + "os" + "path" + "strings" + "sync" + + "github.com/pelletier/go-toml/v2" + "github.com/zzet/gortex/internal/modules" + "github.com/zzet/gortex/internal/resolver" +) + +func cargoTable(m map[string]any, key string) map[string]any { + table, _ := m[key].(map[string]any) + return table +} + +func readCargoManifest(root, dir string) map[string]any { + src, ok := readDiskFile(joinPath(root, path.Join(dir, "Cargo.toml"))) + if !ok { + return nil + } + var manifest map[string]any + if toml.Unmarshal(src, &manifest) != nil { + return nil + } + return manifest +} + +// cargoCrateLookup snapshots root/member manifests once per cross-repo pass. +// Only explicit workspace member globs are expanded; Cargo is never executed. +func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { + type crate struct { + dir, name, root string + deps map[string]string + imports map[string][]resolver.CargoCrate + } + roots := map[string]string{} + for prefix, meta := range mi.AllMetadata() { + if meta != nil && meta.RootPath != "" { + if _, err := os.Stat(joinPath(meta.RootPath, "Cargo.toml")); err == nil { + roots[prefix] = meta.RootPath + } + } + } + if len(roots) == 0 { + return nil + } + owners := map[string]*crate{} + packages := map[string][]*crate{} + ownerCache := map[string]*crate{} + var ownerMu sync.Mutex + var once sync.Once + return func(file, name string) []resolver.CargoCrate { + once.Do(func() { + for prefix, root := range roots { + manifest := readCargoManifest(root, ".") + src, _ := readDiskFile(joinPath(root, "Cargo.toml")) + members, exclude := modules.ParseCargoWorkspace(src) + dirs := []string{"."} + for _, member := range modules.ResolveWorkspaceMembers(root, &modules.WorkspaceManifest{Ecosystem: modules.WorkspaceCargo, Patterns: members, Exclude: exclude}) { + if !path.IsAbs(member.Dir) && !hasDotDotSegment(member.Dir) { + dirs = append(dirs, member.Dir) + } + } + inherited := cargoTable(cargoTable(manifest, "workspace"), "dependencies") + for _, dir := range dirs { + m := manifest + if dir != "." { + m = readCargoManifest(root, dir) + } + if m == nil { + continue + } + pkg, _ := cargoTable(m, "package")["name"].(string) + if pkg == "" { + continue + } + lib := cargoTable(m, "lib") + crateName, _ := lib["name"].(string) + if crateName == "" { + crateName = strings.ReplaceAll(pkg, "-", "_") + } + libPath, _ := lib["path"].(string) + if libPath == "" { + libPath = "src/lib.rs" + } + if path.IsAbs(libPath) || hasDotDotSegment(libPath) { + continue + } + deps := map[string]string{} + tables := []map[string]any{m} + for _, target := range cargoTable(m, "target") { + if table, ok := target.(map[string]any); ok { + tables = append(tables, table) + } + } + for _, table := range tables { + for _, key := range []string{"dependencies", "dev-dependencies", "build-dependencies"} { + for alias, value := range cargoTable(table, key) { + dep, isTable := value.(map[string]any) + if isTable && dep["workspace"] == true { + value = inherited[alias] + dep, isTable = value.(map[string]any) + } + _, isVersion := value.(string) + if !isTable && !isVersion { + continue + } + target := alias + if packageName, ok := dep["package"].(string); ok { + target = packageName + } + // Conflicting target-specific aliases cannot identify one package. + if old, exists := deps[alias]; exists && old != target { + target = "" + } + deps[alias] = target + } + } + } + c := &crate{dir: path.Join(prefix, dir), name: crateName, root: path.Join(prefix, dir, libPath), deps: deps} + if owners[c.dir] == nil { + owners[c.dir] = c + packages[pkg] = append(packages[pkg], c) + } + } + } + for _, owner := range owners { + owner.imports = map[string][]resolver.CargoCrate{} + for alias, pkg := range owner.deps { + for _, provider := range packages[pkg] { + name := provider.name + if alias != pkg { + name = strings.ReplaceAll(alias, "-", "_") + } + owner.imports[name] = append(owner.imports[name], resolver.CargoCrate{Root: provider.root, Name: provider.name}) + } + } + } + }) + // Cache the nearest manifest boundary, including nonmembers and excludes. + // A nested crate must never borrow an ancestor package's dependencies. + ownerMu.Lock() + defer ownerMu.Unlock() + var prefix string + for dir := path.Dir(file); dir != "."; dir = path.Dir(dir) { + if roots[dir] != "" { + prefix = dir + break + } + } + if prefix == "" { + return nil + } + var owner *crate + var visited []string + for dir := path.Dir(file); ; dir = path.Dir(dir) { + if cached, ok := ownerCache[dir]; ok { + owner = cached + break + } + visited = append(visited, dir) + if c := owners[dir]; c != nil { + owner = c + break + } + rel := strings.TrimPrefix(dir, prefix+"/") + if dir == prefix { + rel = "." + } + _, err := os.Stat(joinPath(roots[prefix], path.Join(rel, "Cargo.toml"))) + if !os.IsNotExist(err) || dir == prefix { + break // Unknown, malformed, or unreadable manifest: no ownership evidence. + } + } + for _, dir := range visited { + ownerCache[dir] = owner + } + if owner == nil { + return nil + } + return owner.imports[name] + } +} diff --git a/internal/indexer/cargo_crate_test.go b/internal/indexer/cargo_crate_test.go new file mode 100644 index 000000000..1dd21442e --- /dev/null +++ b/internal/indexer/cargo_crate_test.go @@ -0,0 +1,171 @@ +package indexer + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" + "go.uber.org/zap" +) + +func TestMultiIndexer_CargoCrateImport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"item", "module", "directory", "custom lib", "rename", "dev", "build", "target", "workspace", "undeclared", "sibling declaration", "ambiguous", "disallowed workspace", "same named repo", "ambiguous module", "missing inherited", "reserved", "escaped lib", "nested nonmember", "excluded member", "wrong lib name", "renamed policy allowed", "renamed policy refused", "incremental frontier", "incremental file"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + pkgDir, entry, spec := "", "src/lib.rs", "other_crate::Item" + providerManifest := "[package]\nname = \"other-crate\"\nversion = \"0.1.0\"\n" + dep := "[dependencies]\nother-crate = \"1\"\n" + switch mode { + case "module", "directory", "ambiguous module": + spec = "other_crate::a::b::Item" + entry = "src/a/b.rs" + if mode == "directory" { + entry = "src/a/b/mod.rs" + } + case "wrong lib name": + providerManifest += "[lib]\nname = \"engine\"\n" + case "custom lib": + providerManifest += "[lib]\nname = \"engine\"\npath = \"code/root.rs\"\n" + entry, spec = "code/root.rs", "engine::Item" + case "escaped lib": + providerManifest += "[lib]\npath = \"../src/lib.rs\"\n" + case "reserved": + dep, spec = "[dependencies]\ncore = { package = \"other-crate\", version = \"1\" }\n", "core::Item" + case "missing inherited": + dep = "[dependencies]\nother-crate = { workspace = true }\n" + case "rename", "renamed policy allowed", "renamed policy refused": + dep, spec = "[dependencies]\nalias = { package = \"other-crate\", version = \"1\" }\n", "alias::Item" + case "dev", "build": + dep = "[" + mode + "-dependencies]\nother-crate = \"1\"\n" + case "target": + dep = "[target.'cfg(unix)'.dependencies]\nother-crate = \"1\"\n" + case "workspace": + pkgDir = "crates/engine" + writeWorkspaceFile(t, provider, "Cargo.toml", "[workspace]\nmembers = [\"crates/*\"]\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[workspace]\nmembers = [\"apps/*\"]\n[workspace.dependencies]\nalias = { package = \"other-crate\", version = \"1\" }\n") + dep, spec = "[dependencies]\nalias = { workspace = true }\n", "alias::Item" + case "undeclared", "same named repo", "sibling declaration": + dep = "" + } + writeWorkspaceFile(t, provider, filepath.Join(pkgDir, "Cargo.toml"), providerManifest) + writeWorkspaceFile(t, provider, filepath.Join(pkgDir, "src/lib.rs"), "pub struct Item;\n") + writeWorkspaceFile(t, provider, filepath.Join(pkgDir, entry), "pub struct Item;\n") + if mode == "module" || mode == "directory" || mode == "ambiguous module" { + writeWorkspaceFile(t, provider, "src/lib.rs", "pub mod a;\n") + writeWorkspaceFile(t, provider, "src/a.rs", "pub mod b;\n") + } + if mode == "ambiguous module" { + writeWorkspaceFile(t, provider, "src/a/b/mod.rs", "pub struct Item;\n") + } + consumerDir := "" + if mode == "nested nonmember" || mode == "excluded member" { + consumerDir = "tools/child" + rootManifest := "[package]\nname = \"root\"\n" + dep + if mode == "excluded member" { + rootManifest += "[workspace]\nmembers = [\"tools/*\"]\nexclude = [\"tools/child\"]\n" + } + writeWorkspaceFile(t, consumer, "Cargo.toml", rootManifest) + dep = "" + } + if mode == "workspace" || mode == "sibling declaration" { + consumerDir = "apps/app" + } + if mode == "sibling declaration" { + writeWorkspaceFile(t, consumer, "Cargo.toml", "[workspace]\nmembers = [\"apps/*\"]\n") + writeWorkspaceFile(t, consumer, "apps/sibling/Cargo.toml", "[package]\nname = \"sibling\"\n[dependencies]\nother-crate = \"1\"\n") + } + writeWorkspaceFile(t, consumer, filepath.Join(consumerDir, "Cargo.toml"), "[package]\nname = \"app\"\nversion = \"0.1.0\"\n"+dep) + caller := filepath.Join(consumerDir, "src/lib.rs") + writeWorkspaceFile(t, consumer, caller, "use "+spec+";\n") + providerWS, providerName := "shared", "provider" + if mode == "disallowed workspace" || strings.HasPrefix(mode, "renamed policy") { + providerWS = "private" + } + if mode == "same named repo" { + providerName = "other_crate" + } + entries := []config.RepoEntry{{Path: provider, Name: providerName, Workspace: providerWS}, {Path: consumer, Name: "consumer", Workspace: "shared"}} + if mode == "ambiguous" { + duplicate := t.TempDir() + writeWorkspaceFile(t, duplicate, "Cargo.toml", providerManifest) + writeWorkspaceFile(t, duplicate, "src/lib.rs", "pub struct Item;\n") + entries = append(entries, config.RepoEntry{Path: duplicate, Name: "duplicate", Workspace: "shared"}) + } + if strings.HasPrefix(mode, "renamed policy") { + module := "other_crate" + if mode == "renamed policy refused" { + module = "alias" + } + writeWorkspaceFile(t, consumer, ".gortex.yaml", "cross_workspace_deps:\n - workspace: private\n modules: ["+module+"]\n mode: read-only\n") + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + reg := newTestRegistry() + reg.Register(languages.NewRustExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + _, err = mi.IndexAll() + require.NoError(t, err) + want := filepath.ToSlash(filepath.Join(providerName, pkgDir, entry)) + require.NotNil(t, g.GetNode(want)) + resolves := mode != "nested nonmember" && mode != "excluded member" && mode != "wrong lib name" && mode != "renamed policy refused" && mode != "undeclared" && mode != "same named repo" && mode != "sibling declaration" && mode != "ambiguous" && mode != "disallowed workspace" && mode != "ambiguous module" && mode != "missing inherited" && mode != "reserved" && mode != "escaped lib" + if strings.HasPrefix(mode, "incremental") { + var initial []string + for e := range g.EdgesByKind(graph.EdgeImports) { + if e.FilePath == "consumer/src/lib.rs" { + initial = append(initial, e.To) + } + } + require.Equal(t, []string{want}, initial, "initial binding") + writeWorkspaceFile(t, consumer, caller, "use "+spec+";\npub fn edited() {}\n") + require.NoError(t, mi.GetIndexer("consumer").IndexFileNoResolve(caller)) + if mode == "incremental frontier" { + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/src/lib.rs"}, nil)) + } else { + mi.runMasterResolveFiles([]string{"consumer/src/lib.rs"}, true) + mi.newCrossRepoResolver().ResolveForFile("consumer", "src/lib.rs") + } + } + var imports int + for e := range g.EdgesByKind(graph.EdgeImports) { + if e.FilePath != filepath.ToSlash(filepath.Join("consumer", caller)) { + continue + } + imports++ + t.Logf("import %s -> %s crossRepo=%v", e.From, e.To, e.CrossRepo) + assert.Equal(t, resolves, e.CrossRepo) + if resolves { + assert.Equal(t, want, e.To) + } else { + assert.Equal(t, "external::"+strings.ReplaceAll(spec, "::", "/"), e.To) + } + } + assert.Equal(t, 1, imports) + }) + } + } +} + +func TestCargoCrateLookupNoManifest(t *testing.T) { + mi := &MultiIndexer{repos: map[string]*RepoMetadata{"plain": {RootPath: t.TempDir()}}} + assert.Nil(t, mi.cargoCrateLookup(), "no Cargo manifest must leave the retry disabled") +} diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 191371cfc..30502fd39 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -497,6 +497,7 @@ func (mi *MultiIndexer) newCrossRepoResolver() *resolver.CrossRepoResolver { cr.SetNpmAliasResolver(mi.npmAliasResolver()) cr.SetNpmDependencyLookup(mi.npmDependencyLookup()) cr.SetNpmPackageLookup(mi.npmPackageLookup()) + cr.SetCargoCrateLookup(mi.cargoCrateLookup()) cr.SetPathAliasResolver(mi.pathAliasResolver()) cr.SetWorkspaceMembership(mi.workspaceMembershipResolver()) mi.applyRemoteStitch(cr) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go new file mode 100644 index 000000000..97d4943a8 --- /dev/null +++ b/internal/resolver/cargo_crate.go @@ -0,0 +1,160 @@ +package resolver + +import ( + "context" + "iter" + "path" + "strings" + + "github.com/zzet/gortex/internal/graph" +) + +// CargoCrate identifies a provider library and its exposed crate name. +type CargoCrate struct { + Root string // graph-path library file + Name string // exposed provider crate name, never the consumer's alias +} + +// CargoCrateLookup returns providers declared by the crate owning callerFile. +// The resolver owns workspace eligibility. +type CargoCrateLookup func(callerFile, crateName string) []CargoCrate + +// SetCargoCrateLookup installs manifest evidence for Rust import/file edges. +func (cr *CrossRepoResolver) SetCargoCrateLookup(lookup CargoCrateLookup) { cr.cargoCrates = lookup } + +func (cr *CrossRepoResolver) cargoImportTarget(e *graph.Edge, spec string) *graph.Node { + if cr.cargoCrates == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { + return nil + } + segs := strings.Split(spec, "/") + switch segs[0] { + case "", "crate", "self", "super", "std", "core", "alloc": + return nil + } + for _, seg := range segs { + if seg == ".." || seg == "." || seg == "" { + return nil + } + } + crates := cr.cargoCrates(e.FilePath, segs[0]) + if len(crates) == 0 { + return nil + } + callerRepo, callerWS := cr.callerRepoPrefix(e), cr.callerWorkspaceID(e) + var root *graph.Node + for _, crate := range crates { + candidate := cr.cachedGetNode(crate.Root) + if candidate == nil || candidate.Kind != graph.KindFile || candidate.Language != "rust" { + continue + } + if candidate.RepoPrefix == callerRepo { + return nil // same-repo resolution stays unchanged + } + if !cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(candidate), crate.Name+strings.TrimPrefix(spec, segs[0])) { + continue + } + if root != nil { + return nil // two eligible providers, even if only one has the module + } + root = candidate + } + if root == nil { + return nil + } + // Same longest-prefix module-file probes as resolveRustUseFile, anchored + // at the manifest's library root rather than a guessed src directory. + for n := len(segs); n > 1; n-- { + stem := path.Join(append([]string{path.Dir(root.ID)}, segs[1:n]...)...) + var found *graph.Node + for _, id := range []string{stem + ".rs", stem + "/mod.rs"} { + if node := cr.cachedGetNode(id); node != nil && node.Kind == graph.KindFile && node.Language == "rust" { + if found != nil { + return nil + } + found = node + } + } + if found != nil { + return found + } + } + return root // item or glob import from the crate itself +} + +// Per-repo resolution converts missing imports to external stubs. Retry just +// those Rust imports before the cross-repo unresolved pass builds reachability. +func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats) error { + if cr.cargoCrates == nil { + return ctx.Err() + } + // Keep metadata out of the scan. Only declared, tracked crates need hydration. + var eligible []*graph.Edge + for e := range edges { + if err := ctx.Err(); err != nil { + return err + } + spec := cargoImportSpec(e) + if spec == "" { + continue + } + name, _, _ := strings.Cut(spec, "/") + if len(cr.cargoCrates(e.FilePath, name)) > 0 { + eligible = append(eligible, e) + } + } + // Close the light cursor before hydration or writes; retain metadata for + // only one bounded page, as in the tracked npm bridge. + for start := 0; start < len(eligible); start += resolvePendingPageRows { + if err := ctx.Err(); err != nil { + return err + } + page := eligible[start:min(start+resolvePendingPageRows, len(eligible))] + sites := make([]graph.EdgeSite, 0, len(page)) + for _, e := range page { + sites = append(sites, graph.EdgeSite{From: e.From, Line: e.Line, Kind: graph.EdgeImports}) + } + full := cr.graph.GetEdgeCandidates(nil, sites) + var batch []graph.EdgeReindex + for _, projected := range page { + for _, e := range full.Site(projected.From, projected.Line, graph.EdgeImports) { + if e.To != projected.To { + continue + } + if node := cr.cargoImportTarget(e, cargoImportSpec(e)); node != nil { + oldTo := e.To + cr.bindCargoImport(e, node, stats) + batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) + projected.To, projected.CrossRepo = e.To, e.CrossRepo + } + } + } + if len(batch) > 0 { + cr.graph.ReindexEdges(batch) + } + } + return ctx.Err() +} + +func cargoImportSpec(e *graph.Edge) string { + if e == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { + return "" + } + spec, external := strings.CutPrefix(e.To, "external::") + if !external { + return "" + } + name, _, _ := strings.Cut(spec, "/") + switch name { + case "", "crate", "self", "super", "std", "core", "alloc": + return "" + } + return spec +} + +func (cr *CrossRepoResolver) bindCargoImport(e *graph.Edge, node *graph.Node, stats *CrossRepoStats) { + e.To, e.CrossRepo = node.ID, true + clearEdgeTerminal(e) + stats.Resolved++ + stats.CrossRepoEdges++ + stats.ByRepo[node.RepoPrefix]++ +} diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go new file mode 100644 index 000000000..5ce523804 --- /dev/null +++ b/internal/resolver/cargo_crate_test.go @@ -0,0 +1,106 @@ +package resolver + +import ( + "context" + "iter" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestCargoImportUntrackedSkipsNodeReads(t *testing.T) { + g := &resolverBatchCountingStore{Store: graph.New()} + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(_, _ string) []CargoCrate { return nil }) + e := &graph.Edge{From: "app/src/lib.rs", To: "external::serde/Item", Kind: graph.EdgeImports, FilePath: "app/src/lib.rs"} + assert.Nil(t, cr.cargoImportTarget(e, "serde/Item")) + assert.Zero(t, g.getNodeCalls, "unknown crates must be rejected before caller node reads") +} + +type cargoPagingStore struct { + graph.Store + fullScans, lightScans, pages, maxSites int + cursorOpen bool +} + +func (s *cargoPagingStore) EdgesByKind(kind graph.EdgeKind) iter.Seq[*graph.Edge] { + s.fullScans++ + return s.Store.EdgesByKind(kind) +} + +func (s *cargoPagingStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + s.lightScans++ + return func(yield func(*graph.Edge) bool) { + s.cursorOpen = true + defer func() { s.cursorOpen = false }() + for e := range graph.EdgesLightSeq(s.Store, kinds...) { + if !yield(e) { + return + } + } + } +} + +func (s *cargoPagingStore) GetEdgeCandidates(endpoints []graph.EdgeEndpoint, sites []graph.EdgeSite) graph.EdgeCandidateSet { + if s.cursorOpen { + panic("hydration while light cursor is open") + } + s.pages++ + s.maxSites = max(s.maxSites, len(sites)) + return s.Store.GetEdgeCandidates(endpoints, sites) +} + +func TestCargoImportLightPaging(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var store graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store = db + } + g := &cargoPagingStore{Store: store} + const count = resolvePendingPageRows + 1 + edges := make([]*graph.Edge, 0, count+1) + for line := 1; line <= count; line++ { + edges = append(edges, &graph.Edge{From: "app/src/lib.rs", To: "external::other_crate/Item", Kind: graph.EdgeImports, FilePath: "app/src/lib.rs", Line: line, Meta: map[string]any{"marker": "preserved"}}) + } + edges = append(edges, &graph.Edge{From: "app/src/lib.rs", To: "external::serde/Item", Kind: graph.EdgeImports, FilePath: "app/src/lib.rs", Line: count + 1}) + g.AddBatch([]*graph.Node{ + {ID: "app/src/lib.rs", Kind: graph.KindFile, FilePath: "app/src/lib.rs", RepoPrefix: "app", Language: "rust"}, + {ID: "lib/src/lib.rs", Kind: graph.KindFile, FilePath: "lib/src/lib.rs", RepoPrefix: "lib", Language: "rust"}, + }, edges) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(_, name string) []CargoCrate { + if name != "other_crate" { + return nil + } + return []CargoCrate{{Root: "lib/src/lib.rs", Name: "other_crate"}} + }) + stats := &CrossRepoStats{ByRepo: map[string]int{}} + require.NoError(t, cr.resolveCargoImports(context.Background(), graph.EdgesLightSeq(g, graph.EdgeImports), stats)) + assert.Zero(t, g.fullScans) + assert.Equal(t, 1, g.lightScans) + assert.Equal(t, 2, g.pages) + assert.Equal(t, resolvePendingPageRows, g.maxSites) + assert.Equal(t, count, stats.Resolved) + imports := g.GetOutEdges("app/src/lib.rs") + require.Len(t, imports, count+1) + for _, edge := range imports { + if edge.Line == count+1 { + assert.Equal(t, "external::serde/Item", edge.To) + assert.False(t, edge.CrossRepo) + continue + } + assert.Equal(t, "lib/src/lib.rs", edge.To) + assert.True(t, edge.CrossRepo) + assert.Equal(t, "preserved", edge.Meta["marker"]) + } + }) + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 5c006047b..dd055b2f7 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -154,6 +154,9 @@ type CrossRepoResolver struct { // npm_dependency.go. npmDep NpmDependencyLookup npmPackages NpmPackageLookup + + cargoCrates CargoCrateLookup + // pathAlias expands a JS/TS tsconfig/jsconfig path-alias / baseUrl // import specifier to the repo-prefixed file stem it targets. Same // contract as the field of the same name on Resolver — see @@ -428,6 +431,9 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS if err := cr.resolveTrackedNpmImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { return stats, err } + if err := cr.resolveCargoImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { + return stats, err + } // Share the master resolver's stable high-water/keyset stream. The cold // residual can exceed 200k edges; retaining it plus the cross-repo name, // raw-name, repo and qualified-name caches was the second whole-corpus heap @@ -713,6 +719,7 @@ func (cr *CrossRepoResolver) ResolveForFile(repoPrefix, relPath string) *CrossRe func (cr *CrossRepoResolver) resolveScopedLocked(edges []*graph.Edge) *CrossRepoStats { stats := &CrossRepoStats{ByRepo: make(map[string]int)} _ = cr.resolveTrackedNpmImports(context.Background(), slices.Values(edges), stats) + _ = cr.resolveCargoImports(context.Background(), slices.Values(edges), stats) pending := make([]*graph.Edge, 0, len(edges)) for _, edge := range edges { if edge != nil && graph.IsUnresolvedTarget(edge.To) { @@ -1538,6 +1545,11 @@ func (cr *CrossRepoResolver) resolveImport(e *graph.Edge, importPath string, sta // evidence fallbacks below before deciding that the import is ambiguous. } + if node := cr.cargoImportTarget(e, importPath); node != nil { + cr.bindCargoImport(e, node, stats) + return + } + // Look for file nodes whose directory matches the import path. Two // inverted indexes (built once per Resolve* pass) replace what used // to be an O(N) scan of the entire graph per import edge. diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 04f52d5b4..9cf852f5c 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -99,10 +99,21 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed } } } + // The per-repo pass has made missing Rust imports external already. + // Keep outgoing imports for the tracked-crate retry too. + if cr.cargoCrates != nil { + for _, edges := range frontier.outByNode { + for _, edge := range edges { + if cargoImportSpec(edge) != "" { + frontier.pending = append(frontier.pending, edge) + } + } + } + } pending := make([]*graph.Edge, 0, len(frontier.pending)) seen := make(map[graph.EdgeIdentity]struct{}, len(frontier.pending)) for _, edge := range frontier.pending { - if edge == nil || !graph.IsUnresolvedTarget(edge.To) && trackedNpmImportSpec(edge) == "" { + if edge == nil || !graph.IsUnresolvedTarget(edge.To) && trackedNpmImportSpec(edge) == "" && cargoImportSpec(edge) == "" { continue } identity := graph.EdgeIdentityFor(edge) From baee2d8b1db5f7d9adbf92dd5cdbff7215648f13 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 00:49:34 -0700 Subject: [PATCH 014/111] fix(indexer): preserve tracked Cargo imports across provider replacement #806's whole-shadow retirement and save restub paths can delete imports owned by unchanged consumers. Keep the exact pre-link stub on Cargo imports, park those imports and their cross-repo calls before provider eviction, and carry consumer files through the bounded cross-repo tail. Retry declared consumers after IndexRepo, track and full_retrack. Keep warmup's cross-repo-only files separate from provider derived plans. Mirror the approved npm lifecycle shape without reparsing importers. Cover both stores, both track orders, saves, full replacement, warm full_retrack, untrack/retrack, unchanged repositories and linked calls. --- cmd/gortex/daemon_state.go | 7 + cmd/gortex/daemon_warm_cargo_frontier_test.go | 25 +++ internal/indexer/affected_by_bound_test.go | 2 +- .../indexer/cargo_import_lifecycle_test.go | 142 ++++++++++++++++++ internal/indexer/cargo_import_restub.go | 124 +++++++++++++++ internal/indexer/cargo_import_restub_test.go | 22 +++ internal/indexer/file_index_failures_test.go | 2 +- internal/indexer/file_meta.go | 1 + internal/indexer/file_mtime_writers_test.go | 2 +- .../go_module_import_lifecycle_test.go | 2 +- .../indexer/go_module_source_store_test.go | 2 +- internal/indexer/incremental_batch.go | 22 ++- .../indexer/incremental_restub_gate_test.go | 2 +- internal/indexer/incremental_watcher_batch.go | 3 +- internal/indexer/indexer.go | 21 ++- internal/indexer/legacy_file_identity_test.go | 4 +- internal/indexer/multi.go | 4 + internal/indexer/npm_lifecycle_test.go | 5 +- internal/indexer/repo_eviction_scope.go | 1 + internal/indexer/repository_untrack.go | 1 + internal/resolver/cargo_crate.go | 7 + 21 files changed, 375 insertions(+), 26 deletions(-) create mode 100644 cmd/gortex/daemon_warm_cargo_frontier_test.go create mode 100644 internal/indexer/cargo_import_lifecycle_test.go create mode 100644 internal/indexer/cargo_import_restub.go create mode 100644 internal/indexer/cargo_import_restub_test.go diff --git a/cmd/gortex/daemon_state.go b/cmd/gortex/daemon_state.go index 8f097b733..a338b3b4b 100644 --- a/cmd/gortex/daemon_state.go +++ b/cmd/gortex/daemon_state.go @@ -367,6 +367,13 @@ func (frontier *warmupDeltaFrontier) invalidate() { func (frontier *warmupDeltaFrontier) record(result *indexer.IndexResult) { frontier.mu.Lock() defer frontier.mu.Unlock() + if result != nil { + for _, file := range result.CrossRepoResolveFiles { + if file != "" { + frontier.crossRepoFiles[file] = struct{}{} + } + } + } if result == nil || result.RepoPrefix == "" || result.FullRetrack || len(result.FailedFiles) > 0 { frontier.exact = false return diff --git a/cmd/gortex/daemon_warm_cargo_frontier_test.go b/cmd/gortex/daemon_warm_cargo_frontier_test.go new file mode 100644 index 000000000..bbb23a5b4 --- /dev/null +++ b/cmd/gortex/daemon_warm_cargo_frontier_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/indexer" +) + +func TestWarmupCargoConsumersStayOutsideDerivedPlan(t *testing.T) { + for _, full := range []bool{false, true} { + frontier := newWarmupDeltaFrontier() + frontier.record(&indexer.IndexResult{ + RepoPrefix: "provider", FullRetrack: full, + DerivedInvalidation: indexer.DerivedInvalidationPlan{Files: []string{"provider/src/lib.rs"}}, + CrossRepoResolveFiles: []string{"consumer/src/lib.rs"}, + }) + plans, files, exact := frontier.snapshot(1, false) + require.Contains(t, files, "consumer/src/lib.rs") + require.Equal(t, !full, exact) + for _, plan := range plans { + require.NotContains(t, plan.Files, "consumer/src/lib.rs") + } + } +} diff --git a/internal/indexer/affected_by_bound_test.go b/internal/indexer/affected_by_bound_test.go index 812e5518c..d726a0e55 100644 --- a/internal/indexer/affected_by_bound_test.go +++ b/internal/indexer/affected_by_bound_test.go @@ -387,7 +387,7 @@ func TestDeletionShapedStagesCarryNoInEdges(t *testing.T) { stages := []*incrementalBatchStage{{graphPath: restubDefPath, priorNodes: prior}} view := loadIncrementalPriorView(st, stages) - carried, _ := restubIncomingRefsFromView(st, stages, view) + carried, _, _ := restubIncomingRefsFromView(st, stages, view) require.Empty(t, carried, "a deletion stage must park every in-edge; a carried edge has no re-state path here") assert.Equal(t, 1, st.restubWrites("Foo"), diff --git a/internal/indexer/cargo_import_lifecycle_test.go b/internal/indexer/cargo_import_lifecycle_test.go new file mode 100644 index 000000000..9a3a37dc6 --- /dev/null +++ b/internal/indexer/cargo_import_lifecycle_test.go @@ -0,0 +1,142 @@ +package indexer + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedCargoImportLifecycle(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, operation := range []string{"consumer_save", "provider_save", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track"} { + order := "consumer_first" + if providerFirst { + order = "provider_first" + } + t.Run(backend+"/"+order+"/"+operation, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") + writeWorkspaceFile(t, provider, "src/lib.rs", "pub fn action() {}\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") + writeWorkspaceFile(t, consumer, "src/lib.rs", "use other_crate::action;\npub fn run() { action(); }\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + if providerFirst { + gc.Repos = []config.RepoEntry{entries[1], entries[0]} + } + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + var g graph.Store = graph.New() + if backend == "sqlite" { + disk, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, disk.Close()) }) + g = disk + } + reg := newTestRegistry() + reg.Register(languages.NewRustExtractor()) + core, logs := observer.New(zap.InfoLevel) + logger := zap.New(core) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + if operation == "track" { + for _, entry := range gc.Repos { + _, err = mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + } else { + _, err = mi.IndexAll() + require.NoError(t, err) + } + assertImport := func() { + t.Helper() + var imports []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/src/lib.rs") { + if edge.Kind == graph.EdgeImports { + imports = append(imports, edge) + } + } + require.Len(t, imports, 1) + require.Equal(t, "provider/src/lib.rs", imports[0].To) + require.True(t, imports[0].CrossRepo) + var calls []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { + if edge.Kind == graph.EdgeCalls { + calls = append(calls, edge) + } + } + require.Len(t, calls, 1) + require.Equal(t, "provider/src/lib.rs::action", calls[0].To) + require.True(t, calls[0].CrossRepo) + } + assertImport() + logs.TakeAll() + switch operation { + case "consumer_save": + writeWorkspaceFile(t, consumer, "src/lib.rs", "use other_crate::action;\npub fn run() { action(); }\npub fn edited() {}\n") + _, err = mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "src/lib.rs")}) + case "provider_save": + writeWorkspaceFile(t, provider, "src/lib.rs", "pub fn action() {}\npub fn added() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/lib.rs")}) + case "consumer_full": + _, err = mi.IndexRepo("consumer") + case "provider_full": + _, err = mi.IndexRepo("provider") + case "warm_full_retrack": + prior := map[string]map[string]int64{"consumer": mi.FileMtimes("consumer"), "provider": mi.FileMtimes("provider")} + restarted := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, restarted.Close(context.Background())) }) + restarted.BeginBatch() + _, err = restarted.ReconcileRepoCtx(context.Background(), entries[0], prior["consumer"]) + require.NoError(t, err) + t.Setenv("GORTEX_WARMUP_FULL_RETRACK", "1") + result, restoreErr := restarted.ReconcileRepoCtx(context.Background(), entries[1], prior["provider"]) + require.NoError(t, restoreErr) + require.True(t, result.FullRetrack) + data, marshalErr := json.Marshal(result) + require.NoError(t, marshalErr) + var output struct { + Files []string `json:"cross_repo_resolve_files"` + } + require.NoError(t, json.Unmarshal(data, &output)) + if backend == "sqlite" { + require.Contains(t, output.Files, "consumer/src/lib.rs") + } else { + require.Empty(t, output.Files, "memory replacement preserved the existing import") + } + require.NotContains(t, result.DerivedInvalidation.Files, "consumer/src/lib.rs") + err = restarted.RunPreEnrichResolveFiles(context.Background(), output.Files, nil) + restarted.EndBatch() + case "no_change": + result, noChangeErr := mi.IncrementalReindexRepo("provider", nil) + require.NoError(t, noChangeErr) + require.Zero(t, result.StaleFileCount) + case "untrack_retrack": + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") + require.NoError(t, err) + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + } + require.NoError(t, err) + assertImport() + require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) + require.Empty(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveAll").All()) + }) + } + } + } +} diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go new file mode 100644 index 000000000..34f296f67 --- /dev/null +++ b/internal/indexer/cargo_import_restub.go @@ -0,0 +1,124 @@ +package indexer + +import ( + "maps" + "slices" + "strings" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +// Preserve importer-owned facts before their provider's files disappear. +func restubTrackedCargoRepoIncoming(g graph.Store, repo string) []string { + if repo == "" { + return nil + } + var files, ids []string + flush := func() { + incoming := g.GetInEdgesByNodeIDs(ids) + evicted := make(map[string]struct{}) + for _, edges := range incoming { + for _, edge := range edges { + if edge != nil && graph.RepoPrefixOfID(edge.From) == repo { + evicted[edge.From] = struct{}{} + } + } + } + files = appendUniqueSorted(files, restubTrackedCargoImportsFromView(g, evicted, incoming)...) + ids = ids[:0] + } + for node := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFile) { + if !strings.HasSuffix(node.ID, ".rs") { + continue + } + ids = append(ids, node.ID) + if len(ids) == 512 { + flush() + } + } + if len(ids) > 0 { + flush() + } + return files +} + +func restubTrackedCargoImportsFromView(g graph.Store, evicted map[string]struct{}, incoming map[string][]*graph.Edge) []string { + consumers := make(map[string]map[string]bool) + var batch []graph.EdgeReindex + for _, edges := range incoming { + for _, edge := range edges { + if edge == nil || edge.Kind != graph.EdgeImports { + continue + } + stub, _ := edge.Meta[resolver.CargoImportStubMeta].(string) + if _, gone := evicted[edge.From]; stub == "" || gone || edge.To == stub { + continue + } + if consumers[edge.FilePath] == nil { + consumers[edge.FilePath] = make(map[string]bool) + } + consumers[edge.FilePath][graph.RepoPrefixOfID(edge.To)] = true + old := edge.To + edge.To, edge.CrossRepo = stub, false + batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: old}) + } + } + files := slices.Sorted(maps.Keys(consumers)) + if len(files) == 0 { + return nil + } + var ids []string + for _, nodes := range g.GetFileNodesByPaths(files) { + for _, node := range nodes { + if node != nil { + ids = append(ids, node.ID) + } + } + } + for _, edges := range g.GetOutEdgesByNodeIDs(ids) { + for _, edge := range edges { + if edge == nil || edge.Kind != graph.EdgeCalls || !edge.CrossRepo || !consumers[edge.FilePath][graph.RepoPrefixOfID(edge.To)] { + continue + } + if _, gone := evicted[edge.From]; gone { + continue + } + old := edge.To + _, name, symbol := strings.Cut(old, "::") + if !symbol { + continue + } + graph.StashRestubProvenance(edge) + edge.To, edge.CrossRepo, edge.ConfidenceLabel = graph.UnresolvedMarker+name, false, "" + batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: old}) + } + } + if len(batch) > 0 { + g.ReindexEdges(batch) + } + return files +} + +// Retry only parked imports claimed by this provider, or owned by this repo. +func (mi *MultiIndexer) cargoConsumerFilesForRepo(repo string) []string { + lookup := mi.cargoCrateLookup() + if lookup == nil { + return nil + } + files := make(map[string]struct{}) + for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { + spec, external := strings.CutPrefix(edge.To, "external::") + if !external || !strings.HasSuffix(edge.FilePath, ".rs") { + continue + } + name, _, _ := strings.Cut(spec, "/") + for _, crate := range lookup(edge.FilePath, name) { + if graph.RepoPrefixOfID(crate.Root) == repo || graph.RepoPrefixOfID(edge.From) == repo { + files[edge.FilePath] = struct{}{} + break + } + } + } + return sortedStringKeys(files) +} diff --git a/internal/indexer/cargo_import_restub_test.go b/internal/indexer/cargo_import_restub_test.go new file mode 100644 index 000000000..01c9680fb --- /dev/null +++ b/internal/indexer/cargo_import_restub_test.go @@ -0,0 +1,22 @@ +package indexer + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" +) + +func TestCargoRestubLeavesUnmarkedImportsUnchanged(t *testing.T) { + for _, file := range []string{"provider/src/lib.rs", "provider/main.go"} { + t.Run(file, func(t *testing.T) { + g := graph.New() + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "provider"}) + edge := &graph.Edge{From: "consumer/source", FilePath: "consumer/source", To: file, Kind: graph.EdgeImports, CrossRepo: true} + g.AddEdge(edge) + require.Empty(t, restubTrackedCargoRepoIncoming(g, "provider")) + require.Equal(t, file, edge.To) + require.True(t, edge.CrossRepo) + }) + } +} diff --git a/internal/indexer/file_index_failures_test.go b/internal/indexer/file_index_failures_test.go index 007be7c7a..2169d308d 100644 --- a/internal/indexer/file_index_failures_test.go +++ b/internal/indexer/file_index_failures_test.go @@ -311,7 +311,7 @@ func TestIncrementalRetryLogsTerminalReadError(t *testing.T) { require.Len(t, warnings, 1) require.Contains(t, warnings[0].ContextMap()["error"], "no space left on device") var plan DerivedInvalidationPlan - idx.evictDeletedFilesBatched([]string{"source.go"}, &plan) + idx.evictDeletedFilesBatched([]string{"source.go"}, &plan, nil) require.Empty(t, LoadFileIndexFailures(g, ""), "explicit deletion clears even a never-indexed failure") } diff --git a/internal/indexer/file_meta.go b/internal/indexer/file_meta.go index dd646109a..ff35a42e3 100644 --- a/internal/indexer/file_meta.go +++ b/internal/indexer/file_meta.go @@ -197,6 +197,7 @@ const reparsePendingEnrichmentBatchSize = 256 type reparsePendingEnrichmentBatch struct { // npmConsumerFiles are unchanged importers parked by this mutation. npmConsumerFiles []string + cargoConsumerFiles []string byFile map[string]bool deferResolverCatchup bool deferredAffectedFiles map[string]struct{} diff --git a/internal/indexer/file_mtime_writers_test.go b/internal/indexer/file_mtime_writers_test.go index 22085f115..554f8af91 100644 --- a/internal/indexer/file_mtime_writers_test.go +++ b/internal/indexer/file_mtime_writers_test.go @@ -43,7 +43,7 @@ func TestBatchedFileMtimeEvictionDetachesPublishedSnapshot(t *testing.T) { idx.fileMtimes = map[string]int64{"a.go": 1, "b.go": 2} published := idx.publishFileMtimes() - idx.evictDeletedFilesBatched([]string{"a.go"}, &DerivedInvalidationPlan{}) + idx.evictDeletedFilesBatched([]string{"a.go"}, &DerivedInvalidationPlan{}, nil) assert.Equal(t, int64(1), published["a.go"]) assert.NotContains(t, idx.fileMtimes, "a.go") diff --git a/internal/indexer/go_module_import_lifecycle_test.go b/internal/indexer/go_module_import_lifecycle_test.go index 85b8a8f4d..0bbd4ddac 100644 --- a/internal/indexer/go_module_import_lifecycle_test.go +++ b/internal/indexer/go_module_import_lifecycle_test.go @@ -149,7 +149,7 @@ func TestGoModuleImportParkBeforeEviction(t *testing.T) { restubGoModuleRepoIncoming(g, "b") } else if route == "deleted_batch" { provider := g.GetNode("b/lib.go") - carried, _ := restubIncomingRefsFromView(g, + carried, _, _ := restubIncomingRefsFromView(g, []*incrementalBatchStage{{graphPath: "b/lib.go", priorNodes: []*graph.Node{provider}}}, incrementalPriorView{inByNode: g.GetInEdgesByNodeIDs([]string{provider.ID})}) assert.Empty(t, carried, "a deleted file cannot carry a resolved target") diff --git a/internal/indexer/go_module_source_store_test.go b/internal/indexer/go_module_source_store_test.go index a805c76b6..877728a04 100644 --- a/internal/indexer/go_module_source_store_test.go +++ b/internal/indexer/go_module_source_store_test.go @@ -87,7 +87,7 @@ func TestGoModuleSourceStoreDropsTrustAfterConcurrentEviction(t *testing.T) { if reads == 1 { // The file is deleted and evicted during the read. require.NoError(t, os.Remove(main)) - idx.evictDeletedFilesBatched([]string{"main.go"}, &DerivedInvalidationPlan{}) + idx.evictDeletedFilesBatched([]string{"main.go"}, &DerivedInvalidationPlan{}, nil) } return src, err } diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index dfe8b67dd..b7c3b5e92 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -158,7 +158,7 @@ func (idx *Indexer) reindexIncrementalFilesBatched( deleteTiming := idx.startReconcilePhase(idx.logger, idx.repoPrefix, "graph_delete", zap.Int("deleted_files", len(deletedFiles))) defer deleteTiming.abort() - idx.evictDeletedFilesBatched(deletedFiles, &invalidation) + idx.evictDeletedFilesBatched(deletedFiles, &invalidation, markerBatch) deleteTiming.complete(nil) passPlan, reparsed, failed, versionChanged := idx.reindexIncrementalStalePass( @@ -1110,9 +1110,10 @@ func (idx *Indexer) commitStructuralIncrementalBatch( markerBatch.recordDeferredPriorBindings(stagePriorBindings(idx.graph, stages, view)) } idx.applyLap("prior_bindings") - carried, npmConsumers := restubIncomingRefsFromView(idx.graph, stages, view) + carried, npmConsumers, cargoConsumers := restubIncomingRefsFromView(idx.graph, stages, view) if markerBatch != nil { markerBatch.npmConsumerFiles = appendUniqueSorted(markerBatch.npmConsumerFiles, npmConsumers...) + markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, cargoConsumers...) } idx.applyLap("restub") // The capability state of the files, read before the eviction below @@ -1370,8 +1371,9 @@ func restubIncomingRefsFromView( g graph.Store, stages []*incrementalBatchStage, view incrementalPriorView, -) ([]*graph.Edge, []string) { +) ([]*graph.Edge, []string, []string) { evicted := structuralPriorIDs(stages) + consumers := restubTrackedCargoImportsFromView(g, evicted, view.inByNode) reparsed := reparsedPaths(stages) npmConsumers := restubTrackedNpmImportsFromView(g, evicted, view.inByNode) var reindexes []graph.EdgeReindex @@ -1410,7 +1412,7 @@ func restubIncomingRefsFromView( } } else { for _, edge := range view.inByNode[node.ID] { - if edge == nil || edge.Kind != graph.EdgeImports { + if edge == nil || edge.Kind != graph.EdgeImports || edge.To != node.ID { continue } if _, sourceEvicted := evicted[edge.From]; sourceEvicted { @@ -1552,7 +1554,7 @@ func restubIncomingRefsFromView( if len(reindexes) > 0 { g.ReindexEdges(reindexes) } - return canonicalizeLegacyCarried(stages, evicted, carried), npmConsumers + return canonicalizeLegacyCarried(stages, evicted, carried), npmConsumers, consumers } // dataflowSourceSite is a reference's site: the file that records it and its @@ -2425,7 +2427,7 @@ func (idx *Indexer) evictFileIncrementalRaw(relPath string) forcedFileEviction { // recover this file's records from a sibling's canonical scalar payload. idx.ensureIncrementalContractRegistry() var invalidation DerivedInvalidationPlan - nodesRemoved, edgesRemoved := idx.evictDeletedFilesBatched([]string{relPath}, &invalidation) + nodesRemoved, edgesRemoved := idx.evictDeletedFilesBatched([]string{relPath}, &invalidation, nil) graphPath := idx.prefixPath(relPath) idx.removeIncrementalContractsForFile(graphPath, &invalidation) invalidation.Files = appendUniqueSorted(invalidation.Files, dependencyFiles...) @@ -2475,7 +2477,7 @@ func (idx *Indexer) removeIncrementalContractsForFile(graphPath string, plan *De plan.ContractSymbolIDs = appendUniqueSorted(plan.ContractSymbolIDs, refresh.SymbolIDs...) } -func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInvalidationPlan) (nodesRemoved, edgesRemoved int) { +func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInvalidationPlan, markerBatch *reparsePendingEnrichmentBatch) (nodesRemoved, edgesRemoved int) { if len(deleted) == 0 { return 0, 0 } @@ -2539,7 +2541,11 @@ func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInva // carry therefore means a deletion stage stopped being conservative, // which would destroy those edges with no compile error to show for // it — so the assumption is checked, not just documented. - if carried, _ := restubIncomingRefsFromView(idx.graph, stages, view); len(carried) > 0 { + carried, _, consumers := restubIncomingRefsFromView(idx.graph, stages, view) + if markerBatch != nil { + markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, consumers...) + } + if len(carried) > 0 { idx.logger.Error("indexer: deletion restub carried in-edges with no re-state path", zap.String("repo", idx.repoPrefix), zap.Int("carried", len(carried)), diff --git a/internal/indexer/incremental_restub_gate_test.go b/internal/indexer/incremental_restub_gate_test.go index 7f25e99ba..8edc5283d 100644 --- a/internal/indexer/incremental_restub_gate_test.go +++ b/internal/indexer/incremental_restub_gate_test.go @@ -118,7 +118,7 @@ func runRestubStage( stages := []*incrementalBatchStage{stage} view := loadIncrementalPriorView(st, stages) - carried, _ := restubIncomingRefsFromView(st, stages, view) + carried, _, _ := restubIncomingRefsFromView(st, stages, view) evictFilesBatched(st, []string{restubDefPath}) st.AddBatch(fresh, append(append([]*graph.Edge{}, freshEdges...), carried...)) diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index 86f241bbf..ce8274718 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -539,6 +539,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( crossRepoFiles := appendUniqueSorted(nil, files...) if batch != nil { crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.npmConsumerFiles...) + crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.cargoConsumerFiles...) } if result != nil { crossRepoFiles = appendUniqueSorted(crossRepoFiles, result.DerivedInvalidation.Files...) @@ -588,7 +589,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } crossRepoFiles = appendUniqueSorted(crossRepoFiles, semanticFiles...) } - if needed || len(semanticFiles) > 0 || batch != nil && len(batch.npmConsumerFiles) > 0 { + if needed || len(semanticFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || len(batch.cargoConsumerFiles) > 0) { if idx != nil { idx.observeIncrementalCatchup("cross_repo", crossRepoFiles) } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 70d8f8fde..618d59428 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -3442,6 +3442,10 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index var err error shadowReplacementStarted = true restubTrackedNpmRepoIncoming(diskTarget, idx.RepoPrefix()) + parked := restubTrackedCargoRepoIncoming(diskTarget, idx.RepoPrefix()) + if result != nil { + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, parked...) + } evictedNodes, evictedEdges, err = sqlite.EvictRepoForShadowReplacement(ctx, idx.RepoPrefix()) if err != nil { retErr = fmt.Errorf("indexer: retire shadow replacement rows: %w", err) @@ -5169,9 +5173,9 @@ func (idx *Indexer) indexFile( // batch replacement by deleting its prior documents explicitly; // canonical contracts follow the store's owner-retention decision. idx.deleteSymbolFTS(oldFTSNodeIDs) - var npmConsumers []string + var npmConsumers, cargoConsumers []string if len(legacyIDs) == 0 { - npmConsumers = idx.restubIncomingRefs(graphPath) + npmConsumers, cargoConsumers = idx.restubIncomingRefs(graphPath) } else { // Extraction is still unprefixed here. Build only the identity surface // needed by the same incoming frontier used by the structural batch. @@ -5186,10 +5190,11 @@ func (idx *Indexer) indexFile( } stage := &incrementalBatchStage{graphPath: graphPath, legacyPath: legacyPath, priorNodes: priorNodes, result: fresh} stages := []*incrementalBatchStage{stage} - legacyCarried, npmConsumers = restubIncomingRefsFromView(idx.graph, stages, loadIncrementalPriorView(idx.graph, stages)) + legacyCarried, npmConsumers, cargoConsumers = restubIncomingRefsFromView(idx.graph, stages, loadIncrementalPriorView(idx.graph, stages)) } if markerBatch != nil { markerBatch.npmConsumerFiles = appendUniqueSorted(markerBatch.npmConsumerFiles, npmConsumers...) + markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, cargoConsumers...) } if len(legacyIDs) == 0 { idx.graph.EvictFile(graphPath) @@ -5952,10 +5957,10 @@ func (idx *Indexer) reresolveFileScopedRaw(filePath string) error { // agnostic: GetInEdges + ReindexEdges are the same Store primitives the // resolver uses, so this behaves identically on the in-memory and disk // stores. -func (idx *Indexer) restubIncomingRefs(graphPath string) []string { +func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string) { nodes := idx.graph.GetFileNodes(graphPath) if len(nodes) == 0 { - return nil + return nil, nil } evicted := make(map[string]struct{}, len(nodes)) refIDs := make([]string, 0, len(nodes)) @@ -5969,13 +5974,14 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) []string { } } if len(refIDs) == 0 { - return nil + return nil, nil } // Fetch the whole changed file's incoming adjacency in one bounded store // operation. On SQLite this is chunked under the parameter limit; more // importantly, it avoids one transaction/query per definition in the file. inEdges := idx.graph.GetInEdgesByNodeIDs(refIDs) npmConsumers := restubTrackedNpmImportsFromView(idx.graph, evicted, inEdges) + consumers := restubTrackedCargoImportsFromView(idx.graph, evicted, inEdges) var batch []graph.EdgeReindex for _, n := range nodes { if n == nil || n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) && n.Kind != graph.KindFile { @@ -6009,7 +6015,7 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) []string { if len(batch) > 0 { idx.graph.ReindexEdges(batch) } - return npmConsumers + return npmConsumers, consumers } // embeddingDimsOrDefault returns the embedder's reported vector width, @@ -7537,6 +7543,7 @@ func (idx *Indexer) incrementalReindexPathsMode( capabilityPrior: idx.takeCapabilityPrior(), } result.CrossRepoResolveFiles = appendUniqueSorted(nil, markerBatch.npmConsumerFiles...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, markerBatch.cargoConsumerFiles...) for _, path := range failedFiles { result.Errors = append(result.Errors, IndexError{FilePath: path, Error: idx.fileIndexFailureError(path).Error()}) } diff --git a/internal/indexer/legacy_file_identity_test.go b/internal/indexer/legacy_file_identity_test.go index a5f494302..f0fbc5182 100644 --- a/internal/indexer/legacy_file_identity_test.go +++ b/internal/indexer/legacy_file_identity_test.go @@ -230,7 +230,7 @@ func TestLegacyIdentityRepairRebindsForeignCaller(t *testing.T) { prior, ids, native := mergeLegacyFilePrior(legacy, fresh, old) stage := &incrementalBatchStage{graphPath: path, legacyPath: native, legacyIDs: ids, priorNodes: prior, result: &parser.ExtractionResult{Nodes: fresh}} stages := []*incrementalBatchStage{stage} - carried, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) + carried, _, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) idx := New(store, parser.NewRegistry(), config.Default().Index, zap.NewNop()) idx.SetRepoPrefix("repo") idx.evictFileIdentities([]string{path}, ids, []string{native}) @@ -308,7 +308,7 @@ func TestLegacyIdentityRepairRespectsDataflowFrontier(t *testing.T) { Nodes: fresh, Edges: []*graph.Edge{{From: fresh[2].ID, To: fresh[1].ID, Kind: graph.EdgeParamOf, FilePath: path}}, }} stages := []*incrementalBatchStage{stage} - carried, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) + carried, _, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) store.(graph.FileIdentityBatchEvicter).EvictFilesWithNodeIDs([]string{path}, ids) store.AddBatch(fresh, append(stage.result.Edges, carried...)) find := func(from, to string, kind graph.EdgeKind) bool { diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index f2507de14..afedc1a21 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -2789,6 +2789,7 @@ func (mi *MultiIndexer) indexRepoRaw(repoPrefix string) (*IndexResult, error) { if err := mi.postPublishGoModules(context.Background(), idx, false); err != nil { mi.logger.Warn("post-publish Go module follow-up", zap.Error(err)) } + mi.runCrossRepoResolveFiles(mi.cargoConsumerFilesForRepo(repoPrefix)) mi.ReconcileContractEdges() @@ -3434,6 +3435,7 @@ func (mi *MultiIndexer) trackPreparedRepoSourceCtx( // repository's first registration, never a rewrite of an entry. mi.markRepoRegistered(prefix, absPath) mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix)) + mi.runCrossRepoResolveFiles(mi.cargoConsumerFilesForRepo(prefix)) // Add to global config. entry.Path = absPath @@ -3741,8 +3743,10 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE }) } } else if result.FullRetrack { + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(prefix)...) if !batchMode.deferResolve { mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix)) + mi.runCrossRepoResolveFiles(result.CrossRepoResolveFiles) } if !batchMode.deferGlobalPasses { if err := mi.postPublishGoModules(ctx, idx, false); err != nil { diff --git a/internal/indexer/npm_lifecycle_test.go b/internal/indexer/npm_lifecycle_test.go index 4dc83a4f6..97e4b4fd6 100644 --- a/internal/indexer/npm_lifecycle_test.go +++ b/internal/indexer/npm_lifecycle_test.go @@ -38,11 +38,12 @@ func TestTrackedNpmImportsSurviveEviction(t *testing.T) { switch mode { case "file": idx := New(g, newTestRegistry(), config.IndexConfig{}, zap.NewNop()) - assert.Equal(t, []string{"app/a.ts"}, idx.restubIncomingRefs("lib/index.ts")) + consumers, _ := idx.restubIncomingRefs("lib/index.ts") + assert.Equal(t, []string{"app/a.ts"}, consumers) g.EvictFile("lib/index.ts") case "batch": stages := []*incrementalBatchStage{{graphPath: "lib/index.ts", priorNodes: nodes}} - _, consumers := restubIncomingRefsFromView(g, stages, loadIncrementalPriorView(g, stages)) + _, consumers, _ := restubIncomingRefsFromView(g, stages, loadIncrementalPriorView(g, stages)) assert.Equal(t, []string{"app/a.ts"}, consumers) g.EvictFile("lib/index.ts") case "replace": diff --git a/internal/indexer/repo_eviction_scope.go b/internal/indexer/repo_eviction_scope.go index bed6e0478..131a6f5c3 100644 --- a/internal/indexer/repo_eviction_scope.go +++ b/internal/indexer/repo_eviction_scope.go @@ -11,6 +11,7 @@ import ( // Store eviction is the compatible fallback. func evictRepoCurrentGeneration(target graph.Store, repoPrefix string) (nodesRemoved, edgesRemoved int) { restubTrackedNpmRepoIncoming(target, repoPrefix) + restubTrackedCargoRepoIncoming(target, repoPrefix) if scoped, ok := target.(graph.CurrentGenerationRepoEvicter); ok { return scoped.EvictRepoCurrentGeneration(repoPrefix) } diff --git a/internal/indexer/repository_untrack.go b/internal/indexer/repository_untrack.go index 3915be809..9622f74d5 100644 --- a/internal/indexer/repository_untrack.go +++ b/internal/indexer/repository_untrack.go @@ -291,6 +291,7 @@ func (mi *MultiIndexer) purgeRepositoryPayload( ) (nodesRemoved, edgesRemoved int, err error) { restubTrackedNpmRepoIncoming(mi.graph, repoPrefix) parked := restubGoModuleRepoIncoming(mi.graph, repoPrefix) + restubTrackedCargoRepoIncoming(mi.graph, repoPrefix) if purger, ok := mi.graph.(interface{ PurgeRepo(string) error }); ok { if err := purger.PurgeRepo(repoPrefix); err != nil { return 0, 0, fmt.Errorf("purge repository payload: %w", err) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index 97d4943a8..a88580aa6 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -9,6 +9,9 @@ import ( "github.com/zzet/gortex/internal/graph" ) +// CargoImportStubMeta preserves the import's exact target before a tracked bind. +const CargoImportStubMeta = "cargo_import_stub" + // CargoCrate identifies a provider library and its exposed crate name. type CargoCrate struct { Root string // graph-path library file @@ -152,6 +155,10 @@ func cargoImportSpec(e *graph.Edge) string { } func (cr *CrossRepoResolver) bindCargoImport(e *graph.Edge, node *graph.Node, stats *CrossRepoStats) { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + e.Meta[CargoImportStubMeta] = e.To e.To, e.CrossRepo = node.ID, true clearEdgeTerminal(e) stats.Resolved++ From ebf503d99c44f14085dc53259043d8d152254d3e Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 01:42:42 -0700 Subject: [PATCH 015/111] fix(indexer): preserve exact Cargo call stubs within eviction scope Record the pre-link function or method target instead of deriving a stub from the resolved symbol ID. Park only incoming calls to evicted targets, using the existing view rather than re-reading SQLite edge copies. Keep #806's save carry/restub decisions and dataflow sites. Apply the recorded call stub only when that existing path chooses to restub, and pass forced file eviction's existing batch through the deletion core. Compare imports and calls with a fresh cold index on both stores and track orders, including associated functions, methods and forced eviction. --- .../indexer/cargo_import_lifecycle_test.go | 79 +++++++++++++++---- internal/indexer/cargo_import_restub.go | 62 +++++---------- internal/indexer/incremental_batch.go | 7 +- internal/indexer/indexer.go | 5 +- internal/resolver/cargo_crate.go | 3 + internal/resolver/cross_repo.go | 6 ++ 6 files changed, 104 insertions(+), 58 deletions(-) diff --git a/internal/indexer/cargo_import_lifecycle_test.go b/internal/indexer/cargo_import_lifecycle_test.go index 9a3a37dc6..2ea9e073e 100644 --- a/internal/indexer/cargo_import_lifecycle_test.go +++ b/internal/indexer/cargo_import_lifecycle_test.go @@ -3,7 +3,11 @@ package indexer import ( "context" "encoding/json" + "fmt" + "os" "path/filepath" + "slices" + "strings" "testing" "github.com/stretchr/testify/require" @@ -20,7 +24,7 @@ import ( func TestTrackedCargoImportLifecycle(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { for _, providerFirst := range []bool{false, true} { - for _, operation := range []string{"consumer_save", "provider_save", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track"} { + for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track"} { order := "consumer_first" if providerFirst { order = "provider_first" @@ -28,9 +32,10 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { t.Run(backend+"/"+order+"/"+operation, func(t *testing.T) { provider, consumer := t.TempDir(), t.TempDir() writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") - writeWorkspaceFile(t, provider, "src/lib.rs", "pub fn action() {}\n") + writeWorkspaceFile(t, provider, "src/lib.rs", cargoLifecycleProvider) + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") - writeWorkspaceFile(t, consumer, "src/lib.rs", "use other_crate::action;\npub fn run() { action(); }\n") + writeWorkspaceFile(t, consumer, "src/lib.rs", cargoLifecycleConsumer) entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} cfgPath := filepath.Join(t.TempDir(), "config.yaml") gc := &config.GlobalConfig{Repos: entries} @@ -71,28 +76,47 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { imports = append(imports, edge) } } - require.Len(t, imports, 1) - require.Equal(t, "provider/src/lib.rs", imports[0].To) - require.True(t, imports[0].CrossRepo) - var calls []*graph.Edge + require.Len(t, imports, 3) + for _, edge := range imports { + require.True(t, edge.CrossRepo) + } + calls := make(map[string]string) for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { if edge.Kind == graph.EdgeCalls { - calls = append(calls, edge) + require.True(t, edge.CrossRepo) + stub, _ := edge.Meta["cargo_call_stub"].(string) + calls[edge.To] = stub } } - require.Len(t, calls, 1) - require.Equal(t, "provider/src/lib.rs::action", calls[0].To) - require.True(t, calls[0].CrossRepo) + require.Equal(t, map[string]string{ + "provider/src/lib.rs::Foo.new": "unresolved::new", + "provider/src/lib.rs::Foo.go": "unresolved::*.go", + "provider/src/lib.rs::action": "unresolved::action", + "provider/src/util.rs::helper": "unresolved::helper", + }, calls) } assertImport() logs.TakeAll() switch operation { case "consumer_save": - writeWorkspaceFile(t, consumer, "src/lib.rs", "use other_crate::action;\npub fn run() { action(); }\npub fn edited() {}\n") + writeWorkspaceFile(t, consumer, "src/lib.rs", cargoLifecycleConsumer+"pub fn edited() {}\n") _, err = mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "src/lib.rs")}) case "provider_save": - writeWorkspaceFile(t, provider, "src/lib.rs", "pub fn action() {}\npub fn added() {}\n") + writeWorkspaceFile(t, provider, "src/lib.rs", cargoLifecycleProvider+"pub fn added() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/lib.rs")}) + case "provider_save_util": + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\npub fn added() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) + case "provider_save_lib_contract": + writeWorkspaceFile(t, provider, "src/lib.rs", strings.Replace(cargoLifecycleProvider, "pub fn action() {}", "pub fn action(x: u8) {}", 1)) _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/lib.rs")}) + case "provider_delete_util_batched": + require.NoError(t, os.Remove(filepath.Join(provider, "src/util.rs"))) + writeWorkspaceFile(t, provider, "src/lib.rs", strings.Replace(cargoLifecycleProvider, "pub mod util;\n", "", 1)) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs"), filepath.Join(provider, "src/lib.rs")}) + case "forced_evict_util": + require.NoError(t, os.Remove(filepath.Join(provider, "src/util.rs"))) + mi.GetIndexer("provider").EvictFile(filepath.Join(provider, "src/util.rs")) case "consumer_full": _, err = mi.IndexRepo("consumer") case "provider_full": @@ -132,7 +156,18 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { _, err = mi.TrackRepoCtx(context.Background(), entries[1]) } require.NoError(t, err) - assertImport() + var cold graph.Store = graph.New() + if backend == "sqlite" { + db, openErr := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, openErr) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + cold = db + } + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, cargoLifecycleBindings(cold), cargoLifecycleBindings(g), "after %s", operation) require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) require.Empty(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveAll").All()) }) @@ -140,3 +175,19 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { } } } + +const cargoLifecycleProvider = "pub mod util;\npub struct Foo;\nimpl Foo {\n pub fn new() -> Foo { Foo }\n pub fn go(&self) {}\n}\npub fn action() {}\n" +const cargoLifecycleConsumer = "use other_crate::Foo;\nuse other_crate::action;\nuse other_crate::util::helper;\npub fn run() {\n let f = Foo::new();\n f.go();\n action();\n helper();\n}\n" + +func cargoLifecycleBindings(g graph.Store) []string { + var bindings []string + for _, node := range g.GetFileNodes("consumer/src/lib.rs") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind == graph.EdgeImports || edge.Kind == graph.EdgeCalls { + bindings = append(bindings, fmt.Sprintf("%s -%s-> %s cross=%v line=%d", edge.From, edge.Kind, edge.To, edge.CrossRepo, edge.Line)) + } + } + } + slices.Sort(bindings) + return bindings +} diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go index 34f296f67..dfec5fa0c 100644 --- a/internal/indexer/cargo_import_restub.go +++ b/internal/indexer/cargo_import_restub.go @@ -1,8 +1,6 @@ package indexer import ( - "maps" - "slices" "strings" "github.com/zzet/gortex/internal/graph" @@ -25,11 +23,12 @@ func restubTrackedCargoRepoIncoming(g graph.Store, repo string) []string { } } } - files = appendUniqueSorted(files, restubTrackedCargoImportsFromView(g, evicted, incoming)...) + files = appendUniqueSorted(files, restubTrackedCargoImportsFromView(g, evicted, incoming, true)...) ids = ids[:0] } - for node := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFile) { - if !strings.HasSuffix(node.ID, ".rs") { + for node := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFile, graph.KindFunction, graph.KindMethod) { + file, _, _ := strings.Cut(node.ID, "::") + if !strings.HasSuffix(file, ".rs") { continue } ids = append(ids, node.ID) @@ -43,61 +42,42 @@ func restubTrackedCargoRepoIncoming(g graph.Store, repo string) []string { return files } -func restubTrackedCargoImportsFromView(g graph.Store, evicted map[string]struct{}, incoming map[string][]*graph.Edge) []string { - consumers := make(map[string]map[string]bool) +func restubTrackedCargoImportsFromView(g graph.Store, evicted map[string]struct{}, incoming map[string][]*graph.Edge, parkCalls bool) []string { + consumers := make(map[string]struct{}) var batch []graph.EdgeReindex for _, edges := range incoming { for _, edge := range edges { - if edge == nil || edge.Kind != graph.EdgeImports { + if edge == nil || !edge.CrossRepo { continue } - stub, _ := edge.Meta[resolver.CargoImportStubMeta].(string) - if _, gone := evicted[edge.From]; stub == "" || gone || edge.To == stub { + key := resolver.CargoImportStubMeta + if edge.Kind == graph.EdgeCalls { + key = resolver.CargoCallStubMeta + } else if edge.Kind != graph.EdgeImports { continue } - if consumers[edge.FilePath] == nil { - consumers[edge.FilePath] = make(map[string]bool) - } - consumers[edge.FilePath][graph.RepoPrefixOfID(edge.To)] = true - old := edge.To - edge.To, edge.CrossRepo = stub, false - batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: old}) - } - } - files := slices.Sorted(maps.Keys(consumers)) - if len(files) == 0 { - return nil - } - var ids []string - for _, nodes := range g.GetFileNodesByPaths(files) { - for _, node := range nodes { - if node != nil { - ids = append(ids, node.ID) - } - } - } - for _, edges := range g.GetOutEdgesByNodeIDs(ids) { - for _, edge := range edges { - if edge == nil || edge.Kind != graph.EdgeCalls || !edge.CrossRepo || !consumers[edge.FilePath][graph.RepoPrefixOfID(edge.To)] { + stub, _ := edge.Meta[key].(string) + if _, gone := evicted[edge.From]; stub == "" || gone || edge.To == stub { continue } - if _, gone := evicted[edge.From]; gone { + consumers[edge.FilePath] = struct{}{} + // Save paths keep the existing carry/restub decision and its dataflow sites. + if edge.Kind == graph.EdgeCalls && !parkCalls { continue } old := edge.To - _, name, symbol := strings.Cut(old, "::") - if !symbol { - continue + if edge.Kind == graph.EdgeCalls { + graph.StashRestubProvenance(edge) + edge.ConfidenceLabel = "" } - graph.StashRestubProvenance(edge) - edge.To, edge.CrossRepo, edge.ConfidenceLabel = graph.UnresolvedMarker+name, false, "" + edge.To, edge.CrossRepo = stub, false batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: old}) } } if len(batch) > 0 { g.ReindexEdges(batch) } - return files + return sortedStringKeys(consumers) } // Retry only parked imports claimed by this provider, or owned by this repo. diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index b7c3b5e92..e55b69289 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -1373,7 +1373,7 @@ func restubIncomingRefsFromView( view incrementalPriorView, ) ([]*graph.Edge, []string, []string) { evicted := structuralPriorIDs(stages) - consumers := restubTrackedCargoImportsFromView(g, evicted, view.inByNode) + consumers := restubTrackedCargoImportsFromView(g, evicted, view.inByNode, false) reparsed := reparsedPaths(stages) npmConsumers := restubTrackedNpmImportsFromView(g, evicted, view.inByNode) var reindexes []graph.EdgeReindex @@ -1547,6 +1547,9 @@ func restubIncomingRefsFromView( if edge.To != stub { edge.CrossRepo, edge.ConfidenceLabel = false, "" } + if original, _ := edge.Meta[resolver.CargoCallStubMeta].(string); edge.Kind == graph.EdgeCalls && original != "" { + edge.To, edge.CrossRepo, edge.ConfidenceLabel = original, false, "" + } reindexes = append(reindexes, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) } } @@ -2427,7 +2430,7 @@ func (idx *Indexer) evictFileIncrementalRaw(relPath string) forcedFileEviction { // recover this file's records from a sibling's canonical scalar payload. idx.ensureIncrementalContractRegistry() var invalidation DerivedInvalidationPlan - nodesRemoved, edgesRemoved := idx.evictDeletedFilesBatched([]string{relPath}, &invalidation, nil) + nodesRemoved, edgesRemoved := idx.evictDeletedFilesBatched([]string{relPath}, &invalidation, batch) graphPath := idx.prefixPath(relPath) idx.removeIncrementalContractsForFile(graphPath, &invalidation) invalidation.Files = appendUniqueSorted(invalidation.Files, dependencyFiles...) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 618d59428..33b710a17 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -5981,7 +5981,7 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string) { // importantly, it avoids one transaction/query per definition in the file. inEdges := idx.graph.GetInEdgesByNodeIDs(refIDs) npmConsumers := restubTrackedNpmImportsFromView(idx.graph, evicted, inEdges) - consumers := restubTrackedCargoImportsFromView(idx.graph, evicted, inEdges) + consumers := restubTrackedCargoImportsFromView(idx.graph, evicted, inEdges, false) var batch []graph.EdgeReindex for _, n := range nodes { if n == nil || n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) && n.Kind != graph.KindFile { @@ -6009,6 +6009,9 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string) { if e.Kind == graph.EdgeImports { e.CrossRepo, e.ConfidenceLabel = false, "" } + if original, _ := e.Meta[resolver.CargoCallStubMeta].(string); e.Kind == graph.EdgeCalls && original != "" { + e.To, e.CrossRepo, e.ConfidenceLabel = original, false, "" + } batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) } } diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index a88580aa6..9d1f2c328 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -12,6 +12,9 @@ import ( // CargoImportStubMeta preserves the import's exact target before a tracked bind. const CargoImportStubMeta = "cargo_import_stub" +// CargoCallStubMeta retains the extractor's function or method target before linking. +const CargoCallStubMeta = "cargo_call_stub" + // CargoCrate identifies a provider library and its exposed crate name. type CargoCrate struct { Root string // graph-path library file diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index dd055b2f7..645188b3b 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -1286,6 +1286,12 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b } if e.To != oldTo { + if e.Kind == graph.EdgeCalls && e.CrossRepo && cr.cargoCrates != nil && strings.HasSuffix(e.FilePath, ".rs") { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + e.Meta[CargoCallStubMeta] = oldTo + } *batch = append(*batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) cr.noteRetargetedCall(e) } From 2446124d7c935a011b5882791665a1c0ccd62b14 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 02:12:30 -0700 Subject: [PATCH 016/111] fix(indexer): honor Cargo manifest sources and deferred frontiers Use main's installed manifest-tree boundary for Cargo roots, workspace members and nearest-manifest checks instead of reading another revision's checkout. Keep metadata-only targets out of save-time import parking. Retain IndexRepo's cross-repo consumer result and skip its runtime-track resolver tail when the batch defers resolution. Cover both failures and snapshot manifests, and make warm-retrack tests use the configured order. --- internal/indexer/cargo_crate.go | 40 +++++++------ .../indexer/cargo_import_boundary_test.go | 56 ++++++++++++++++++ .../indexer/cargo_import_lifecycle_test.go | 57 ++++++++++++++----- internal/indexer/cargo_import_restub.go | 5 +- internal/indexer/cargo_import_restub_test.go | 4 ++ internal/indexer/multi.go | 7 ++- 6 files changed, 137 insertions(+), 32 deletions(-) create mode 100644 internal/indexer/cargo_import_boundary_test.go diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go index bf755a82a..ceb051d21 100644 --- a/internal/indexer/cargo_crate.go +++ b/internal/indexer/cargo_crate.go @@ -1,7 +1,6 @@ package indexer import ( - "os" "path" "strings" "sync" @@ -16,8 +15,8 @@ func cargoTable(m map[string]any, key string) map[string]any { return table } -func readCargoManifest(root, dir string) map[string]any { - src, ok := readDiskFile(joinPath(root, path.Join(dir, "Cargo.toml"))) +func readCargoManifest(tree manifestTree, dir string) map[string]any { + src, ok := tree.readFile(path.Join(dir, "Cargo.toml")) if !ok { return nil } @@ -36,11 +35,12 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { deps map[string]string imports map[string][]resolver.CargoCrate } - roots := map[string]string{} + roots := map[string]manifestTree{} for prefix, meta := range mi.AllMetadata() { if meta != nil && meta.RootPath != "" { - if _, err := os.Stat(joinPath(meta.RootPath, "Cargo.toml")); err == nil { - roots[prefix] = meta.RootPath + tree := mi.manifestTreeFor(prefix, meta.RootPath) + if tree.isFile("Cargo.toml") { + roots[prefix] = tree } } } @@ -54,21 +54,30 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { var once sync.Once return func(file, name string) []resolver.CargoCrate { once.Do(func() { - for prefix, root := range roots { - manifest := readCargoManifest(root, ".") - src, _ := readDiskFile(joinPath(root, "Cargo.toml")) + for prefix, tree := range roots { + manifest := readCargoManifest(tree, ".") + src, _ := tree.readFile("Cargo.toml") members, exclude := modules.ParseCargoWorkspace(src) dirs := []string{"."} - for _, member := range modules.ResolveWorkspaceMembers(root, &modules.WorkspaceManifest{Ecosystem: modules.WorkspaceCargo, Patterns: members, Exclude: exclude}) { - if !path.IsAbs(member.Dir) && !hasDotDotSegment(member.Dir) { - dirs = append(dirs, member.Dir) + skip := map[string]bool{".": true} + for _, pattern := range exclude { + for _, dir := range tree.matchDirs(pattern) { + skip[dir] = true + } + } + for _, pattern := range members { + for _, dir := range tree.matchDirs(pattern) { + if !path.IsAbs(dir) && !hasDotDotSegment(dir) && !skip[dir] && tree.isFile(path.Join(dir, "Cargo.toml")) { + dirs = append(dirs, dir) + skip[dir] = true + } } } inherited := cargoTable(cargoTable(manifest, "workspace"), "dependencies") for _, dir := range dirs { m := manifest if dir != "." { - m = readCargoManifest(root, dir) + m = readCargoManifest(tree, dir) } if m == nil { continue @@ -146,7 +155,7 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { defer ownerMu.Unlock() var prefix string for dir := path.Dir(file); dir != "."; dir = path.Dir(dir) { - if roots[dir] != "" { + if roots[dir] != nil { prefix = dir break } @@ -170,8 +179,7 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { if dir == prefix { rel = "." } - _, err := os.Stat(joinPath(roots[prefix], path.Join(rel, "Cargo.toml"))) - if !os.IsNotExist(err) || dir == prefix { + if roots[prefix].isFile(path.Join(rel, "Cargo.toml")) || dir == prefix { break // Unknown, malformed, or unreadable manifest: no ownership evidence. } } diff --git a/internal/indexer/cargo_import_boundary_test.go b/internal/indexer/cargo_import_boundary_test.go new file mode 100644 index 000000000..63fe1e730 --- /dev/null +++ b/internal/indexer/cargo_import_boundary_test.go @@ -0,0 +1,56 @@ +package indexer + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +func TestCargoCrateLookupUsesInstalledManifestTree(t *testing.T) { + for _, workspace := range []bool{false, true} { + t.Run(map[bool]string{false: "root", true: "workspace"}[workspace], func(t *testing.T) { + provider, consumer, providerSnapshot, consumerSnapshot := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'wrong-checkout'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'wrong-app'\n") + providerDir, consumerDir := "", "" + dependency := "[dependencies]\nalias = { package = 'other-crate', version = '1' }\n" + if workspace { + providerDir, consumerDir = "crates/engine/", "apps/app/" + writeWorkspaceFile(t, providerSnapshot, "Cargo.toml", "[workspace]\nmembers = ['crates/*']\n") + writeWorkspaceFile(t, consumerSnapshot, "Cargo.toml", "[workspace]\nmembers = ['apps/*']\nexclude = ['apps/excluded']\n[workspace.dependencies]\nalias = { package = 'other-crate', version = '1' }\n") + writeWorkspaceFile(t, consumerSnapshot, "apps/excluded/Cargo.toml", "[package]\nname = 'excluded'\n") + dependency = "[dependencies]\nalias = { workspace = true }\n" + } + writeWorkspaceFile(t, providerSnapshot, providerDir+"Cargo.toml", "[package]\nname = 'other-crate'\n") + writeWorkspaceFile(t, consumerSnapshot, consumerDir+"Cargo.toml", "[package]\nname = 'app'\n"+dependency) + g := graph.New() + providerIdx, consumerIdx := sideChannelIndexer(t, g, provider), sideChannelIndexer(t, g, consumer) + providerIdx.SetContentSource(snapshotOf(t, providerSnapshot)) + consumerIdx.SetContentSource(snapshotOf(t, consumerSnapshot)) + mi := &MultiIndexer{graph: g, + repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}, + indexers: map[string]*Indexer{"provider": providerIdx, "consumer": consumerIdx}} + lookup := mi.cargoCrateLookup() + require.NotNil(t, lookup) + require.Equal(t, []resolver.CargoCrate{{Root: "provider/" + providerDir + "src/lib.rs", Name: "other_crate"}}, lookup("consumer/"+consumerDir+"src/lib.rs", "alias")) + if workspace { + require.Empty(t, lookup("consumer/apps/excluded/src/lib.rs", "alias")) + } + }) + } +} + +func TestCargoRestubIgnoresMetadataOnlyTargets(t *testing.T) { + g := graph.New() + target := "provider/src/meta.rs" + edge := &graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: target, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{resolver.CargoImportStubMeta: "external::other_crate/meta"}} + g.AddNode(&graph.Node{ID: target, Kind: graph.KindFile, RepoPrefix: "provider"}) + g.AddEdge(edge) + incoming := g.GetInEdgesByNodeIDs([]string{target}) + require.Empty(t, restubTrackedCargoImportsFromView(g, map[string]struct{}{"provider/src/changed.rs": {}}, incoming, false)) + require.Equal(t, target, edge.To) + require.True(t, edge.CrossRepo) +} diff --git a/internal/indexer/cargo_import_lifecycle_test.go b/internal/indexer/cargo_import_lifecycle_test.go index 2ea9e073e..94d7571e1 100644 --- a/internal/indexer/cargo_import_lifecycle_test.go +++ b/internal/indexer/cargo_import_lifecycle_test.go @@ -24,7 +24,10 @@ import ( func TestTrackedCargoImportLifecycle(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { for _, providerFirst := range []bool{false, true} { - for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track"} { + for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track", "deferred_track"} { + if operation == "deferred_track" && providerFirst { + continue // This case starts with an already tracked consumer. + } order := "consumer_first" if providerFirst { order = "provider_first" @@ -59,7 +62,20 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { logger := zap.New(core) mi := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) - if operation == "track" { + if operation == "deferred_track" { + _, err = mi.TrackRepoCtx(context.Background(), entries[0]) + require.NoError(t, err) + mi.BeginParallelBatch() + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + require.NoError(t, err) + for _, edge := range g.GetOutEdges("consumer/src/lib.rs") { + if edge.Kind == graph.EdgeImports { + require.False(t, edge.CrossRepo, "deferred track must not resolve shared imports early") + } + } + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs"}, nil)) + mi.EndBatch() + } else if operation == "track" { for _, entry := range gc.Repos { _, err = mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -120,30 +136,45 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { case "consumer_full": _, err = mi.IndexRepo("consumer") case "provider_full": - _, err = mi.IndexRepo("provider") + var result *IndexResult + result, err = mi.IndexRepo("provider") + require.NoError(t, err) + require.Contains(t, result.CrossRepoResolveFiles, "consumer/src/lib.rs") case "warm_full_retrack": prior := map[string]map[string]int64{"consumer": mi.FileMtimes("consumer"), "provider": mi.FileMtimes("provider")} restarted := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) t.Cleanup(func() { require.NoError(t, restarted.Close(context.Background())) }) restarted.BeginBatch() - _, err = restarted.ReconcileRepoCtx(context.Background(), entries[0], prior["consumer"]) - require.NoError(t, err) - t.Setenv("GORTEX_WARMUP_FULL_RETRACK", "1") - result, restoreErr := restarted.ReconcileRepoCtx(context.Background(), entries[1], prior["provider"]) - require.NoError(t, restoreErr) - require.True(t, result.FullRetrack) - data, marshalErr := json.Marshal(result) - require.NoError(t, marshalErr) var output struct { Files []string `json:"cross_repo_resolve_files"` } - require.NoError(t, json.Unmarshal(data, &output)) + for _, entry := range gc.Repos { + force := "0" + if entry.Name == "provider" { + force = "1" + } + t.Setenv("GORTEX_WARMUP_FULL_RETRACK", force) + result, restoreErr := restarted.ReconcileRepoCtx(context.Background(), entry, prior[entry.Name]) + require.NoError(t, restoreErr) + if entry.Name == "provider" { + require.True(t, result.FullRetrack) + } + data, marshalErr := json.Marshal(result) + require.NoError(t, marshalErr) + var current struct { + Files []string `json:"cross_repo_resolve_files"` + } + require.NoError(t, json.Unmarshal(data, ¤t)) + output.Files = appendUniqueSorted(output.Files, current.Files...) + if entry.Name == "provider" { + require.NotContains(t, result.DerivedInvalidation.Files, "consumer/src/lib.rs") + } + } if backend == "sqlite" { require.Contains(t, output.Files, "consumer/src/lib.rs") } else { require.Empty(t, output.Files, "memory replacement preserved the existing import") } - require.NotContains(t, result.DerivedInvalidation.Files, "consumer/src/lib.rs") err = restarted.RunPreEnrichResolveFiles(context.Background(), output.Files, nil) restarted.EndBatch() case "no_change": diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go index dfec5fa0c..72a0a3ee2 100644 --- a/internal/indexer/cargo_import_restub.go +++ b/internal/indexer/cargo_import_restub.go @@ -45,7 +45,10 @@ func restubTrackedCargoRepoIncoming(g graph.Store, repo string) []string { func restubTrackedCargoImportsFromView(g graph.Store, evicted map[string]struct{}, incoming map[string][]*graph.Edge, parkCalls bool) []string { consumers := make(map[string]struct{}) var batch []graph.EdgeReindex - for _, edges := range incoming { + for id, edges := range incoming { + if _, changing := evicted[id]; !parkCalls && !changing { + continue + } for _, edge := range edges { if edge == nil || !edge.CrossRepo { continue diff --git a/internal/indexer/cargo_import_restub_test.go b/internal/indexer/cargo_import_restub_test.go index 01c9680fb..51c8f7719 100644 --- a/internal/indexer/cargo_import_restub_test.go +++ b/internal/indexer/cargo_import_restub_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/require" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" ) func TestCargoRestubLeavesUnmarkedImportsUnchanged(t *testing.T) { @@ -13,6 +14,9 @@ func TestCargoRestubLeavesUnmarkedImportsUnchanged(t *testing.T) { g := graph.New() g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "provider"}) edge := &graph.Edge{From: "consumer/source", FilePath: "consumer/source", To: file, Kind: graph.EdgeImports, CrossRepo: true} + if file == "provider/main.go" { + edge.Meta = map[string]any{resolver.CargoImportStubMeta: "external::other_crate/action"} + } g.AddEdge(edge) require.Empty(t, restubTrackedCargoRepoIncoming(g, "provider")) require.Equal(t, file, edge.To) diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index afedc1a21..04955a7f6 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -2789,7 +2789,8 @@ func (mi *MultiIndexer) indexRepoRaw(repoPrefix string) (*IndexResult, error) { if err := mi.postPublishGoModules(context.Background(), idx, false); err != nil { mi.logger.Warn("post-publish Go module follow-up", zap.Error(err)) } - mi.runCrossRepoResolveFiles(mi.cargoConsumerFilesForRepo(repoPrefix)) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(repoPrefix)...) + mi.runCrossRepoResolveFiles(result.CrossRepoResolveFiles) mi.ReconcileContractEdges() @@ -3435,7 +3436,9 @@ func (mi *MultiIndexer) trackPreparedRepoSourceCtx( // repository's first registration, never a rewrite of an entry. mi.markRepoRegistered(prefix, absPath) mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix)) - mi.runCrossRepoResolveFiles(mi.cargoConsumerFilesForRepo(prefix)) + if !batchMode.deferResolve { + mi.runCrossRepoResolveFiles(mi.cargoConsumerFilesForRepo(prefix)) + } // Add to global config. entry.Path = absPath From 33dbf35b5e72c7b55f07aa284de67537087fcb3e Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 04:06:26 -0700 Subject: [PATCH 017/111] fix(indexer): retain Cargo call stubs across provider file recovery --- .../indexer/cargo_import_lifecycle_test.go | 106 ++++++++++++++++-- internal/indexer/cargo_import_restub.go | 7 +- internal/indexer/cargo_import_restub_test.go | 25 +++++ internal/indexer/edit_delta_prior_bindings.go | 19 +++- internal/indexer/file_meta.go | 1 + internal/indexer/incremental_batch.go | 9 +- internal/indexer/incremental_watcher_batch.go | 9 +- internal/indexer/indexer.go | 2 +- internal/indexer/multi.go | 6 +- internal/indexer/workspace_resolve.go | 9 +- internal/resolver/cargo_crate.go | 20 +++- .../resolver/cargo_import_retarget_test.go | 83 ++++++++++++++ internal/resolver/cross_repo.go | 3 +- internal/resolver/cross_repo_incremental.go | 4 +- 14 files changed, 270 insertions(+), 33 deletions(-) create mode 100644 internal/resolver/cargo_import_retarget_test.go diff --git a/internal/indexer/cargo_import_lifecycle_test.go b/internal/indexer/cargo_import_lifecycle_test.go index 94d7571e1..8b75e49b1 100644 --- a/internal/indexer/cargo_import_lifecycle_test.go +++ b/internal/indexer/cargo_import_lifecycle_test.go @@ -24,7 +24,7 @@ import ( func TestTrackedCargoImportLifecycle(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { for _, providerFirst := range []bool{false, true} { - for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track", "deferred_track"} { + for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "provider_gains_empty_module", "no_new_file_noop", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track", "deferred_track"} { if operation == "deferred_track" && providerFirst { continue // This case starts with an already tracked consumer. } @@ -35,10 +35,23 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { t.Run(backend+"/"+order+"/"+operation, func(t *testing.T) { provider, consumer := t.TempDir(), t.TempDir() writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") - writeWorkspaceFile(t, provider, "src/lib.rs", cargoLifecycleProvider) + providerSource := cargoLifecycleProvider + if operation == "provider_gains_new_module" || operation == "provider_gains_empty_module" { + providerSource += "pub mod newmod;\n" + } + writeWorkspaceFile(t, provider, "src/lib.rs", providerSource) writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + if operation == "no_new_file_noop" { + writeWorkspaceFile(t, provider, "src/spare.rs", "pub fn unused() {}\n") + } writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") - writeWorkspaceFile(t, consumer, "src/lib.rs", cargoLifecycleConsumer) + consumerSource := cargoLifecycleConsumer + if operation == "provider_gains_new_module" { + consumerSource += "use other_crate::newmod::later;\npub fn after() { later(); }\n" + } else if operation == "provider_gains_empty_module" { + consumerSource += "use other_crate::newmod::*;\n" + } + writeWorkspaceFile(t, consumer, "src/lib.rs", consumerSource) entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} cfgPath := filepath.Join(t.TempDir(), "config.yaml") gc := &config.GlobalConfig{Repos: entries} @@ -88,7 +101,7 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { t.Helper() var imports []*graph.Edge for _, edge := range g.GetOutEdges("consumer/src/lib.rs") { - if edge.Kind == graph.EdgeImports { + if edge.Kind == graph.EdgeImports && edge.Line <= 3 { imports = append(imports, edge) } } @@ -130,6 +143,41 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { require.NoError(t, os.Remove(filepath.Join(provider, "src/util.rs"))) writeWorkspaceFile(t, provider, "src/lib.rs", strings.Replace(cargoLifecycleProvider, "pub mod util;\n", "", 1)) _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs"), filepath.Join(provider, "src/lib.rs")}) + case "no_new_file_noop": + observed := &cargoImportReindexObserver{Store: mi.graph} + mi.graph = observed + writeWorkspaceFile(t, provider, "src/spare.rs", "pub fn unused() {}\npub fn added() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/spare.rs")}) + require.NoError(t, err) + require.Zero(t, observed.importReindexes, "an ordinary provider edit must not reindex working imports") + require.Zero(t, observed.importReads, "bound imports must not be admitted for an ordinary edit") + case "provider_gains_empty_module": + writeWorkspaceFile(t, provider, "src/newmod.rs", "") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/newmod.rs")}) + case "provider_gains_new_module": + writeWorkspaceFile(t, provider, "src/newmod.rs", "pub fn later() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/newmod.rs")}) + case "delete_restore_util", "consumer_save_then_restore": + if operation == "consumer_save_then_restore" { + writeWorkspaceFile(t, consumer, "src/lib.rs", cargoLifecycleConsumer+"pub fn edited() {}\n") + _, err = mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "src/lib.rs")}) + require.NoError(t, err) + assertImport() + } + require.NoError(t, os.Remove(filepath.Join(provider, "src/util.rs"))) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) + require.NoError(t, err) + var parked []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 8 { + parked = append(parked, edge) + } + } + require.Len(t, parked, 1) + require.Equal(t, "unresolved::helper", parked[0].To) + require.False(t, parked[0].CrossRepo) + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) case "forced_evict_util": require.NoError(t, os.Remove(filepath.Join(provider, "src/util.rs"))) mi.GetIndexer("provider").EvictFile(filepath.Join(provider, "src/util.rs")) @@ -187,6 +235,18 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { _, err = mi.TrackRepoCtx(context.Background(), entries[1]) } require.NoError(t, err) + if operation == "provider_delete_util_batched" || operation == "forced_evict_util" { + var parked []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 8 { + parked = append(parked, edge) + } + } + require.Len(t, parked, 1) + require.Equal(t, "unresolved::helper", parked[0].To) + require.Equal(t, parked[0].To, parked[0].Meta["cargo_call_stub"]) + require.False(t, parked[0].CrossRepo) + } var cold graph.Store = graph.New() if backend == "sqlite" { db, openErr := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) @@ -198,7 +258,7 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) _, err = coldIndexer.IndexAll() require.NoError(t, err) - require.Equal(t, cargoLifecycleBindings(cold), cargoLifecycleBindings(g), "after %s", operation) + require.Equal(t, cargoLifecycleBindings(cold, operation), cargoLifecycleBindings(g, operation), "after %s", operation) require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) require.Empty(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveAll").All()) }) @@ -210,15 +270,47 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { const cargoLifecycleProvider = "pub mod util;\npub struct Foo;\nimpl Foo {\n pub fn new() -> Foo { Foo }\n pub fn go(&self) {}\n}\npub fn action() {}\n" const cargoLifecycleConsumer = "use other_crate::Foo;\nuse other_crate::action;\nuse other_crate::util::helper;\npub fn run() {\n let f = Foo::new();\n f.go();\n action();\n helper();\n}\n" -func cargoLifecycleBindings(g graph.Store) []string { +func cargoLifecycleBindings(g graph.Store, operation string) []string { var bindings []string for _, node := range g.GetFileNodes("consumer/src/lib.rs") { for _, edge := range g.GetOutEdges(node.ID) { if edge.Kind == graph.EdgeImports || edge.Kind == graph.EdgeCalls { - bindings = append(bindings, fmt.Sprintf("%s -%s-> %s cross=%v line=%d", edge.From, edge.Kind, edge.To, edge.CrossRepo, edge.Line)) + var stub any + if edge.Kind == graph.EdgeCalls { + stub = edge.Meta["cargo_call_stub"] + if edge.Line == 8 && (operation == "provider_delete_util_batched" || operation == "forced_evict_util") { + stub = nil // Cold has no prior bind; the exact parked marker is asserted separately. + } + } + bindings = append(bindings, fmt.Sprintf("%s -%s-> %s cross=%v line=%d stub=%v", edge.From, edge.Kind, edge.To, edge.CrossRepo, edge.Line, stub)) } } } slices.Sort(bindings) return bindings } + +// Observe only the shared resolver; the per-repo indexer retains its native store. +type cargoImportReindexObserver struct { + graph.Store + importReindexes int + importReads int +} + +func (s *cargoImportReindexObserver) ReindexEdges(batch []graph.EdgeReindex) { + for _, change := range batch { + if change.Edge != nil && change.Edge.Kind == graph.EdgeImports && change.Edge.FilePath == "consumer/src/lib.rs" { + s.importReindexes++ + } + } + s.Store.ReindexEdges(batch) +} + +func (s *cargoImportReindexObserver) GetEdgeCandidates(endpoints []graph.EdgeEndpoint, sites []graph.EdgeSite) graph.EdgeCandidateSet { + for _, site := range sites { + if site.Kind == graph.EdgeImports && site.From == "consumer/src/lib.rs" { + s.importReads++ + } + } + return s.Store.GetEdgeCandidates(endpoints, sites) +} diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go index 72a0a3ee2..7ebdc3339 100644 --- a/internal/indexer/cargo_import_restub.go +++ b/internal/indexer/cargo_import_restub.go @@ -83,8 +83,8 @@ func restubTrackedCargoImportsFromView(g graph.Store, evicted map[string]struct{ return sortedStringKeys(consumers) } -// Retry only parked imports claimed by this provider, or owned by this repo. -func (mi *MultiIndexer) cargoConsumerFilesForRepo(repo string) []string { +// Retry parked imports; new Rust files can also retarget marked provider bindings. +func (mi *MultiIndexer) cargoConsumerFilesForRepo(repo string, retargetBound bool) []string { lookup := mi.cargoCrateLookup() if lookup == nil { return nil @@ -93,6 +93,9 @@ func (mi *MultiIndexer) cargoConsumerFilesForRepo(repo string) []string { for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { spec, external := strings.CutPrefix(edge.To, "external::") if !external || !strings.HasSuffix(edge.FilePath, ".rs") { + if retargetBound && edge.CrossRepo && strings.HasSuffix(edge.FilePath, ".rs") && graph.RepoPrefixOfID(edge.To) == repo { + files[edge.FilePath] = struct{}{} + } continue } name, _, _ := strings.Cut(spec, "/") diff --git a/internal/indexer/cargo_import_restub_test.go b/internal/indexer/cargo_import_restub_test.go index 51c8f7719..5b2c99cd4 100644 --- a/internal/indexer/cargo_import_restub_test.go +++ b/internal/indexer/cargo_import_restub_test.go @@ -24,3 +24,28 @@ func TestCargoRestubLeavesUnmarkedImportsUnchanged(t *testing.T) { }) } } + +func TestCargoRestubKeepsSameRepoCallsOnNativePath(t *testing.T) { + for _, mode := range []string{"point", "batch"} { + t.Run(mode, func(t *testing.T) { + g := graph.New() + file, definition, caller := "repo/a.rs", "repo/a.rs::foo", "repo/b.rs::run" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "repo"}) + g.AddNode(&graph.Node{ID: definition, FilePath: file, Name: "foo", Kind: graph.KindFunction, RepoPrefix: "repo"}) + g.AddNode(&graph.Node{ID: caller, FilePath: "repo/b.rs", Name: "run", Kind: graph.KindFunction, RepoPrefix: "repo"}) + edge := &graph.Edge{From: caller, FilePath: "repo/b.rs", Kind: graph.EdgeCalls, To: definition, + Meta: map[string]any{resolver.CargoCallStubMeta: "unresolved::*.foo"}} + g.AddEdge(edge) + if mode == "point" { + idx := &Indexer{graph: g} + idx.restubIncomingRefs(file) + } else { + stages := []*incrementalBatchStage{{graphPath: file, priorNodes: g.GetFileNodes(file)}} + view := loadIncrementalPriorView(g, stages) + restubIncomingRefsFromView(g, stages, view) + } + require.Equal(t, "unresolved::foo", edge.To) + require.False(t, edge.CrossRepo) + }) + } +} diff --git a/internal/indexer/edit_delta_prior_bindings.go b/internal/indexer/edit_delta_prior_bindings.go index 7cb4e605d..f57b6f08a 100644 --- a/internal/indexer/edit_delta_prior_bindings.go +++ b/internal/indexer/edit_delta_prior_bindings.go @@ -1,6 +1,7 @@ package indexer import ( + "slices" "strings" "github.com/zzet/gortex/internal/graph" @@ -259,13 +260,18 @@ func pruneBuiltinReuse(reuse map[reuseKey]*reuseVal) { // edge (semantic.ConfirmEdge). var reuseSemanticMetaKeys = []string{"confirmed_from_origin", "semantic_source"} +// reuseBindingMetaKeys are importer-owned binding stubs a reused bind keeps +// even without confirmation reuse; applyReuseSemanticMeta rewrites each to the +// fresh call spelling. +var reuseBindingMetaKeys = []string{"cargo_call_stub"} + // reuseCarriedMetaKeys are importer-owned binding markers a reused bind keeps // verbatim, even when its confirmation provenance is dropped. go_import lets a // parked cross-module Go call relink when its provider returns. var reuseCarriedMetaKeys = []string{"go_import"} // reuseSemanticMeta is the confirmation provenance and binding markers of a -// captured bind, nil when it carries neither. A confirmed bind's origin is +// captured bind, nil when it carries none. A confirmed bind's origin is // LSP-grade, and a later confirmation of an edge that already carries an // LSP-grade origin leaves it as it is, so a reuse that restored the origin // without this provenance would leave the edge different from the one a whole @@ -275,7 +281,7 @@ func reuseSemanticMeta(edge *graph.Edge) map[string]any { return nil } var out map[string]any - for _, keys := range [][]string{reuseSemanticMetaKeys, reuseCarriedMetaKeys} { + for _, keys := range [][]string{reuseSemanticMetaKeys, reuseBindingMetaKeys, reuseCarriedMetaKeys} { for _, key := range keys { if value, ok := edge.Meta[key]; ok { if out == nil { @@ -288,9 +294,9 @@ func reuseSemanticMeta(edge *graph.Edge) map[string]any { return out } -// applyReuseSemanticMeta restores a reused bind's confirmation provenance and -// carried binding markers. -func applyReuseSemanticMeta(edge *graph.Edge, meta map[string]any) { +// applyReuseSemanticMeta restores provenance, carried markers and the fresh +// call spelling of binding stubs. +func applyReuseSemanticMeta(edge *graph.Edge, meta map[string]any, originalTarget string) { if len(meta) == 0 { return } @@ -298,6 +304,9 @@ func applyReuseSemanticMeta(edge *graph.Edge, meta map[string]any) { edge.Meta = make(map[string]any, len(meta)) } for key, value := range meta { + if slices.Contains(reuseBindingMetaKeys, key) { + value = originalTarget // A name-level reuse can cover different call spellings. + } edge.Meta[key] = value } } diff --git a/internal/indexer/file_meta.go b/internal/indexer/file_meta.go index ff35a42e3..b14ca5652 100644 --- a/internal/indexer/file_meta.go +++ b/internal/indexer/file_meta.go @@ -197,6 +197,7 @@ const reparsePendingEnrichmentBatchSize = 256 type reparsePendingEnrichmentBatch struct { // npmConsumerFiles are unchanged importers parked by this mutation. npmConsumerFiles []string + cargoAddedFile bool cargoConsumerFiles []string byFile map[string]bool deferResolverCatchup bool diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index e55b69289..77f02ca23 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -996,7 +996,7 @@ func applyResolvedOutEdgesFromView( edge.Origin = value.origin edge.Tier = value.tier applyReuseResolutionTag(edge, value.resolution) - applyReuseSemanticMeta(edge, value.semanticMeta) + applyReuseSemanticMeta(edge, value.semanticMeta, oldTo) retargeted = append(retargeted, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) reused++ } @@ -1114,6 +1114,11 @@ func (idx *Indexer) commitStructuralIncrementalBatch( if markerBatch != nil { markerBatch.npmConsumerFiles = appendUniqueSorted(markerBatch.npmConsumerFiles, npmConsumers...) markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, cargoConsumers...) + for _, stage := range stages { + if strings.HasSuffix(stage.graphPath, ".rs") && len(stage.priorNodes) == 0 { + markerBatch.cargoAddedFile = true + } + } } idx.applyLap("restub") // The capability state of the files, read before the eviction below @@ -1547,7 +1552,7 @@ func restubIncomingRefsFromView( if edge.To != stub { edge.CrossRepo, edge.ConfidenceLabel = false, "" } - if original, _ := edge.Meta[resolver.CargoCallStubMeta].(string); edge.Kind == graph.EdgeCalls && original != "" { + if original, _ := edge.Meta[resolver.CargoCallStubMeta].(string); edge.Kind == graph.EdgeCalls && edge.CrossRepo && original != "" { edge.To, edge.CrossRepo, edge.ConfidenceLabel = original, false, "" } reindexes = append(reindexes, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index ce8274718..055ba0ac7 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -537,9 +537,14 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( batch.dropDeferredPriorBindings() } crossRepoFiles := appendUniqueSorted(nil, files...) + cargoRetargetRepo := "" if batch != nil { crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.npmConsumerFiles...) crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.cargoConsumerFiles...) + if batch.cargoAddedFile { + crossRepoFiles = appendUniqueSorted(crossRepoFiles, mi.cargoConsumerFilesForRepo(repoPrefix, true)...) + cargoRetargetRepo = repoPrefix + } } if result != nil { crossRepoFiles = appendUniqueSorted(crossRepoFiles, result.DerivedInvalidation.Files...) @@ -589,12 +594,12 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } crossRepoFiles = appendUniqueSorted(crossRepoFiles, semanticFiles...) } - if needed || len(semanticFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || len(batch.cargoConsumerFiles) > 0) { + if needed || len(semanticFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || batch.cargoAddedFile || len(batch.cargoConsumerFiles) > 0) { if idx != nil { idx.observeIncrementalCatchup("cross_repo", crossRepoFiles) } if len(crossRepoFiles) > 0 { - mi.runCrossRepoResolveFiles(crossRepoFiles) + mi.runCrossRepoResolveFiles(crossRepoFiles, cargoRetargetRepo) } else { mi.runCrossRepoResolve(false) } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 33b710a17..8114439cf 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -6009,7 +6009,7 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string) { if e.Kind == graph.EdgeImports { e.CrossRepo, e.ConfidenceLabel = false, "" } - if original, _ := e.Meta[resolver.CargoCallStubMeta].(string); e.Kind == graph.EdgeCalls && original != "" { + if original, _ := e.Meta[resolver.CargoCallStubMeta].(string); e.Kind == graph.EdgeCalls && e.CrossRepo && original != "" { e.To, e.CrossRepo, e.ConfidenceLabel = original, false, "" } batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 04955a7f6..12ad09d6a 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -2789,7 +2789,7 @@ func (mi *MultiIndexer) indexRepoRaw(repoPrefix string) (*IndexResult, error) { if err := mi.postPublishGoModules(context.Background(), idx, false); err != nil { mi.logger.Warn("post-publish Go module follow-up", zap.Error(err)) } - result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(repoPrefix)...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(repoPrefix, false)...) mi.runCrossRepoResolveFiles(result.CrossRepoResolveFiles) mi.ReconcileContractEdges() @@ -3437,7 +3437,7 @@ func (mi *MultiIndexer) trackPreparedRepoSourceCtx( mi.markRepoRegistered(prefix, absPath) mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix)) if !batchMode.deferResolve { - mi.runCrossRepoResolveFiles(mi.cargoConsumerFilesForRepo(prefix)) + mi.runCrossRepoResolveFiles(mi.cargoConsumerFilesForRepo(prefix, false)) } // Add to global config. @@ -3746,7 +3746,7 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE }) } } else if result.FullRetrack { - result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(prefix)...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(prefix, false)...) if !batchMode.deferResolve { mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix)) mi.runCrossRepoResolveFiles(result.CrossRepoResolveFiles) diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 30502fd39..6fbc3ca83 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -545,15 +545,15 @@ func (mi *MultiIndexer) runCrossRepoResolveContext(ctx context.Context, reconcil // for one coordinated mutation frontier. Its caller owns the repository lane // and global topology-writer gate, so the graph cannot expose a half-resolved // batch between same-repo, semantic, and cross-repository phases. -func (mi *MultiIndexer) runCrossRepoResolveFiles(filePaths []string) { - mi.runCrossRepoResolveMutationFiles(filePaths, filePaths) +func (mi *MultiIndexer) runCrossRepoResolveFiles(filePaths []string, cargoRetargetRepo ...string) { + mi.runCrossRepoResolveMutationFrontiers(filePaths, filePaths, filePaths, cargoRetargetRepo...) } func (mi *MultiIndexer) runCrossRepoResolveMutationFiles(resolutionFiles, materializationFiles []string) { mi.runCrossRepoResolveMutationFrontiers(resolutionFiles, materializationFiles, materializationFiles) } -func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles []string) { +func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles []string, cargoRetargetRepo ...string) { if len(resolutionFiles) == 0 && len(edgeSourceFiles) == 0 && len(definitionFiles) == 0 { return } @@ -561,6 +561,9 @@ func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, ed if cr == nil { return } + if len(cargoRetargetRepo) > 0 { + cr.SetCargoImportRetargetRepo(cargoRetargetRepo[0]) + } cr.ResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles) mi.finishGoModuleCalls(cr) mi.reconcileRetargetedTestCalls(cr.TakeRetargetedTestCallFiles()) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index 9d1f2c328..37a418ff9 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -28,6 +28,9 @@ type CargoCrateLookup func(callerFile, crateName string) []CargoCrate // SetCargoCrateLookup installs manifest evidence for Rust import/file edges. func (cr *CrossRepoResolver) SetCargoCrateLookup(lookup CargoCrateLookup) { cr.cargoCrates = lookup } +// SetCargoImportRetargetRepo refreshes marked bindings only after a provider file is added. +func (cr *CrossRepoResolver) SetCargoImportRetargetRepo(repo string) { cr.cargoRetargetRepo = repo } + func (cr *CrossRepoResolver) cargoImportTarget(e *graph.Edge, spec string) *graph.Node { if cr.cargoCrates == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { return nil @@ -99,7 +102,7 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter if err := ctx.Err(); err != nil { return err } - spec := cargoImportSpec(e) + spec := cr.cargoImportSpec(e) if spec == "" { continue } @@ -126,7 +129,7 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter if e.To != projected.To { continue } - if node := cr.cargoImportTarget(e, cargoImportSpec(e)); node != nil { + if node := cr.cargoImportTarget(e, cr.cargoImportSpec(e)); node != nil && node.ID != e.To { oldTo := e.To cr.bindCargoImport(e, node, stats) batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) @@ -141,11 +144,16 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter return ctx.Err() } -func cargoImportSpec(e *graph.Edge) string { +func (cr *CrossRepoResolver) cargoImportSpec(e *graph.Edge) string { if e == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { return "" } - spec, external := strings.CutPrefix(e.To, "external::") + target := e.To + if cr.cargoRetargetRepo != "" && e.CrossRepo && graph.RepoPrefixOfID(e.To) == cr.cargoRetargetRepo { + target, _ = e.Meta[CargoImportStubMeta].(string) + target = strings.Replace(target, "unresolved::import::", "external::", 1) + } + spec, external := strings.CutPrefix(target, "external::") if !external { return "" } @@ -161,7 +169,9 @@ func (cr *CrossRepoResolver) bindCargoImport(e *graph.Edge, node *graph.Node, st if e.Meta == nil { e.Meta = make(map[string]any) } - e.Meta[CargoImportStubMeta] = e.To + if original, _ := e.Meta[CargoImportStubMeta].(string); original == "" { + e.Meta[CargoImportStubMeta] = e.To + } e.To, e.CrossRepo = node.ID, true clearEdgeTerminal(e) stats.Resolved++ diff --git a/internal/resolver/cargo_import_retarget_test.go b/internal/resolver/cargo_import_retarget_test.go new file mode 100644 index 000000000..519ba45fd --- /dev/null +++ b/internal/resolver/cargo_import_retarget_test.go @@ -0,0 +1,83 @@ +package resolver + +import ( + "context" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +type cargoRetargetWriteStore struct { + graph.Store + writes int +} + +func (s *cargoRetargetWriteStore) ReindexEdges(batch []graph.EdgeReindex) { + s.writes += len(batch) + s.Store.ReindexEdges(batch) +} + +func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scenario := range []string{"unresolved_marker", "external_marker", "unmarked", "other_provider", "already_correct", "disabled"} { + t.Run(backend+"/"+scenario, func(t *testing.T) { + var native graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + native = db + } + g := &cargoRetargetWriteStore{Store: native} + file, root, module := "consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs" + for _, path := range []string{file, root, module, "other/src/lib.rs"} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), WorkspaceID: "shared", Language: "rust"}) + } + original := "unresolved::import::other_crate/util" + if scenario == "external_marker" { + original = "external::other_crate/util" + } + edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeImports, To: root, CrossRepo: true, Line: 1, + Meta: map[string]any{CargoImportStubMeta: original}} + if scenario == "unmarked" { + edge.Meta = nil + } + if scenario == "other_provider" { + edge.To = "other/src/lib.rs" + } + if scenario == "already_correct" { + edge.To = module + } + priorTarget := edge.To + g.AddEdge(edge) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return []CargoCrate{{Root: root, Name: "other_crate"}} }) + if scenario != "disabled" { + cr.SetCargoImportRetargetRepo("provider") + } + stats := &CrossRepoStats{ByRepo: map[string]int{}} + require.NoError(t, cr.resolveCargoImports(context.Background(), slices.Values([]*graph.Edge{edge}), stats)) + imports := outEdgesOfKind(g, file, graph.EdgeImports) + require.Len(t, imports, 1) + changed := scenario == "unresolved_marker" || scenario == "external_marker" + if changed { + require.Equal(t, module, imports[0].To) + require.Equal(t, 1, g.writes) + } else { + require.Equal(t, priorTarget, imports[0].To) + require.Zero(t, g.writes) + } + if scenario != "unmarked" { + require.Equal(t, original, imports[0].Meta[CargoImportStubMeta]) + } + priorWrites := g.writes + require.NoError(t, cr.resolveCargoImports(context.Background(), slices.Values(imports), stats)) + require.Equal(t, priorWrites, g.writes, "an unchanged native match must not reindex") + }) + } + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 645188b3b..b12355e6d 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -155,7 +155,8 @@ type CrossRepoResolver struct { npmDep NpmDependencyLookup npmPackages NpmPackageLookup - cargoCrates CargoCrateLookup + cargoCrates CargoCrateLookup + cargoRetargetRepo string // pathAlias expands a JS/TS tsconfig/jsconfig path-alias / baseUrl // import specifier to the repo-prefixed file stem it targets. Same diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 9cf852f5c..b88390f93 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -104,7 +104,7 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed if cr.cargoCrates != nil { for _, edges := range frontier.outByNode { for _, edge := range edges { - if cargoImportSpec(edge) != "" { + if cr.cargoImportSpec(edge) != "" { frontier.pending = append(frontier.pending, edge) } } @@ -113,7 +113,7 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed pending := make([]*graph.Edge, 0, len(frontier.pending)) seen := make(map[graph.EdgeIdentity]struct{}, len(frontier.pending)) for _, edge := range frontier.pending { - if edge == nil || !graph.IsUnresolvedTarget(edge.To) && trackedNpmImportSpec(edge) == "" && cargoImportSpec(edge) == "" { + if edge == nil || !graph.IsUnresolvedTarget(edge.To) && trackedNpmImportSpec(edge) == "" && cr.cargoImportSpec(edge) == "" { continue } identity := graph.EdgeIdentityFor(edge) From 177b9f36725f5c7d68318b98853ff1d1cef8451c Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 05:47:58 -0700 Subject: [PATCH 018/111] fix(indexer): carry Cargo provider recovery through warmup Preserve Rust file additions on IndexResult and registered warm receipts. Consume them in the existing scoped or full resolve after registration, including provider-first warmup and multiple changed providers. Restrict scoped recovery to marked imports into those providers. Keep the native matcher and write only changed targets. Record additions installed by the incremental fallback path too. --- .../indexer/cargo_import_fallback_test.go | 77 +++++++++ .../indexer/cargo_import_lifecycle_test.go | 12 ++ internal/indexer/cargo_import_restub.go | 48 +++++- .../cargo_import_warm_recovery_test.go | 161 ++++++++++++++++++ internal/indexer/incremental_batch.go | 3 + internal/indexer/incremental_watcher_batch.go | 2 +- internal/indexer/indexer.go | 3 + internal/indexer/multi.go | 11 +- internal/indexer/workspace_resolve.go | 7 +- internal/resolver/cargo_crate.go | 25 ++- .../resolver/cargo_import_retarget_test.go | 24 ++- internal/resolver/cross_repo.go | 4 +- 12 files changed, 354 insertions(+), 23 deletions(-) create mode 100644 internal/indexer/cargo_import_fallback_test.go create mode 100644 internal/indexer/cargo_import_warm_recovery_test.go diff --git a/internal/indexer/cargo_import_fallback_test.go b/internal/indexer/cargo_import_fallback_test.go new file mode 100644 index 000000000..6bba9e321 --- /dev/null +++ b/internal/indexer/cargo_import_fallback_test.go @@ -0,0 +1,77 @@ +package indexer + +import ( + "context" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +// cargoFallbackRefreshTransform changes extra.rs's bytes on exactly its second +// application: the probe read sees call 1, takePreparedRefresh sees call 2 +// (mismatch, so the stage falls back), and the fallback indexFile sees call 3. +type cargoFallbackRefreshTransform struct{ calls *atomic.Int32 } + +func (cargoFallbackRefreshTransform) name() string { return "cargo-fallback-refresh" } +func (cargoFallbackRefreshTransform) matches(p string) bool { return strings.HasSuffix(p, "extra.rs") } +func (cargoFallbackRefreshTransform) asLanguage() string { return "" } +func (f cargoFallbackRefreshTransform) apply(p string, src []byte) ([]byte, error) { + if f.calls.Add(1) == 2 { + return append(append([]byte(nil), src...), []byte("\n// concurrent write\n")...), nil + } + return src, nil +} + +func TestTrackedCargoNewModuleThroughFallback(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") + writeWorkspaceFile(t, provider, "src/lib.rs", cargoLifecycleProvider) + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") + writeWorkspaceFile(t, consumer, "src/lib.rs", cargoLifecycleConsumer+"\nuse other_crate::extra::later;\npub fn after() { later(); }\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewRustExtractor()) + g := cargoWarmStore(t, backend) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + _, err = mi.IndexAll() + require.NoError(t, err) + + idx := mi.GetIndexer("provider") + require.NotNil(t, idx) + var calls atomic.Int32 + idx.transforms.transforms = append(idx.transforms.transforms, cargoFallbackRefreshTransform{calls: &calls}) + writeWorkspaceFile(t, provider, "src/extra.rs", "pub fn later() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/extra.rs")}) + require.NoError(t, err) + t.Logf("transform calls=%d", calls.Load()) + require.GreaterOrEqual(t, calls.Load(), int32(3), "fallback path must run") + require.NotNil(t, g.GetNode("provider/src/extra.rs")) + + cold := cargoWarmStore(t, backend) + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, cargoLifecycleBindings(cold, "fallback"), cargoLifecycleBindings(g, "fallback")) + + }) + } +} diff --git a/internal/indexer/cargo_import_lifecycle_test.go b/internal/indexer/cargo_import_lifecycle_test.go index 8b75e49b1..3abb8a031 100644 --- a/internal/indexer/cargo_import_lifecycle_test.go +++ b/internal/indexer/cargo_import_lifecycle_test.go @@ -314,3 +314,15 @@ func (s *cargoImportReindexObserver) GetEdgeCandidates(endpoints []graph.EdgeEnd } return s.Store.GetEdgeCandidates(endpoints, sites) } + +func TestCargoAddedFileWithoutConsumersDoesNotRunGlobalResolve(t *testing.T) { + core, logs := observer.New(zap.InfoLevel) + g := graph.New() + g.AddNode(&graph.Node{ID: "consumer/src/lib.rs::run", Kind: graph.KindFunction, Name: "run", FilePath: "consumer/src/lib.rs", RepoPrefix: "consumer", Language: "rust"}) + g.AddNode(&graph.Node{ID: "provider/src/new.rs::f", Kind: graph.KindFunction, Name: "f", FilePath: "provider/src/new.rs", RepoPrefix: "provider", Language: "rust"}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs::run", To: "unresolved::f", Kind: graph.EdgeCalls, FilePath: "consumer/src/lib.rs"}) + mi := &MultiIndexer{graph: g, logger: zap.New(core)} + mi.resolveIncrementalRepoMutation("provider", &IndexResult{}, &graph.MutationReceipt{Complete: true}, + &reparsePendingEnrichmentBatch{cargoAddedFile: true}) + require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) +} diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go index 7ebdc3339..3952c7adf 100644 --- a/internal/indexer/cargo_import_restub.go +++ b/internal/indexer/cargo_import_restub.go @@ -1,6 +1,8 @@ package indexer import ( + "context" + "sort" "strings" "github.com/zzet/gortex/internal/graph" @@ -90,10 +92,15 @@ func (mi *MultiIndexer) cargoConsumerFilesForRepo(repo string, retargetBound boo return nil } files := make(map[string]struct{}) - for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { + edges := graph.EdgesLightSeq(mi.graph, graph.EdgeImports) + if retargetBound { + edges = mi.graph.EdgesByKind(graph.EdgeImports) + } + for edge := range edges { spec, external := strings.CutPrefix(edge.To, "external::") if !external || !strings.HasSuffix(edge.FilePath, ".rs") { - if retargetBound && edge.CrossRepo && strings.HasSuffix(edge.FilePath, ".rs") && graph.RepoPrefixOfID(edge.To) == repo { + stub, _ := edge.Meta[resolver.CargoImportStubMeta].(string) + if retargetBound && stub != "" && edge.CrossRepo && strings.HasSuffix(edge.FilePath, ".rs") && graph.RepoPrefixOfID(edge.To) == repo { files[edge.FilePath] = struct{}{} } continue @@ -108,3 +115,40 @@ func (mi *MultiIndexer) cargoConsumerFilesForRepo(repo string, retargetBound boo } return sortedStringKeys(files) } + +// Consume warm receipts only after all repositories are registered. +func (mi *MultiIndexer) runWarmCargoResolve(ctx context.Context, files []string, full bool) error { + mi.mu.RLock() + pending := map[string]*IndexResult{} + var repos []string + for repo, meta := range mi.repos { + if meta != nil && meta.cargoRetarget != nil { + pending[repo] = meta.cargoRetarget + repos = append(repos, repo) + } + } + mi.mu.RUnlock() + sort.Strings(repos) + var err error + if full { + err = mi.runCrossRepoResolveContext(ctx, false, repos...) + } else { + for _, repo := range repos { + files = appendUniqueSorted(files, mi.cargoConsumerFilesForRepo(repo, true)...) + } + if err = context.Cause(ctx); err == nil { + mi.runCrossRepoResolveFiles(files, repos...) + err = context.Cause(ctx) + } + } + if err == nil { + mi.mu.Lock() + for repo, receipt := range pending { + if meta := mi.repos[repo]; meta != nil && meta.cargoRetarget == receipt { + meta.cargoRetarget = nil + } + } + mi.mu.Unlock() + } + return err +} diff --git a/internal/indexer/cargo_import_warm_recovery_test.go b/internal/indexer/cargo_import_warm_recovery_test.go new file mode 100644 index 000000000..a7376e411 --- /dev/null +++ b/internal/indexer/cargo_import_warm_recovery_test.go @@ -0,0 +1,161 @@ +package indexer + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedCargoWarmFileRecovery(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, deferred := range []bool{false, true} { + for _, full := range []bool{false, true} { + for _, restore := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/provider_first=%v/deferred=%v/restore=%v/full=%v", backend, providerFirst, deferred, restore, full), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") + writeWorkspaceFile(t, provider, "src/lib.rs", cargoLifecycleProvider) + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") + consumerSource := cargoLifecycleConsumer + if !restore { + consumerSource += "\nuse other_crate::extra::*;\n" + } + writeWorkspaceFile(t, consumer, "src/lib.rs", consumerSource) + for i := 0; i < 6; i++ { + writeWorkspaceFile(t, provider, fmt.Sprintf("src/pad%d.rs", i), "pub fn pad() {}\n") + writeWorkspaceFile(t, consumer, fmt.Sprintf("src/pad%d.rs", i), "pub fn pad() {}\n") + } + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + if providerFirst { + entries[0], entries[1] = entries[1], entries[0] + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewRustExtractor()) + newMI := func(g graph.Store) *MultiIndexer { + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + return mi + } + g := cargoWarmStore(t, backend) + mi := newMI(g) + _, err = mi.IndexAll() + require.NoError(t, err) + if restore { + file := filepath.Join(provider, "src/util.rs") + require.NoError(t, os.Remove(file)) + _, err = mi.IncrementalReindexRepo("provider", []string{file}) + require.NoError(t, err) + } + prior := map[string]map[string]int64{"consumer": mi.FileMtimes("consumer"), "provider": mi.FileMtimes("provider")} + require.NoError(t, mi.Close(context.Background())) + if restore { + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + } else { + writeWorkspaceFile(t, provider, "src/extra.rs", "") + } + restarted := newMI(g) + if deferred { + restarted.BeginParallelBatch() + } else { + restarted.BeginBatch() + } + var files []string + for _, entry := range entries { + result, err := restarted.ReconcileRepoCtx(context.Background(), entry, prior[entry.Name]) + require.NoError(t, err) + require.False(t, result.FullRetrack, "must exercise scoped warm reconcile") + if entry.Name == "provider" { + require.True(t, result.CargoRetarget) + } + files = appendUniqueSorted(files, result.DerivedInvalidation.Files...) + files = appendUniqueSorted(files, result.CrossRepoResolveFiles...) + } + if full { + require.NoError(t, restarted.RunPreEnrichResolve(context.Background(), nil, nil)) + } else { + require.NoError(t, restarted.RunPreEnrichResolveFiles(context.Background(), files, nil)) + } + restarted.EndBatch() + cold := cargoWarmStore(t, backend) + _, err = newMI(cold).IndexAll() + require.NoError(t, err) + require.Equal(t, cargoLifecycleBindings(cold, "warm"), cargoLifecycleBindings(g, "warm")) + }) + } + } + } + } + } +} + +func cargoWarmStore(t *testing.T, backend string) graph.Store { + t.Helper() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db + } + return graph.New() +} + +func TestCargoWarmRetargetReceiptCancellation(t *testing.T) { + a, b := &IndexResult{CargoRetarget: true}, &IndexResult{CargoRetarget: true} + mi := &MultiIndexer{graph: graph.New(), repos: map[string]*RepoMetadata{ + "a": {cargoRetarget: a}, "b": {cargoRetarget: b}, + }} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, mi.runWarmCargoResolve(ctx, nil, false), context.Canceled) + require.Same(t, a, mi.repos["a"].cargoRetarget) + require.Same(t, b, mi.repos["b"].cargoRetarget) + require.NoError(t, mi.runWarmCargoResolve(context.Background(), nil, false)) + require.Nil(t, mi.repos["a"].cargoRetarget) + require.Nil(t, mi.repos["b"].cargoRetarget) + require.True(t, a.CargoRetarget, "consumption must not mutate a returned result") +} + +func TestCargoAddedFileFrontierOnlyMarkedProviderImports(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") + writeWorkspaceFile(t, provider, "src/lib.rs", "") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") + g := graph.New() + for _, row := range []struct { + file, target string + marker any + }{ + {"consumer/marked.rs", "provider/src/lib.rs", "external::other_crate/util"}, + {"consumer/unmarked.rs", "provider/src/lib.rs", nil}, + {"consumer/invalid.rs", "provider/src/lib.rs", 1}, + {"consumer/other.rs", "other/src/lib.rs", "external::other_crate/util"}, + {"consumer/nonrust.py", "provider/src/lib.rs", "external::other_crate/util"}, + } { + g.AddEdge(&graph.Edge{From: row.file, FilePath: row.file, To: row.target, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{"cargo_import_stub": row.marker}}) + } + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{ + "provider": {RootPath: provider}, "consumer": {RootPath: consumer}, + }} + require.Equal(t, []string{"consumer/marked.rs"}, mi.cargoConsumerFilesForRepo("provider", true)) + require.Empty(t, mi.cargoConsumerFilesForRepo("provider", false)) +} diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 77f02ca23..6290a4b99 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -519,6 +519,9 @@ func (idx *Indexer) reindexIncrementalFallback( if err := idx.indexFile(fallback.filePath, resolve, markerBatch); err != nil { return err } + if markerBatch != nil && len(fallback.priorNodes) == 0 && strings.HasSuffix(fallback.graphPath, ".rs") { + markerBatch.cargoAddedFile = true + } freshNodes := idx.graph.GetFileNodesByPaths([]string{fallback.graphPath})[fallback.graphPath] freshDerived := storedDerivedFingerprints(freshNodes) if fallback.probeOK && fallback.probe.derived.complete() { diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index 055ba0ac7..c588f7ff0 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -594,7 +594,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } crossRepoFiles = appendUniqueSorted(crossRepoFiles, semanticFiles...) } - if needed || len(semanticFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || batch.cargoAddedFile || len(batch.cargoConsumerFiles) > 0) { + if needed || len(semanticFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || len(crossRepoFiles) > 0 && (batch.cargoAddedFile || len(batch.cargoConsumerFiles) > 0)) { if idx != nil { idx.observeIncrementalCatchup("cross_repo", crossRepoFiles) } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 8114439cf..6908f7cfd 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -134,6 +134,8 @@ type IndexResult struct { // CrossRepoResolveFiles contains parked consumers outside this repository's // derived frontier. Batch callers must include them in cross-repo resolution. CrossRepoResolveFiles []string `json:"cross_repo_resolve_files,omitempty"` + // CargoRetarget records a Rust file addition for the post-registration warm tail. + CargoRetarget bool `json:"cargo_retarget,omitempty"` // mutationErr carries a point-operation compatibility error through the // resolver/semantic/derived tail. It is intentionally unexported and never @@ -7547,6 +7549,7 @@ func (idx *Indexer) incrementalReindexPathsMode( } result.CrossRepoResolveFiles = appendUniqueSorted(nil, markerBatch.npmConsumerFiles...) result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, markerBatch.cargoConsumerFiles...) + result.CargoRetarget = markerBatch.cargoAddedFile for _, path := range failedFiles { result.Errors = append(result.Errors, IndexError{FilePath: path, Error: idx.fileIndexFailureError(path).Error()}) } diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 12ad09d6a..59ac81798 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -35,6 +35,7 @@ import ( // RepoMetadata holds per-repo indexing state. type RepoMetadata struct { + cargoRetarget *IndexResult // warm receipt, guarded by MultiIndexer.mu RepoPrefix string RootPath string Identity *RepoIdentity @@ -1204,7 +1205,7 @@ func (mi *MultiIndexer) RunPreEnrichResolve(ctx context.Context, scope map[strin // Cross-repo references resolve here so a multi-repo workspace is fully // resolved, not just within each repo. Whole-graph so inbound references // from unchanged repos into the changed repos bind too. - return mi.runCrossRepoResolveContext(ctx, false) + return mi.runWarmCargoResolve(ctx, nil, true) } // RunPreEnrichResolveFiles preserves an exact warm-reconcile frontier through @@ -1234,10 +1235,9 @@ func (mi *MultiIndexer) RunPreEnrichResolveFiles(ctx context.Context, files []st return err } if resolutionAffected > 0 { - return mi.runCrossRepoResolveContext(ctx, false) + return mi.runWarmCargoResolve(ctx, nil, true) } - mi.runCrossRepoResolveFiles(files) - return context.Cause(ctx) + return mi.runWarmCargoResolve(ctx, files, false) } func (mi *MultiIndexer) runDeferredGoModAll() { @@ -3704,6 +3704,9 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE IsWorktree: isWorktree, } mi.indexers[prefix] = idx + if result.CargoRetarget { + mi.repos[prefix].cargoRetarget = result + } mi.mu.Unlock() installed = true mi.markRepoRegistered(prefix, absPath) diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 6fbc3ca83..6a72fbc41 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -510,7 +510,7 @@ func (mi *MultiIndexer) runCrossRepoResolve(reconcileContracts bool) { } } -func (mi *MultiIndexer) runCrossRepoResolveContext(ctx context.Context, reconcileContracts bool) error { +func (mi *MultiIndexer) runCrossRepoResolveContext(ctx context.Context, reconcileContracts bool, cargoRetargetRepo ...string) error { cr := mi.newCrossRepoResolver() if cr == nil { return nil @@ -528,6 +528,7 @@ func (mi *MultiIndexer) runCrossRepoResolveContext(ctx context.Context, reconcil } mi.synthesizeDeferredGoModuleCalls(idx, edges) } + cr.SetCargoImportRetargetRepo(cargoRetargetRepo...) if _, err := cr.ResolveAllContext(ctx); err != nil { return err } @@ -561,9 +562,7 @@ func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, ed if cr == nil { return } - if len(cargoRetargetRepo) > 0 { - cr.SetCargoImportRetargetRepo(cargoRetargetRepo[0]) - } + cr.SetCargoImportRetargetRepo(cargoRetargetRepo...) cr.ResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles) mi.finishGoModuleCalls(cr) mi.reconcileRetargetedTestCalls(cr.TakeRetargetedTestCallFiles()) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index 37a418ff9..ff07551c1 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -29,7 +29,14 @@ type CargoCrateLookup func(callerFile, crateName string) []CargoCrate func (cr *CrossRepoResolver) SetCargoCrateLookup(lookup CargoCrateLookup) { cr.cargoCrates = lookup } // SetCargoImportRetargetRepo refreshes marked bindings only after a provider file is added. -func (cr *CrossRepoResolver) SetCargoImportRetargetRepo(repo string) { cr.cargoRetargetRepo = repo } +func (cr *CrossRepoResolver) SetCargoImportRetargetRepo(repos ...string) { + cr.cargoRetargetRepos = make(map[string]bool, len(repos)) + for _, repo := range repos { + if repo != "" { + cr.cargoRetargetRepos[repo] = true + } + } +} func (cr *CrossRepoResolver) cargoImportTarget(e *graph.Edge, spec string) *graph.Node { if cr.cargoCrates == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { @@ -102,14 +109,20 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter if err := ctx.Err(); err != nil { return err } + if e == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { + continue + } spec := cr.cargoImportSpec(e) - if spec == "" { + if spec == "" && (!e.CrossRepo || !cr.cargoRetargetRepos[graph.RepoPrefixOfID(e.To)]) { continue } - name, _, _ := strings.Cut(spec, "/") - if len(cr.cargoCrates(e.FilePath, name)) > 0 { - eligible = append(eligible, e) + if spec != "" { + name, _, _ := strings.Cut(spec, "/") + if len(cr.cargoCrates(e.FilePath, name)) == 0 { + continue + } } + eligible = append(eligible, e) } // Close the light cursor before hydration or writes; retain metadata for // only one bounded page, as in the tracked npm bridge. @@ -149,7 +162,7 @@ func (cr *CrossRepoResolver) cargoImportSpec(e *graph.Edge) string { return "" } target := e.To - if cr.cargoRetargetRepo != "" && e.CrossRepo && graph.RepoPrefixOfID(e.To) == cr.cargoRetargetRepo { + if e.CrossRepo && cr.cargoRetargetRepos[graph.RepoPrefixOfID(e.To)] { target, _ = e.Meta[CargoImportStubMeta].(string) target = strings.Replace(target, "unresolved::import::", "external::", 1) } diff --git a/internal/resolver/cargo_import_retarget_test.go b/internal/resolver/cargo_import_retarget_test.go index 519ba45fd..e88307dfd 100644 --- a/internal/resolver/cargo_import_retarget_test.go +++ b/internal/resolver/cargo_import_retarget_test.go @@ -23,7 +23,7 @@ func (s *cargoRetargetWriteStore) ReindexEdges(batch []graph.EdgeReindex) { func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { - for _, scenario := range []string{"unresolved_marker", "external_marker", "unmarked", "other_provider", "already_correct", "disabled"} { + for _, scenario := range []string{"unresolved_marker", "external_marker", "unmarked", "other_provider", "already_correct", "disabled", "light_projection", "multiple_enabled", "nil_input", "wrong_kind"} { t.Run(backend+"/"+scenario, func(t *testing.T) { var native graph.Store = graph.New() if backend == "sqlite" { @@ -56,14 +56,26 @@ func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { g.AddEdge(edge) cr := NewCrossRepo(g) cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return []CargoCrate{{Root: root, Name: "other_crate"}} }) - if scenario != "disabled" { + if scenario == "multiple_enabled" { + cr.SetCargoImportRetargetRepo("other", "provider") + } else if scenario != "disabled" { cr.SetCargoImportRetargetRepo("provider") } stats := &CrossRepoStats{ByRepo: map[string]int{}} - require.NoError(t, cr.resolveCargoImports(context.Background(), slices.Values([]*graph.Edge{edge}), stats)) + edges := slices.Values([]*graph.Edge{edge}) + if scenario == "light_projection" { + edges = graph.EdgesLightSeq(g, graph.EdgeImports) + } else if scenario == "nil_input" { + edges = slices.Values([]*graph.Edge{nil}) + } else if scenario == "wrong_kind" { + projected := *edge + projected.Kind = graph.EdgeCalls + edges = slices.Values([]*graph.Edge{&projected}) + } + require.NoError(t, cr.resolveCargoImports(context.Background(), edges, stats)) imports := outEdgesOfKind(g, file, graph.EdgeImports) require.Len(t, imports, 1) - changed := scenario == "unresolved_marker" || scenario == "external_marker" + changed := scenario == "unresolved_marker" || scenario == "external_marker" || scenario == "light_projection" || scenario == "multiple_enabled" if changed { require.Equal(t, module, imports[0].To) require.Equal(t, 1, g.writes) @@ -75,6 +87,10 @@ func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { require.Equal(t, original, imports[0].Meta[CargoImportStubMeta]) } priorWrites := g.writes + if scenario == "nil_input" || scenario == "wrong_kind" { + require.Zero(t, stats.Resolved) + return + } require.NoError(t, cr.resolveCargoImports(context.Background(), slices.Values(imports), stats)) require.Equal(t, priorWrites, g.writes, "an unchanged native match must not reindex") }) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index b12355e6d..030967f3b 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -155,8 +155,8 @@ type CrossRepoResolver struct { npmDep NpmDependencyLookup npmPackages NpmPackageLookup - cargoCrates CargoCrateLookup - cargoRetargetRepo string + cargoCrates CargoCrateLookup + cargoRetargetRepos map[string]bool // pathAlias expands a JS/TS tsconfig/jsconfig path-alias / baseUrl // import specifier to the repo-prefixed file stem it targets. Same From d6488c78af31c688d4b5e594fcea59f6ec296ed5 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Fri, 23 Oct 2026 00:55:12 -0700 Subject: [PATCH 019/111] feat(resolver): link Python imports to a declared tracked distribution A Python `import pkg.mod` or `from pkg import mod` of a package whose source is another tracked repository stayed external. Calls through such imports already linked, but the import edges did not: directory matching cannot map a dotted module to src/pkg/mod.py or pkg/__init__.py. Map each tracked repository's Python distribution ([project].name or [tool.poetry].name, PEP 503-normalized) to its top-level import packages under the repository root or src/. An import links only when the importing repository declares that distribution in its own pyproject.toml ([project] dependencies and optional dependencies, Poetry dependencies and groups) or requirements*.txt, and exactly one eligible tracked repository provides the top-level package. The module file is chosen with the existing Python module matching (.py and __init__.py). The import stays local or external when the importing repository has its own files under that top-level name (including namespace folders without __init__.py), for standard-library names, and for root tooling files such as setup.py and conftest.py. Per-repository attribution keeps the dotted import path on the edge so the retry can use it. The retry runs in the full cross-repo pass, with light edge scans and paged hydration, and in the mutation-frontier pass, so a link survives an edit to the importing file. It does nothing with fewer than two tracked repositories. Limits: .pyi-only modules do not link, because stubs are not indexed. A link stays after the dependency is removed from the manifest until the importing file is re-indexed. Dotted modules do not match cross-workspace module rules, so such imports stay external. Tests index two tracked repositories on memory and SQLite stores. The positive cases fail before this change. --- internal/indexer/python_tracked_import.go | 165 +++++++++++++++++ .../indexer/python_tracked_import_test.go | 159 ++++++++++++++++ internal/indexer/workspace_resolve.go | 3 + internal/resolver/cross_repo.go | 6 +- internal/resolver/cross_repo_incremental.go | 11 +- internal/resolver/module_attribution.go | 6 + internal/resolver/python_tracked_import.go | 175 ++++++++++++++++++ .../resolver/python_tracked_import_test.go | 148 +++++++++++++++ 8 files changed, 671 insertions(+), 2 deletions(-) create mode 100644 internal/indexer/python_tracked_import.go create mode 100644 internal/indexer/python_tracked_import_test.go create mode 100644 internal/resolver/python_tracked_import.go create mode 100644 internal/resolver/python_tracked_import_test.go diff --git a/internal/indexer/python_tracked_import.go b/internal/indexer/python_tracked_import.go new file mode 100644 index 000000000..c231dece1 --- /dev/null +++ b/internal/indexer/python_tracked_import.go @@ -0,0 +1,165 @@ +package indexer + +import ( + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/pelletier/go-toml/v2" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/modules" + "github.com/zzet/gortex/internal/resolver" +) + +var pythonDistributionSeparators = regexp.MustCompile(`[-_.]+`) + +func normalizePythonDistribution(name string) string { + return pythonDistributionSeparators.ReplaceAllString(strings.ToLower(name), "-") +} + +// pythonPackageLookup is built only after a non-stdlib Python import is found. +// Providers require indexed root/src modules or regular packages; consumers also +// block foreign bindings with their own indexed namespace directories. +func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { + roots := mi.AllMetadata() + if len(roots) < 2 { + return nil + } + type entry struct { + name string + deps map[string]bool + } + entries := map[string]*entry{} + for repo, meta := range roots { + if meta == nil || meta.RootPath == "" { + continue + } + src, _ := readDiskFile(filepath.Join(meta.RootPath, "pyproject.toml")) + var manifest struct { + Project struct{ Name string } `toml:"project"` + Tool struct { + Poetry struct { + Name string + Group map[string]struct{ Dependencies map[string]any } `toml:"group"` + } `toml:"poetry"` + } `toml:"tool"` + } + // A malformed manifest supplies no identity or dependencies. + if len(src) > 0 && toml.Unmarshal(src, &manifest) != nil { + continue + } + name := manifest.Project.Name + if name == "" { + name = manifest.Tool.Poetry.Name + } + deps := map[string]bool{} + specs := modules.ParsePyProject(src) + for _, group := range manifest.Tool.Poetry.Group { + for dep := range group.Dependencies { + if dep != "python" { + deps[normalizePythonDistribution(dep)] = true + } + } + } + requirements, _ := filepath.Glob(filepath.Join(meta.RootPath, "requirements*.txt")) + for _, file := range requirements { + src, _ := readDiskFile(file) + specs = append(specs, modules.ParseRequirementsTxt(src)...) + } + for _, dep := range specs { + deps[normalizePythonDistribution(dep.Path)] = true + } + entries[repo] = &entry{name: normalizePythonDistribution(name), deps: deps} + } + // Only declared distributions can supply providers. Discover top-level names + // from their root/src directory entries, then probe exact indexed node IDs. + packages := map[string]map[string][]graph.FileNodeIdentity{} + for repo, pkg := range entries { + declared := false + for caller, consumer := range entries { + if caller != repo && pkg.name != "" && consumer.deps[pkg.name] { + declared = true + break + } + } + if !declared { + continue + } + for _, dir := range []string{"", "src"} { + children, err := os.ReadDir(filepath.Join(roots[repo].RootPath, dir)) + if err != nil { + continue + } + for _, child := range children { + name := child.Name() + if !child.IsDir() && !strings.HasSuffix(name, ".py") { + continue + } + if dir == "" && pythonRootToolingFile(name) { + continue + } + top := strings.TrimSuffix(name, ".py") + file := filepath.ToSlash(filepath.Join(repo, dir, name)) + if child.IsDir() { + file += "/__init__.py" + } + node := mi.graph.GetNode(file) + if node == nil || node.Kind != graph.KindFile || node.RepoPrefix != repo { + continue + } + if packages[top] == nil { + packages[top] = map[string][]graph.FileNodeIdentity{} + } + packages[top][repo] = append(packages[top][repo], graph.FileNodeIdentity{ + ID: node.ID, FilePath: node.FilePath, RepoPrefix: repo, WorkspaceID: node.WorkspaceID, + }) + } + } + } + if len(packages) == 0 { + return nil + } + localTops := map[string]map[string]bool{} + return &resolver.PythonPackageLookup{ + HasTop: func(top string) bool { return len(packages[top]) > 0 }, + Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + consumer := entries[caller] + if consumer == nil || len(packages[top]) == 0 { + return nil + } + local, seen := localTops[caller] + if !seen { + local = map[string]bool{} + for file := range graph.FileNodeIdentitiesSeq(mi.graph, []string{caller}) { + if !strings.HasSuffix(file.FilePath, ".py") { + continue + } + rel := strings.TrimPrefix(file.FilePath, caller+"/") + rel = strings.TrimPrefix(rel, "src/") + name, _, _ := strings.Cut(rel, "/") + local[strings.TrimSuffix(name, ".py")] = true + } + localTops[caller] = local + } + if local[top] { + return nil + } + providers := map[string][]graph.FileNodeIdentity{} + for repo, files := range packages[top] { + if repo != caller && consumer.deps[entries[repo].name] { + providers[repo] = files + } + } + return providers + }, + } +} + +func pythonRootToolingFile(name string) bool { + switch name { + case "setup.py", "conftest.py", "noxfile.py", "fabfile.py", "manage.py": + return true + } + return false +} diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go new file mode 100644 index 000000000..70f089d70 --- /dev/null +++ b/internal/indexer/python_tracked_import_test.go @@ -0,0 +1,159 @@ +package indexer + +import ( + "fmt" + "iter" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestMultiIndexer_TrackedPythonImport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, tc := range []struct { + name, manifest, requirements, entry, statement, providerName, workspace, localFile string + duplicate, cloneMissing, resolves bool + }{ + {name: "src module distribution differs", manifest: "[project]\ndependencies = ['pillow >= 1']", entry: "src/PIL/image.py", statement: "import PIL.image", resolves: true}, + {name: "package init", manifest: "[project]\ndependencies = ['pillow']", entry: "PIL/image/__init__.py", statement: "import PIL.image", resolves: true}, + {name: "requirements", requirements: "PILLOW[extra]>=1; python_version > '3'\n", entry: "src/PIL/image.py", statement: "import PIL.image", resolves: true}, + {name: "from package", manifest: "[project.optional-dependencies]\nimages = ['pillow']", entry: "src/PIL/__init__.py", statement: "from PIL import image", resolves: true}, + {name: "poetry dependency", manifest: "[tool.poetry.dependencies]\npillow = {path = '../provider'}", entry: "PIL/image.py", statement: "from PIL.image import f", resolves: true}, + {name: "poetry group normalized", manifest: "[tool.poetry.group.dev.dependencies]\nimage-dist = '*'", providerName: "Image._Dist", entry: "PIL/image.py", statement: "import PIL.image", resolves: true}, + {name: "top level module", manifest: "[project]\ndependencies = ['pillow']", entry: "image.py", statement: "import image", resolves: true}, + {name: "consumer namespace root", manifest: "[project]\ndependencies = ['pillow']", entry: "tests/helpers.py", statement: "import tests.helpers", localFile: "tests/helpers.py"}, + {name: "consumer namespace src", manifest: "[project]\ndependencies = ['pillow']", entry: "tests/helpers.py", statement: "import tests.helpers", localFile: "src/tests/helpers.py"}, + {name: "stdlib package", manifest: "[project]\ndependencies = ['pillow']", entry: "json/__init__.py", statement: "import json"}, + {name: "root setup script", manifest: "[project]\ndependencies = ['pillow']", entry: "setup.py", statement: "import setup"}, + {name: "root conftest script", manifest: "[project]\ndependencies = ['pillow']", entry: "conftest.py", statement: "import conftest"}, + {name: "root noxfile script", manifest: "[project]\ndependencies = ['pillow']", entry: "noxfile.py", statement: "import noxfile"}, + {name: "root fabfile script", manifest: "[project]\ndependencies = ['pillow']", entry: "fabfile.py", statement: "import fabfile"}, + {name: "root manage script", manifest: "[project]\ndependencies = ['pillow']", entry: "manage.py", statement: "import manage"}, + {name: "malformed consumer", manifest: "[project", entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "undeclared", entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "same named undeclared repo", manifest: "[project]\ndependencies = ['other']", providerName: "PIL", entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "ambiguous", manifest: "[project]\ndependencies = ['pillow']", duplicate: true, entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "ambiguous before module probe", manifest: "[project]\ndependencies = ['pillow']", duplicate: true, cloneMissing: true, entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "workspace refusal", manifest: "[project]\ndependencies = ['pillow']", workspace: "private", entry: "src/PIL/image.py", statement: "import PIL.image"}, + } { + t.Run(backend+"/"+tc.name, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + name := tc.providerName + if name == "" { + name = "Pillow" + } + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = '"+name+"'\n") + packageRoot := "PIL/__init__.py" + if strings.HasPrefix(tc.entry, "src/") { + packageRoot = "src/" + packageRoot + } + writeWorkspaceFile(t, provider, packageRoot, "") + if strings.HasPrefix(tc.entry, "tests/") { + writeWorkspaceFile(t, provider, "tests/__init__.py", "") + } + if tc.localFile != "" { + writeWorkspaceFile(t, consumer, tc.localFile, "def local(): pass\n") + } + writeWorkspaceFile(t, provider, tc.entry, "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", tc.manifest) + writeWorkspaceFile(t, consumer, "requirements-dev.txt", tc.requirements) + writeWorkspaceFile(t, consumer, "app.py", tc.statement+"\n") + ws := tc.workspace + if ws == "" { + ws = "shared" + } + repos := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: ws}} + if tc.duplicate { + clone := t.TempDir() + writeWorkspaceFile(t, clone, "pyproject.toml", "[tool.poetry]\nname = 'pillow'\n") + writeWorkspaceFile(t, clone, "src/PIL/__init__.py", "") + if !tc.cloneMissing { + writeWorkspaceFile(t, clone, tc.entry, "def f(): pass\n") + } + repos = append(repos, config.RepoEntry{Path: clone, Name: "clone", Workspace: "shared"}) + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: repos} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + _, err = mi.IndexAll() + require.NoError(t, err) + var imports []*graph.Edge + for _, e := range g.GetOutEdges("consumer/app.py") { + if e.Kind == graph.EdgeImports { + imports = append(imports, e) + } + } + require.Len(t, imports, 1) + require.Equal(t, tc.resolves, imports[0].CrossRepo, "target: %s", imports[0].To) + if tc.resolves { + require.Equal(t, "provider/"+tc.entry, imports[0].To) + } else { + require.NotEqual(t, "provider/"+tc.entry, imports[0].To) + require.NotEqual(t, "clone/"+tc.entry, imports[0].To) + } + }) + } + } +} + +func TestPythonPackageLookupNoProviders(t *testing.T) { + for _, count := range []int{1, 2} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + mi := &MultiIndexer{graph: graph.New(), repos: map[string]*RepoMetadata{}} + for i := 0; i < count; i++ { + mi.repos[fmt.Sprint(i)] = &RepoMetadata{RootPath: t.TempDir()} + } + require.Nil(t, mi.pythonPackageLookup()) + }) + } +} + +type pythonLookupScanStore struct { + graph.Store + globalScans int +} + +func (s *pythonLookupScanStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { + if repos == nil { + s.globalScans++ + } + return graph.FileNodeIdentitiesSeq(s.Store, repos) +} + +func TestPythonPackageLookupAvoidsGlobalFileScan(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + g := &pythonLookupScanStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "provider/src/PIL/__init__.py", FilePath: "provider/src/PIL/__init__.py", Kind: graph.KindFile, RepoPrefix: "provider"}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{ + "consumer": {RootPath: consumer}, "provider": {RootPath: provider}, + }} + lookup := mi.pythonPackageLookup() + require.NotNil(t, lookup) + require.Len(t, lookup.Providers("consumer", "PIL"), 1) + require.Zero(t, g.globalScans) +} diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 6a72fbc41..85dfd4e99 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -498,6 +498,9 @@ func (mi *MultiIndexer) newCrossRepoResolver() *resolver.CrossRepoResolver { cr.SetNpmDependencyLookup(mi.npmDependencyLookup()) cr.SetNpmPackageLookup(mi.npmPackageLookup()) cr.SetCargoCrateLookup(mi.cargoCrateLookup()) + if mi.IsMultiRepo() { + cr.SetPythonPackageLookup(mi.pythonPackageLookup) + } cr.SetPathAliasResolver(mi.pathAliasResolver()) cr.SetWorkspaceMembership(mi.workspaceMembershipResolver()) mi.applyRemoteStitch(cr) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 030967f3b..3e6138320 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -157,7 +157,7 @@ type CrossRepoResolver struct { cargoCrates CargoCrateLookup cargoRetargetRepos map[string]bool - + pythonPackages func() *PythonPackageLookup // pathAlias expands a JS/TS tsconfig/jsconfig path-alias / baseUrl // import specifier to the repo-prefixed file stem it targets. Same // contract as the field of the same name on Resolver — see @@ -435,6 +435,9 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS if err := cr.resolveCargoImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { return stats, err } + if err := cr.resolveTrackedPythonImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { + return stats, err + } // Share the master resolver's stable high-water/keyset stream. The cold // residual can exceed 200k edges; retaining it plus the cross-repo name, // raw-name, repo and qualified-name caches was the second whole-corpus heap @@ -721,6 +724,7 @@ func (cr *CrossRepoResolver) resolveScopedLocked(edges []*graph.Edge) *CrossRepo stats := &CrossRepoStats{ByRepo: make(map[string]int)} _ = cr.resolveTrackedNpmImports(context.Background(), slices.Values(edges), stats) _ = cr.resolveCargoImports(context.Background(), slices.Values(edges), stats) + _ = cr.resolveTrackedPythonImports(context.Background(), slices.Values(edges), stats) pending := make([]*graph.Edge, 0, len(edges)) for _, edge := range edges { if edge != nil && graph.IsUnresolvedTarget(edge.To) { diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index b88390f93..0772a0f62 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -110,10 +110,19 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed } } } + if cr.pythonPackages != nil { + for _, edges := range frontier.outByNode { + for _, edge := range edges { + if trackedPythonImportSpec(edge) != "" { + frontier.pending = append(frontier.pending, edge) + } + } + } + } pending := make([]*graph.Edge, 0, len(frontier.pending)) seen := make(map[graph.EdgeIdentity]struct{}, len(frontier.pending)) for _, edge := range frontier.pending { - if edge == nil || !graph.IsUnresolvedTarget(edge.To) && trackedNpmImportSpec(edge) == "" && cr.cargoImportSpec(edge) == "" { + if edge == nil || !graph.IsUnresolvedTarget(edge.To) && trackedNpmImportSpec(edge) == "" && cr.cargoImportSpec(edge) == "" && trackedPythonImportSpec(edge) == "" { continue } identity := graph.EdgeIdentityFor(edge) diff --git a/internal/resolver/module_attribution.go b/internal/resolver/module_attribution.go index b9c7f79e1..d4123c7c6 100644 --- a/internal/resolver/module_attribution.go +++ b/internal/resolver/module_attribution.go @@ -144,6 +144,12 @@ func (r *Resolver) attributeNonGoModuleImports() { } dependsBatch := make([]*graph.Edge, 0, dependsBatchCapacity) for _, p := range rewrites { + if strings.HasPrefix(p.moduleID, "module::pypi:") { + if p.edge.Meta == nil { + p.edge.Meta = map[string]any{} + } + p.edge.Meta["python_import"] = strings.TrimPrefix(p.oldTo, "external::") + } p.edge.To = p.moduleID p.edge.Origin = graph.OriginASTResolved reindexBatch = append(reindexBatch, graph.EdgeReindex{Edge: p.edge, OldTo: p.oldTo}) diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go new file mode 100644 index 000000000..5ceaafe3f --- /dev/null +++ b/internal/resolver/python_tracked_import.go @@ -0,0 +1,175 @@ +package resolver + +import ( + "context" + "iter" + "strings" + + "github.com/zzet/gortex/internal/graph" +) + +// PythonPackageLookup exposes only indexed tops from declared tracked +// distributions. The indexer owns manifest I/O and consumer-local checks. +type PythonPackageLookup struct { + HasTop func(top string) bool + Providers func(callerRepo, top string) map[string][]graph.FileNodeIdentity +} + +// SetPythonPackageLookup defers manifest reads until a non-stdlib Python import +// needs a retry. A nil result means no declared tracked providers exist. +func (cr *CrossRepoResolver) SetPythonPackageLookup(build func() *PythonPackageLookup) { + cr.pythonPackages = build +} + +func trackedPythonImportSpec(e *graph.Edge) string { + if e == nil || e.Kind != graph.EdgeImports || !isPythonSourcePath(e.FilePath) { + return "" + } + var spec string + switch { + case strings.HasPrefix(e.To, "module::python:"): + return "" + case strings.HasPrefix(e.To, "unresolved::import::"): + spec = strings.TrimPrefix(e.To, "unresolved::import::") + case strings.HasPrefix(e.To, "external::"): + spec = strings.TrimPrefix(e.To, "external::") + case strings.HasPrefix(e.To, "module::pypi:"): + spec, _ = e.Meta["python_import"].(string) + } + top, _, _ := strings.Cut(spec, ".") + if !isDottedPythonModule(spec) || isPythonStdlibTop(top) { + return "" + } + return spec +} + +// Attributed targets retain the top even in a metadata-free edge projection. +// The complete dotted path must still be recovered from the full edge later. +func trackedPythonImportTop(e *graph.Edge) string { + if e != nil && e.Kind == graph.EdgeImports && isPythonSourcePath(e.FilePath) && strings.HasPrefix(e.To, "module::pypi:") { + top := strings.TrimPrefix(e.To, "module::pypi:") + if isDottedPythonModule(top) && !isPythonStdlibTop(top) { + return top + } + return "" + } + top, _, _ := strings.Cut(trackedPythonImportSpec(e), ".") + return top +} + +func (cr *CrossRepoResolver) trackedPythonImport(caller *graph.Node, spec string, lookup *PythonPackageLookup, nodes map[string]*graph.Node) *graph.Node { + top, _, _ := strings.Cut(spec, ".") + var files []graph.FileNodeIdentity + providers := 0 + for _, candidates := range lookup.Providers(caller.RepoPrefix, top) { + if len(candidates) == 0 || !cr.crossWorkspaceEligible(candidateWorkspaceID(caller), fileCandidateWorkspaceID(candidates[0]), spec) { + continue + } + providers++ + files = candidates + } + if providers != 1 { + return nil + } + var match *graph.Node + for _, file := range files { + base := strings.TrimSuffix(strings.TrimSuffix(file.FilePath, "/__init__.py"), ".py") + stem := strings.TrimSuffix(base, top) + strings.ReplaceAll(spec, ".", "/") + for _, suffix := range []string{".py", "/__init__.py"} { + id := stem + suffix + node, seen := nodes[id] + if !seen { + node = cr.cachedGetNode(id) + nodes[id] = node + } + if node == nil || !pythonModuleFileMatches(node.FilePath, spec) { + continue + } + if match != nil && match.ID != node.ID { + return nil + } + match = node + } + } + return match +} + +// Retry only import/file edges; never reconsider working links. As in the npm +// bridge, close the light cursor before probing nodes or hydrating bounded pages. +func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats) error { + if cr.pythonPackages == nil { + return ctx.Err() + } + var pending []*graph.Edge + for e := range edges { + if err := ctx.Err(); err != nil { + return err + } + if trackedPythonImportTop(e) != "" { + pending = append(pending, e) + } + } + if len(pending) == 0 { + return ctx.Err() + } + lookup := cr.pythonPackages() + if lookup == nil { + return ctx.Err() + } + eligible := pending[:0] + for _, e := range pending { + if lookup.HasTop(trackedPythonImportTop(e)) { + eligible = append(eligible, e) + } + } + nodes := map[string]*graph.Node{} + for start := 0; start < len(eligible); start += resolvePendingPageRows { + if err := ctx.Err(); err != nil { + return err + } + page := eligible[start:min(start+resolvePendingPageRows, len(eligible))] + sites := make([]graph.EdgeSite, 0, len(page)) + ids := make([]string, 0, len(page)) + seen := map[string]bool{} + for _, e := range page { + sites = append(sites, graph.EdgeSite{From: e.From, Line: e.Line, Kind: graph.EdgeImports}) + if !seen[e.From] { + ids = append(ids, e.From) + seen[e.From] = true + } + } + full := cr.graph.GetEdgeCandidates(nil, sites) + callers := cr.hotCachedNodesByIDs(ids) + var batch []graph.EdgeReindex + for _, projected := range page { + caller := callers[projected.From] + if caller == nil { + continue + } + for _, e := range full.Site(projected.From, projected.Line, graph.EdgeImports) { + if e.To != projected.To { + continue + } + spec := trackedPythonImportSpec(e) + if spec == "" { + continue + } + match := cr.trackedPythonImport(caller, spec, lookup, nodes) + if match == nil { + continue + } + old := e.To + e.To, e.CrossRepo = match.ID, true + batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: old}) + projected.To, projected.CrossRepo = e.To, e.CrossRepo + stats.Resolved++ + stats.CrossRepoEdges++ + stats.ByRepo[match.RepoPrefix]++ + } + } + if len(batch) > 0 { + cr.graph.ReindexEdges(batch) + } + } + return ctx.Err() +} diff --git a/internal/resolver/python_tracked_import_test.go b/internal/resolver/python_tracked_import_test.go new file mode 100644 index 000000000..af4ab2a89 --- /dev/null +++ b/internal/resolver/python_tracked_import_test.go @@ -0,0 +1,148 @@ +package resolver + +import ( + "context" + "iter" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestTrackedPythonImportRetry(t *testing.T) { + for _, target := range []string{"unresolved::import::PIL.image", "external::PIL.image", "module::pypi:PIL", "local.py"} { + t.Run(target, func(t *testing.T) { + g := graph.New() + for _, file := range []string{"consumer/app.py", "provider/src/PIL/__init__.py", "provider/src/PIL/image.py", "local.py"} { + repo := "provider" + if file == "consumer/app.py" { + repo = "consumer" + } + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, Language: "python", RepoPrefix: repo, WorkspaceID: "shared"}) + } + e := &graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: target, Kind: graph.EdgeImports, Line: 1, Meta: map[string]any{"python_import": "PIL.image", "keep": "metadata"}} + g.AddEdge(e) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + return &PythonPackageLookup{HasTop: func(top string) bool { return top == "PIL" }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + require.Equal(t, "consumer", caller) + require.Equal(t, "PIL", top) + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/src/PIL/__init__.py", FilePath: "provider/src/PIL/__init__.py", RepoPrefix: "provider", WorkspaceID: "shared"}}} + }} + }) + cr.ResolveMutationFiles([]string{"consumer/app.py"}, nil) + imports := outEdgesOfKind(g, "consumer/app.py", graph.EdgeImports) + require.Len(t, imports, 1) + if target == "local.py" { + require.Equal(t, target, imports[0].To) + return + } + require.Equal(t, "provider/src/PIL/image.py", imports[0].To) + require.True(t, imports[0].CrossRepo) + require.Equal(t, "metadata", imports[0].Meta["keep"]) + }) + } +} + +func TestTrackedPythonImportSkipsStdlibBeforeLookup(t *testing.T) { + for _, target := range []string{"module::python:stdlib::json", "external::json", "unresolved::import::json.tool"} { + t.Run(target, func(t *testing.T) { + g := graph.New() + g.AddNode(&graph.Node{ID: "app/a.py", FilePath: "app/a.py", Kind: graph.KindFile, RepoPrefix: "app"}) + g.AddEdge(&graph.Edge{From: "app/a.py", FilePath: "app/a.py", To: target, Kind: graph.EdgeImports, Meta: map[string]any{"python_import": "json"}}) + lookups := 0 + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + lookups++ + return nil + }) + cr.ResolveMutationFiles([]string{"app/a.py"}, nil) + require.Zero(t, lookups) + }) + } +} + +type pythonImportPageStore struct { + graph.Store + fullScans, callerReads int + hydrated, callerBatches []int +} + +func (s *pythonImportPageStore) EdgesByKind(kind graph.EdgeKind) iter.Seq[*graph.Edge] { + if kind == graph.EdgeImports { + s.fullScans++ + } + return s.Store.EdgesByKind(kind) +} + +func (s *pythonImportPageStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + return graph.EdgesLightSeq(s.Store, kinds...) +} + +func (s *pythonImportPageStore) GetNode(id string) *graph.Node { + if id == "consumer/a.py" { + s.callerReads++ + } + return s.Store.GetNode(id) +} + +func (s *pythonImportPageStore) GetNodesByIDs(ids []string) map[string]*graph.Node { + s.callerBatches = append(s.callerBatches, len(ids)) + return s.Store.GetNodesByIDs(ids) +} + +func (s *pythonImportPageStore) GetEdgeCandidates(endpoints []graph.EdgeEndpoint, sites []graph.EdgeSite) graph.EdgeCandidateSet { + s.hydrated = append(s.hydrated, len(sites)) + return s.Store.GetEdgeCandidates(endpoints, sites) +} + +func TestTrackedPythonImportPagedFullPass(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var store graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store = db + } + g := &pythonImportPageStore{Store: store} + g.AddNode(&graph.Node{ID: "consumer/a.py", FilePath: "consumer/a.py", Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared"}) + g.AddNode(&graph.Node{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: "shared"}) + for line := 1; line <= resolvePendingPageRows+1; line++ { + g.AddEdge(&graph.Edge{From: "consumer/a.py", FilePath: "consumer/a.py", To: "module::pypi:PIL", Kind: graph.EdgeImports, Line: line, Meta: map[string]any{"python_import": "PIL", "keep": "metadata"}}) + } + g.AddEdge(&graph.Edge{From: "consumer/a.py", FilePath: "consumer/a.py", To: "module::pypi:untracked", Kind: graph.EdgeImports, Line: resolvePendingPageRows + 2, Meta: map[string]any{"python_import": "untracked"}}) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + return &PythonPackageLookup{HasTop: func(top string) bool { return top == "PIL" }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + if top != "PIL" { + return nil + } + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", RepoPrefix: "provider", WorkspaceID: "shared"}}} + }} + }) + stats, err := cr.ResolveAllContext(context.Background()) + require.NoError(t, err) + require.Equal(t, resolvePendingPageRows+1, stats.Resolved) + require.Zero(t, g.fullScans, "full import scans must use metadata-free projections") + require.Zero(t, g.callerReads, "caller nodes must be loaded in batches") + require.Equal(t, []int{resolvePendingPageRows, 1}, g.hydrated) + require.Equal(t, []int{1, 1}, g.callerBatches) + imports := g.GetOutEdges("consumer/a.py") + require.Len(t, imports, resolvePendingPageRows+2) + for _, e := range imports { + if e.Kind == graph.EdgeImports && e.Line <= resolvePendingPageRows+1 { + require.Equal(t, "provider/PIL/__init__.py", e.To) + require.Equal(t, "metadata", e.Meta["keep"]) + } else if e.Kind == graph.EdgeImports { + require.Equal(t, "module::pypi:untracked", e.To) + require.False(t, e.CrossRepo) + } + } + }) + } +} From e79898a0c87f891507f4b0569884a2b401d29749 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Fri, 23 Oct 2026 07:22:21 -0700 Subject: [PATCH 020/111] fix(resolver): page the Python import retry and keep src packages local The full-pass retry kept every tracked-import edge before processing any page. With SQLite's projection scanner it now processes bounded pages; other stores drain the light scan first, so no live cursor stays open while the retry reads and writes the graph. The consumer's local names now include both the literal top-level name and the src-layout name. A root `src` package and a package under `src/` each stay local, even when `src/__init__.py` exists. --- internal/indexer/python_tracked_import.go | 10 ++- .../indexer/python_tracked_import_test.go | 12 ++- internal/resolver/cross_repo.go | 2 +- internal/resolver/python_tracked_import.go | 83 +++++++++++++------ .../resolver/python_tracked_import_test.go | 50 ++++++++++- 5 files changed, 122 insertions(+), 35 deletions(-) diff --git a/internal/indexer/python_tracked_import.go b/internal/indexer/python_tracked_import.go index c231dece1..7ee1c4c0d 100644 --- a/internal/indexer/python_tracked_import.go +++ b/internal/indexer/python_tracked_import.go @@ -131,14 +131,18 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { local, seen := localTops[caller] if !seen { local = map[string]bool{} + // Record both the literal top-level name and the src-layout name: + // a root `src` package and src/ must each stay local, and an + // extra local name only blocks a cross-repo link. for file := range graph.FileNodeIdentitiesSeq(mi.graph, []string{caller}) { if !strings.HasSuffix(file.FilePath, ".py") { continue } rel := strings.TrimPrefix(file.FilePath, caller+"/") - rel = strings.TrimPrefix(rel, "src/") - name, _, _ := strings.Cut(rel, "/") - local[strings.TrimSuffix(name, ".py")] = true + for _, path := range []string{rel, strings.TrimPrefix(rel, "src/")} { + name, _, _ := strings.Cut(path, "/") + local[strings.TrimSuffix(name, ".py")] = true + } } localTops[caller] = local } diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go index 70f089d70..185cb8526 100644 --- a/internal/indexer/python_tracked_import_test.go +++ b/internal/indexer/python_tracked_import_test.go @@ -20,8 +20,8 @@ import ( func TestMultiIndexer_TrackedPythonImport(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { for _, tc := range []struct { - name, manifest, requirements, entry, statement, providerName, workspace, localFile string - duplicate, cloneMissing, resolves bool + name, manifest, requirements, entry, statement, providerName, workspace, localFile, extraLocal string + duplicate, cloneMissing, resolves bool }{ {name: "src module distribution differs", manifest: "[project]\ndependencies = ['pillow >= 1']", entry: "src/PIL/image.py", statement: "import PIL.image", resolves: true}, {name: "package init", manifest: "[project]\ndependencies = ['pillow']", entry: "PIL/image/__init__.py", statement: "import PIL.image", resolves: true}, @@ -32,6 +32,8 @@ func TestMultiIndexer_TrackedPythonImport(t *testing.T) { {name: "top level module", manifest: "[project]\ndependencies = ['pillow']", entry: "image.py", statement: "import image", resolves: true}, {name: "consumer namespace root", manifest: "[project]\ndependencies = ['pillow']", entry: "tests/helpers.py", statement: "import tests.helpers", localFile: "tests/helpers.py"}, {name: "consumer namespace src", manifest: "[project]\ndependencies = ['pillow']", entry: "tests/helpers.py", statement: "import tests.helpers", localFile: "src/tests/helpers.py"}, + {name: "consumer root src package", manifest: "[project]\ndependencies = ['pillow']", entry: "src/foo.py", statement: "import src.foo", localFile: "src/__init__.py"}, + {name: "consumer src init with src namespace", manifest: "[project]\ndependencies = ['pillow']", entry: "tests/helpers.py", statement: "import tests.helpers", localFile: "src/__init__.py", extraLocal: "src/tests/helpers.py"}, {name: "stdlib package", manifest: "[project]\ndependencies = ['pillow']", entry: "json/__init__.py", statement: "import json"}, {name: "root setup script", manifest: "[project]\ndependencies = ['pillow']", entry: "setup.py", statement: "import setup"}, {name: "root conftest script", manifest: "[project]\ndependencies = ['pillow']", entry: "conftest.py", statement: "import conftest"}, @@ -60,9 +62,15 @@ func TestMultiIndexer_TrackedPythonImport(t *testing.T) { if strings.HasPrefix(tc.entry, "tests/") { writeWorkspaceFile(t, provider, "tests/__init__.py", "") } + if tc.statement == "import src.foo" { + writeWorkspaceFile(t, provider, "src/__init__.py", "") + } if tc.localFile != "" { writeWorkspaceFile(t, consumer, tc.localFile, "def local(): pass\n") } + if tc.extraLocal != "" { + writeWorkspaceFile(t, consumer, tc.extraLocal, "def local(): pass\n") + } writeWorkspaceFile(t, provider, tc.entry, "def f(): pass\n") writeWorkspaceFile(t, consumer, "pyproject.toml", tc.manifest) writeWorkspaceFile(t, consumer, "requirements-dev.txt", tc.requirements) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 3e6138320..3ec0f5686 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -435,7 +435,7 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS if err := cr.resolveCargoImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { return stats, err } - if err := cr.resolveTrackedPythonImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { + if err := cr.resolveTrackedPythonImports(ctx, cr.trackedPythonImportEdges(), stats); err != nil { return stats, err } // Share the master resolver's stable high-water/keyset stream. The cold diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go index 5ceaafe3f..2b2223bc3 100644 --- a/internal/resolver/python_tracked_import.go +++ b/internal/resolver/python_tracked_import.go @@ -3,6 +3,7 @@ package resolver import ( "context" "iter" + "slices" "strings" "github.com/zzet/gortex/internal/graph" @@ -94,40 +95,73 @@ func (cr *CrossRepoResolver) trackedPythonImport(caller *graph.Node, spec string return match } -// Retry only import/file edges; never reconsider working links. As in the npm -// bridge, close the light cursor before probing nodes or hydrating bounded pages. -func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats) error { - if cr.pythonPackages == nil { - return ctx.Err() - } - var pending []*graph.Edge - for e := range edges { - if err := ctx.Err(); err != nil { - return err +// SQLite's projection scanner closes each page cursor before yielding, so the +// retry may hydrate and reindex edges without retaining the complete light scan. +// Other stores drain the light scan first: EdgesLightSeq can hold a live cursor +// that must be closed before the retry reads or writes the graph. +func (cr *CrossRepoResolver) trackedPythonImportEdges() iter.Seq[*graph.Edge] { + if scanner, ok := cr.graph.(interface { + ScanTestEdgeProjections([]graph.EdgeKind, int, func([]graph.TestEdgeProjection) bool) + }); ok { + return func(yield func(*graph.Edge) bool) { + scanner.ScanTestEdgeProjections([]graph.EdgeKind{graph.EdgeImports}, resolvePendingPageRows, func(page []graph.TestEdgeProjection) bool { + for _, e := range page { + if !yield(&graph.Edge{From: e.From, To: e.To, Kind: e.Kind, FilePath: e.FilePath, Line: e.Line}) { + return false + } + } + return true + }) } + } + var edges []*graph.Edge + for e := range graph.EdgesLightSeq(cr.graph, graph.EdgeImports) { if trackedPythonImportTop(e) != "" { - pending = append(pending, e) + edges = append(edges, e) } } - if len(pending) == 0 { - return ctx.Err() - } - lookup := cr.pythonPackages() - if lookup == nil { + return slices.Values(edges) +} + +// Retry only import/file edges; never reconsider working links. +func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats) error { + if cr.pythonPackages == nil { return ctx.Err() } - eligible := pending[:0] - for _, e := range pending { - if lookup.HasTop(trackedPythonImportTop(e)) { - eligible = append(eligible, e) + next, stop := iter.Pull(edges) + defer stop() + var lookup *PythonPackageLookup + for { + page := make([]*graph.Edge, 0, resolvePendingPageRows) + for len(page) < resolvePendingPageRows { + if err := ctx.Err(); err != nil { + return err + } + e, ok := next() + if !ok { + break + } + top := trackedPythonImportTop(e) + if top == "" { + continue + } + if lookup == nil { + lookup = cr.pythonPackages() + if lookup == nil { + return ctx.Err() + } + } + if lookup.HasTop(top) { + page = append(page, e) + } } - } - nodes := map[string]*graph.Node{} - for start := 0; start < len(eligible); start += resolvePendingPageRows { if err := ctx.Err(); err != nil { return err } - page := eligible[start:min(start+resolvePendingPageRows, len(eligible))] + if len(page) == 0 { + return nil + } + nodes := map[string]*graph.Node{} sites := make([]graph.EdgeSite, 0, len(page)) ids := make([]string, 0, len(page)) seen := map[string]bool{} @@ -171,5 +205,4 @@ func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, ed cr.graph.ReindexEdges(batch) } } - return ctx.Err() } diff --git a/internal/resolver/python_tracked_import_test.go b/internal/resolver/python_tracked_import_test.go index af4ab2a89..a1f17b0a5 100644 --- a/internal/resolver/python_tracked_import_test.go +++ b/internal/resolver/python_tracked_import_test.go @@ -67,8 +67,9 @@ func TestTrackedPythonImportSkipsStdlibBeforeLookup(t *testing.T) { type pythonImportPageStore struct { graph.Store - fullScans, callerReads int - hydrated, callerBatches []int + fullScans, callerReads int + hydrated, callerBatches []int + scanned, processed, peakPending int } func (s *pythonImportPageStore) EdgesByKind(kind graph.EdgeKind) iter.Seq[*graph.Edge] { @@ -79,7 +80,34 @@ func (s *pythonImportPageStore) EdgesByKind(kind graph.EdgeKind) iter.Seq[*graph } func (s *pythonImportPageStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { - return graph.EdgesLightSeq(s.Store, kinds...) + return func(yield func(*graph.Edge) bool) { + for e := range graph.EdgesLightSeq(s.Store, kinds...) { + if trackedPythonImportTop(e) == "PIL" { + s.scanned++ + } + if !yield(e) { + return + } + } + } +} + +type pythonImportProjectionStore struct { + *pythonImportPageStore +} + +func (s *pythonImportProjectionStore) ScanTestEdgeProjections(kinds []graph.EdgeKind, pageSize int, yield func([]graph.TestEdgeProjection) bool) { + s.Store.(graph.TestProjectionScanner).ScanTestEdgeProjections(kinds, pageSize, func(page []graph.TestEdgeProjection) bool { + for _, e := range page { + if e.To == "module::pypi:PIL" { + s.scanned++ + } + if !yield([]graph.TestEdgeProjection{e}) { + return false + } + } + return true + }) } func (s *pythonImportPageStore) GetNode(id string) *graph.Node { @@ -96,6 +124,8 @@ func (s *pythonImportPageStore) GetNodesByIDs(ids []string) map[string]*graph.No func (s *pythonImportPageStore) GetEdgeCandidates(endpoints []graph.EdgeEndpoint, sites []graph.EdgeSite) graph.EdgeCandidateSet { s.hydrated = append(s.hydrated, len(sites)) + s.peakPending = max(s.peakPending, s.scanned-s.processed) + s.processed += len(sites) return s.Store.GetEdgeCandidates(endpoints, sites) } @@ -116,8 +146,14 @@ func TestTrackedPythonImportPagedFullPass(t *testing.T) { g.AddEdge(&graph.Edge{From: "consumer/a.py", FilePath: "consumer/a.py", To: "module::pypi:PIL", Kind: graph.EdgeImports, Line: line, Meta: map[string]any{"python_import": "PIL", "keep": "metadata"}}) } g.AddEdge(&graph.Edge{From: "consumer/a.py", FilePath: "consumer/a.py", To: "module::pypi:untracked", Kind: graph.EdgeImports, Line: resolvePendingPageRows + 2, Meta: map[string]any{"python_import": "untracked"}}) - cr := NewCrossRepo(g) + var retryStore graph.Store = g + if backend == "sqlite" { + retryStore = &pythonImportProjectionStore{g} + } + cr := NewCrossRepo(retryStore) + lookups := 0 cr.SetPythonPackageLookup(func() *PythonPackageLookup { + lookups++ return &PythonPackageLookup{HasTop: func(top string) bool { return top == "PIL" }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { if top != "PIL" { return nil @@ -128,8 +164,14 @@ func TestTrackedPythonImportPagedFullPass(t *testing.T) { stats, err := cr.ResolveAllContext(context.Background()) require.NoError(t, err) require.Equal(t, resolvePendingPageRows+1, stats.Resolved) + require.Equal(t, 1, lookups, "build the lookup once across all pages") require.Zero(t, g.fullScans, "full import scans must use metadata-free projections") require.Zero(t, g.callerReads, "caller nodes must be loaded in batches") + if backend == "sqlite" { + // Stores without the projection scanner drain the light scan + // first so no live cursor stays open during the retry. + require.LessOrEqual(t, g.peakPending, resolvePendingPageRows, "process each bounded page before retaining more tracked imports") + } require.Equal(t, []int{resolvePendingPageRows, 1}, g.hydrated) require.Equal(t, []int{1, 1}, g.callerBatches) imports := g.GetOutEdges("consumer/a.py") From c7265faa7a17f810340aa2acd8536b03163defe4 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 03:12:48 -0700 Subject: [PATCH 021/111] fix(indexer): preserve tracked Python import and call lifecycle --- .../daemon_warm_python_frontier_test.go | 25 ++ internal/indexer/affected_by_bound_test.go | 2 +- internal/indexer/edit_delta_prior_bindings.go | 2 +- internal/indexer/file_meta.go | 1 + .../go_module_import_lifecycle_test.go | 2 +- internal/indexer/incremental_batch.go | 20 +- .../indexer/incremental_restub_gate_test.go | 2 +- internal/indexer/incremental_watcher_batch.go | 11 +- internal/indexer/indexer.go | 23 +- internal/indexer/legacy_file_identity_test.go | 4 +- internal/indexer/multi.go | 3 + internal/indexer/npm_lifecycle_test.go | 4 +- .../indexer/python_import_lifecycle_test.go | 289 ++++++++++++++++++ internal/indexer/python_import_restub.go | 113 +++++++ internal/indexer/python_import_restub_test.go | 60 ++++ internal/indexer/python_import_reuse_test.go | 25 ++ internal/indexer/repo_eviction_scope.go | 1 + internal/indexer/repository_untrack.go | 1 + internal/resolver/cross_repo.go | 48 ++- internal/resolver/python_absolute_imports.go | 8 +- .../resolver/python_import_call_retry_test.go | 172 +++++++++++ internal/resolver/python_tracked_import.go | 75 ++++- .../resolver/python_tracked_import_test.go | 7 +- internal/resolver/resolver.go | 8 + 24 files changed, 871 insertions(+), 35 deletions(-) create mode 100644 cmd/gortex/daemon_warm_python_frontier_test.go create mode 100644 internal/indexer/python_import_lifecycle_test.go create mode 100644 internal/indexer/python_import_restub.go create mode 100644 internal/indexer/python_import_restub_test.go create mode 100644 internal/indexer/python_import_reuse_test.go create mode 100644 internal/resolver/python_import_call_retry_test.go diff --git a/cmd/gortex/daemon_warm_python_frontier_test.go b/cmd/gortex/daemon_warm_python_frontier_test.go new file mode 100644 index 000000000..fc4a825bc --- /dev/null +++ b/cmd/gortex/daemon_warm_python_frontier_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/indexer" +) + +func TestWarmupPythonConsumersStayOutsideDerivedPlan(t *testing.T) { + for _, full := range []bool{false, true} { + frontier := newWarmupDeltaFrontier() + frontier.record(&indexer.IndexResult{ + RepoPrefix: "provider", FullRetrack: full, + DerivedInvalidation: indexer.DerivedInvalidationPlan{Files: []string{"provider/src/PIL/image.py"}}, + CrossRepoResolveFiles: []string{"consumer/app.py"}, + }) + plans, files, exact := frontier.snapshot(1, false) + require.Contains(t, files, "consumer/app.py") + require.Equal(t, !full, exact) + for _, plan := range plans { + require.NotContains(t, plan.Files, "consumer/app.py") + } + } +} diff --git a/internal/indexer/affected_by_bound_test.go b/internal/indexer/affected_by_bound_test.go index d726a0e55..062a1f5c8 100644 --- a/internal/indexer/affected_by_bound_test.go +++ b/internal/indexer/affected_by_bound_test.go @@ -387,7 +387,7 @@ func TestDeletionShapedStagesCarryNoInEdges(t *testing.T) { stages := []*incrementalBatchStage{{graphPath: restubDefPath, priorNodes: prior}} view := loadIncrementalPriorView(st, stages) - carried, _, _ := restubIncomingRefsFromView(st, stages, view) + carried, _, _, _ := restubIncomingRefsFromView(st, stages, view) require.Empty(t, carried, "a deletion stage must park every in-edge; a carried edge has no re-state path here") assert.Equal(t, 1, st.restubWrites("Foo"), diff --git a/internal/indexer/edit_delta_prior_bindings.go b/internal/indexer/edit_delta_prior_bindings.go index f57b6f08a..051e0ac43 100644 --- a/internal/indexer/edit_delta_prior_bindings.go +++ b/internal/indexer/edit_delta_prior_bindings.go @@ -263,7 +263,7 @@ var reuseSemanticMetaKeys = []string{"confirmed_from_origin", "semantic_source"} // reuseBindingMetaKeys are importer-owned binding stubs a reused bind keeps // even without confirmation reuse; applyReuseSemanticMeta rewrites each to the // fresh call spelling. -var reuseBindingMetaKeys = []string{"cargo_call_stub"} +var reuseBindingMetaKeys = []string{"cargo_call_stub", "python_call_stub"} // reuseCarriedMetaKeys are importer-owned binding markers a reused bind keeps // verbatim, even when its confirmation provenance is dropped. go_import lets a diff --git a/internal/indexer/file_meta.go b/internal/indexer/file_meta.go index b14ca5652..2bfeceec8 100644 --- a/internal/indexer/file_meta.go +++ b/internal/indexer/file_meta.go @@ -199,6 +199,7 @@ type reparsePendingEnrichmentBatch struct { npmConsumerFiles []string cargoAddedFile bool cargoConsumerFiles []string + pythonConsumerFiles []string byFile map[string]bool deferResolverCatchup bool deferredAffectedFiles map[string]struct{} diff --git a/internal/indexer/go_module_import_lifecycle_test.go b/internal/indexer/go_module_import_lifecycle_test.go index 0bbd4ddac..0871a7229 100644 --- a/internal/indexer/go_module_import_lifecycle_test.go +++ b/internal/indexer/go_module_import_lifecycle_test.go @@ -149,7 +149,7 @@ func TestGoModuleImportParkBeforeEviction(t *testing.T) { restubGoModuleRepoIncoming(g, "b") } else if route == "deleted_batch" { provider := g.GetNode("b/lib.go") - carried, _, _ := restubIncomingRefsFromView(g, + carried, _, _, _ := restubIncomingRefsFromView(g, []*incrementalBatchStage{{graphPath: "b/lib.go", priorNodes: []*graph.Node{provider}}}, incrementalPriorView{inByNode: g.GetInEdgesByNodeIDs([]string{provider.ID})}) assert.Empty(t, carried, "a deleted file cannot carry a resolved target") diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 6290a4b99..bf3b6be62 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -1113,8 +1113,9 @@ func (idx *Indexer) commitStructuralIncrementalBatch( markerBatch.recordDeferredPriorBindings(stagePriorBindings(idx.graph, stages, view)) } idx.applyLap("prior_bindings") - carried, npmConsumers, cargoConsumers := restubIncomingRefsFromView(idx.graph, stages, view) + carried, npmConsumers, cargoConsumers, pythonConsumers := restubIncomingRefsFromView(idx.graph, stages, view) if markerBatch != nil { + markerBatch.pythonConsumerFiles = appendUniqueSorted(markerBatch.pythonConsumerFiles, pythonConsumers...) markerBatch.npmConsumerFiles = appendUniqueSorted(markerBatch.npmConsumerFiles, npmConsumers...) markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, cargoConsumers...) for _, stage := range stages { @@ -1379,9 +1380,10 @@ func restubIncomingRefsFromView( g graph.Store, stages []*incrementalBatchStage, view incrementalPriorView, -) ([]*graph.Edge, []string, []string) { +) ([]*graph.Edge, []string, []string, []string) { evicted := structuralPriorIDs(stages) - consumers := restubTrackedCargoImportsFromView(g, evicted, view.inByNode, false) + cargoConsumers := restubTrackedCargoImportsFromView(g, evicted, view.inByNode, false) + pythonConsumers := restubTrackedPythonImportsFromView(g, evicted, view.inByNode, false) reparsed := reparsedPaths(stages) npmConsumers := restubTrackedNpmImportsFromView(g, evicted, view.inByNode) var reindexes []graph.EdgeReindex @@ -1405,7 +1407,7 @@ func restubIncomingRefsFromView( // is not reparsed — so re-state it. if _, survives := frontier.survivingIDs[node.ID]; !survives && (!frontier.conservative || stage.result == nil) { for _, edge := range view.inByNode[node.ID] { - if edge == nil || edge.Kind != graph.EdgeImports { + if edge == nil || edge.Kind != graph.EdgeImports || edge.To != node.ID { continue } if _, sourceEvicted := evicted[edge.From]; sourceEvicted { @@ -1558,6 +1560,9 @@ func restubIncomingRefsFromView( if original, _ := edge.Meta[resolver.CargoCallStubMeta].(string); edge.Kind == graph.EdgeCalls && edge.CrossRepo && original != "" { edge.To, edge.CrossRepo, edge.ConfidenceLabel = original, false, "" } + if original, _ := edge.Meta[resolver.PythonCallStubMeta].(string); edge.Kind == graph.EdgeCalls && original != "" { + edge.To, edge.CrossRepo, edge.ConfidenceLabel = original, false, "" + } reindexes = append(reindexes, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) } } @@ -1565,7 +1570,7 @@ func restubIncomingRefsFromView( if len(reindexes) > 0 { g.ReindexEdges(reindexes) } - return canonicalizeLegacyCarried(stages, evicted, carried), npmConsumers, consumers + return canonicalizeLegacyCarried(stages, evicted, carried), npmConsumers, cargoConsumers, pythonConsumers } // dataflowSourceSite is a reference's site: the file that records it and its @@ -2552,9 +2557,10 @@ func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInva // carry therefore means a deletion stage stopped being conservative, // which would destroy those edges with no compile error to show for // it — so the assumption is checked, not just documented. - carried, _, consumers := restubIncomingRefsFromView(idx.graph, stages, view) + carried, _, cargoConsumers, pythonConsumers := restubIncomingRefsFromView(idx.graph, stages, view) if markerBatch != nil { - markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, consumers...) + markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, cargoConsumers...) + markerBatch.pythonConsumerFiles = appendUniqueSorted(markerBatch.pythonConsumerFiles, pythonConsumers...) } if len(carried) > 0 { idx.logger.Error("indexer: deletion restub carried in-edges with no re-state path", diff --git a/internal/indexer/incremental_restub_gate_test.go b/internal/indexer/incremental_restub_gate_test.go index 8edc5283d..af42f0003 100644 --- a/internal/indexer/incremental_restub_gate_test.go +++ b/internal/indexer/incremental_restub_gate_test.go @@ -118,7 +118,7 @@ func runRestubStage( stages := []*incrementalBatchStage{stage} view := loadIncrementalPriorView(st, stages) - carried, _, _ := restubIncomingRefsFromView(st, stages, view) + carried, _, _, _ := restubIncomingRefsFromView(st, stages, view) evictFilesBatched(st, []string{restubDefPath}) st.AddBatch(fresh, append(append([]*graph.Edge{}, freshEdges...), carried...)) diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index c588f7ff0..3b7cac7d1 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -537,6 +537,14 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( batch.dropDeferredPriorBindings() } crossRepoFiles := appendUniqueSorted(nil, files...) + if needed { + for _, file := range files { + if strings.HasSuffix(file, ".py") { + crossRepoFiles = appendUniqueSorted(crossRepoFiles, mi.pythonConsumerFilesForRepo(repoPrefix)...) + break + } + } + } cargoRetargetRepo := "" if batch != nil { crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.npmConsumerFiles...) @@ -545,6 +553,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( crossRepoFiles = appendUniqueSorted(crossRepoFiles, mi.cargoConsumerFilesForRepo(repoPrefix, true)...) cargoRetargetRepo = repoPrefix } + crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.pythonConsumerFiles...) } if result != nil { crossRepoFiles = appendUniqueSorted(crossRepoFiles, result.DerivedInvalidation.Files...) @@ -594,7 +603,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } crossRepoFiles = appendUniqueSorted(crossRepoFiles, semanticFiles...) } - if needed || len(semanticFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || len(crossRepoFiles) > 0 && (batch.cargoAddedFile || len(batch.cargoConsumerFiles) > 0)) { + if needed || len(semanticFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || len(batch.pythonConsumerFiles) > 0 || len(crossRepoFiles) > 0 && (batch.cargoAddedFile || len(batch.cargoConsumerFiles) > 0)) { if idx != nil { idx.observeIncrementalCatchup("cross_repo", crossRepoFiles) } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 6908f7cfd..60cde9dd4 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -3445,6 +3445,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index shadowReplacementStarted = true restubTrackedNpmRepoIncoming(diskTarget, idx.RepoPrefix()) parked := restubTrackedCargoRepoIncoming(diskTarget, idx.RepoPrefix()) + parked = appendUniqueSorted(parked, restubTrackedPythonRepoIncoming(diskTarget, idx.RepoPrefix())...) if result != nil { result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, parked...) } @@ -5175,9 +5176,9 @@ func (idx *Indexer) indexFile( // batch replacement by deleting its prior documents explicitly; // canonical contracts follow the store's owner-retention decision. idx.deleteSymbolFTS(oldFTSNodeIDs) - var npmConsumers, cargoConsumers []string + var npmConsumers, cargoConsumers, pythonConsumers []string if len(legacyIDs) == 0 { - npmConsumers, cargoConsumers = idx.restubIncomingRefs(graphPath) + npmConsumers, cargoConsumers, pythonConsumers = idx.restubIncomingRefs(graphPath) } else { // Extraction is still unprefixed here. Build only the identity surface // needed by the same incoming frontier used by the structural batch. @@ -5192,11 +5193,12 @@ func (idx *Indexer) indexFile( } stage := &incrementalBatchStage{graphPath: graphPath, legacyPath: legacyPath, priorNodes: priorNodes, result: fresh} stages := []*incrementalBatchStage{stage} - legacyCarried, npmConsumers, cargoConsumers = restubIncomingRefsFromView(idx.graph, stages, loadIncrementalPriorView(idx.graph, stages)) + legacyCarried, npmConsumers, cargoConsumers, pythonConsumers = restubIncomingRefsFromView(idx.graph, stages, loadIncrementalPriorView(idx.graph, stages)) } if markerBatch != nil { markerBatch.npmConsumerFiles = appendUniqueSorted(markerBatch.npmConsumerFiles, npmConsumers...) markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, cargoConsumers...) + markerBatch.pythonConsumerFiles = appendUniqueSorted(markerBatch.pythonConsumerFiles, pythonConsumers...) } if len(legacyIDs) == 0 { idx.graph.EvictFile(graphPath) @@ -5959,10 +5961,10 @@ func (idx *Indexer) reresolveFileScopedRaw(filePath string) error { // agnostic: GetInEdges + ReindexEdges are the same Store primitives the // resolver uses, so this behaves identically on the in-memory and disk // stores. -func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string) { +func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string, []string) { nodes := idx.graph.GetFileNodes(graphPath) if len(nodes) == 0 { - return nil, nil + return nil, nil, nil } evicted := make(map[string]struct{}, len(nodes)) refIDs := make([]string, 0, len(nodes)) @@ -5976,14 +5978,15 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string) { } } if len(refIDs) == 0 { - return nil, nil + return nil, nil, nil } // Fetch the whole changed file's incoming adjacency in one bounded store // operation. On SQLite this is chunked under the parameter limit; more // importantly, it avoids one transaction/query per definition in the file. inEdges := idx.graph.GetInEdgesByNodeIDs(refIDs) npmConsumers := restubTrackedNpmImportsFromView(idx.graph, evicted, inEdges) - consumers := restubTrackedCargoImportsFromView(idx.graph, evicted, inEdges, false) + cargoConsumers := restubTrackedCargoImportsFromView(idx.graph, evicted, inEdges, false) + pythonConsumers := restubTrackedPythonImportsFromView(idx.graph, evicted, inEdges, false) var batch []graph.EdgeReindex for _, n := range nodes { if n == nil || n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) && n.Kind != graph.KindFile { @@ -6014,13 +6017,16 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string) { if original, _ := e.Meta[resolver.CargoCallStubMeta].(string); e.Kind == graph.EdgeCalls && e.CrossRepo && original != "" { e.To, e.CrossRepo, e.ConfidenceLabel = original, false, "" } + if original, _ := e.Meta[resolver.PythonCallStubMeta].(string); e.Kind == graph.EdgeCalls && original != "" { + e.To, e.CrossRepo, e.ConfidenceLabel = original, false, "" + } batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) } } if len(batch) > 0 { idx.graph.ReindexEdges(batch) } - return npmConsumers, consumers + return npmConsumers, cargoConsumers, pythonConsumers } // embeddingDimsOrDefault returns the embedder's reported vector width, @@ -7550,6 +7556,7 @@ func (idx *Indexer) incrementalReindexPathsMode( result.CrossRepoResolveFiles = appendUniqueSorted(nil, markerBatch.npmConsumerFiles...) result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, markerBatch.cargoConsumerFiles...) result.CargoRetarget = markerBatch.cargoAddedFile + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, markerBatch.pythonConsumerFiles...) for _, path := range failedFiles { result.Errors = append(result.Errors, IndexError{FilePath: path, Error: idx.fileIndexFailureError(path).Error()}) } diff --git a/internal/indexer/legacy_file_identity_test.go b/internal/indexer/legacy_file_identity_test.go index f0fbc5182..0389142b1 100644 --- a/internal/indexer/legacy_file_identity_test.go +++ b/internal/indexer/legacy_file_identity_test.go @@ -230,7 +230,7 @@ func TestLegacyIdentityRepairRebindsForeignCaller(t *testing.T) { prior, ids, native := mergeLegacyFilePrior(legacy, fresh, old) stage := &incrementalBatchStage{graphPath: path, legacyPath: native, legacyIDs: ids, priorNodes: prior, result: &parser.ExtractionResult{Nodes: fresh}} stages := []*incrementalBatchStage{stage} - carried, _, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) + carried, _, _, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) idx := New(store, parser.NewRegistry(), config.Default().Index, zap.NewNop()) idx.SetRepoPrefix("repo") idx.evictFileIdentities([]string{path}, ids, []string{native}) @@ -308,7 +308,7 @@ func TestLegacyIdentityRepairRespectsDataflowFrontier(t *testing.T) { Nodes: fresh, Edges: []*graph.Edge{{From: fresh[2].ID, To: fresh[1].ID, Kind: graph.EdgeParamOf, FilePath: path}}, }} stages := []*incrementalBatchStage{stage} - carried, _, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) + carried, _, _, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) store.(graph.FileIdentityBatchEvicter).EvictFilesWithNodeIDs([]string{path}, ids) store.AddBatch(fresh, append(stage.result.Edges, carried...)) find := func(from, to string, kind graph.EdgeKind) bool { diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 59ac81798..8b0778bed 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -2790,6 +2790,7 @@ func (mi *MultiIndexer) indexRepoRaw(repoPrefix string) (*IndexResult, error) { mi.logger.Warn("post-publish Go module follow-up", zap.Error(err)) } result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(repoPrefix, false)...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.pythonConsumerFilesForRepo(repoPrefix)...) mi.runCrossRepoResolveFiles(result.CrossRepoResolveFiles) mi.ReconcileContractEdges() @@ -3438,6 +3439,7 @@ func (mi *MultiIndexer) trackPreparedRepoSourceCtx( mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix)) if !batchMode.deferResolve { mi.runCrossRepoResolveFiles(mi.cargoConsumerFilesForRepo(prefix, false)) + mi.runCrossRepoResolveFiles(mi.pythonConsumerFilesForRepo(prefix)) } // Add to global config. @@ -3750,6 +3752,7 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE } } else if result.FullRetrack { result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(prefix, false)...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.pythonConsumerFilesForRepo(prefix)...) if !batchMode.deferResolve { mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix)) mi.runCrossRepoResolveFiles(result.CrossRepoResolveFiles) diff --git a/internal/indexer/npm_lifecycle_test.go b/internal/indexer/npm_lifecycle_test.go index 97e4b4fd6..0122f9abf 100644 --- a/internal/indexer/npm_lifecycle_test.go +++ b/internal/indexer/npm_lifecycle_test.go @@ -38,12 +38,12 @@ func TestTrackedNpmImportsSurviveEviction(t *testing.T) { switch mode { case "file": idx := New(g, newTestRegistry(), config.IndexConfig{}, zap.NewNop()) - consumers, _ := idx.restubIncomingRefs("lib/index.ts") + consumers, _, _ := idx.restubIncomingRefs("lib/index.ts") assert.Equal(t, []string{"app/a.ts"}, consumers) g.EvictFile("lib/index.ts") case "batch": stages := []*incrementalBatchStage{{graphPath: "lib/index.ts", priorNodes: nodes}} - _, consumers, _ := restubIncomingRefsFromView(g, stages, loadIncrementalPriorView(g, stages)) + _, consumers, _, _ := restubIncomingRefsFromView(g, stages, loadIncrementalPriorView(g, stages)) assert.Equal(t, []string{"app/a.ts"}, consumers) g.EvictFile("lib/index.ts") case "replace": diff --git a/internal/indexer/python_import_lifecycle_test.go b/internal/indexer/python_import_lifecycle_test.go new file mode 100644 index 000000000..5e18215bb --- /dev/null +++ b/internal/indexer/python_import_lifecycle_test.go @@ -0,0 +1,289 @@ +package indexer + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedPythonImportLifecycle(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track", "deferred_track"} { + if operation == "deferred_track" && providerFirst { + continue // This case starts with an already tracked consumer. + } + order := "consumer_first" + if providerFirst { + order = "provider_first" + } + t.Run(backend+"/"+order+"/"+operation, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider) + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + consumerSource := pythonLifecycleConsumer + if operation == "provider_gains_new_module" { + consumerSource += "\nfrom PIL.extra import later\ndef after(): return later()\n" + } + writeWorkspaceFile(t, consumer, "app.py", consumerSource) + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + if providerFirst { + gc.Repos = []config.RepoEntry{entries[1], entries[0]} + } + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + var g graph.Store = graph.New() + if backend == "sqlite" { + disk, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, disk.Close()) }) + g = disk + } + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + core, logs := observer.New(zap.InfoLevel) + logger := zap.New(core) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + if operation == "deferred_track" { + _, err = mi.TrackRepoCtx(context.Background(), entries[0]) + require.NoError(t, err) + mi.BeginParallelBatch() + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + require.NoError(t, err) + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports { + require.False(t, edge.CrossRepo, "deferred track must not resolve shared imports early") + } + } + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/app.py", "provider/src/PIL/image.py", "provider/src/PIL/util.py"}, nil)) + mi.EndBatch() + } else if operation == "track" { + for _, entry := range gc.Repos { + _, err = mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + } else { + _, err = mi.IndexAll() + require.NoError(t, err) + } + assertImport := func() { + t.Helper() + var imports []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports && edge.Line <= 4 { + imports = append(imports, edge) + } + } + require.Len(t, imports, 4) + for _, edge := range imports { + require.True(t, edge.CrossRepo, "%#v", edge) + } + calls := make(map[string]int) + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls { + require.True(t, edge.CrossRepo, "%#v", edge) + require.NotEmpty(t, edge.Meta["python_call_stub"], "%#v", edge) + require.NotContains(t, edge.Meta, "external_call") + calls[edge.To]++ + } + } + require.Equal(t, map[string]int{ + "provider/src/PIL/image.py::Image": 1, + "provider/src/PIL/image.py::Image.show": 1, + "provider/src/PIL/image.py::f": 2, + "provider/src/PIL/util.py::helper": 1, + }, calls) + } + assertImport() + logs.TakeAll() + switch operation { + case "consumer_save": + writeWorkspaceFile(t, consumer, "app.py", pythonLifecycleConsumer+"def edited(): pass\n") + _, err = mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "app.py")}) + case "provider_save": + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider+"def added(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/image.py")}) + case "provider_save_util": + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\ndef added(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + case "provider_save_lib_contract": + writeWorkspaceFile(t, provider, "src/PIL/image.py", strings.Replace(pythonLifecycleProvider, "def f(): pass", "def f(x): pass", 1)) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/image.py")}) + case "provider_delete_util_batched": + require.NoError(t, os.Remove(filepath.Join(provider, "src/PIL/util.py"))) + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider+"def added(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py"), filepath.Join(provider, "src/PIL/image.py")}) + case "provider_gains_new_module": + writeWorkspaceFile(t, provider, "src/PIL/extra.py", "def later(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/extra.py")}) + case "delete_restore_util", "consumer_save_then_restore": + if operation == "consumer_save_then_restore" { + writeWorkspaceFile(t, consumer, "app.py", pythonLifecycleConsumer+"def edited(): pass\n") + _, err = mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "app.py")}) + require.NoError(t, err) + assertImport() + } + require.NoError(t, os.Remove(filepath.Join(provider, "src/PIL/util.py"))) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + require.NoError(t, err) + var parked []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 10 { + parked = append(parked, edge) + } + } + require.Len(t, parked, 1) + require.Equal(t, "unresolved::extern::PIL.util.helper::helper", parked[0].To) + require.False(t, parked[0].CrossRepo) + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + case "forced_evict_util": + require.NoError(t, os.Remove(filepath.Join(provider, "src/PIL/util.py"))) + mi.GetIndexer("provider").EvictFile(filepath.Join(provider, "src/PIL/util.py")) + case "consumer_full": + _, err = mi.IndexRepo("consumer") + case "provider_full": + var result *IndexResult + result, err = mi.IndexRepo("provider") + require.NoError(t, err) + data, marshalErr := json.Marshal(result) + require.NoError(t, marshalErr) + var output struct { + Files []string `json:"cross_repo_resolve_files"` + } + require.NoError(t, json.Unmarshal(data, &output)) + require.Contains(t, output.Files, "consumer/app.py") + case "warm_full_retrack": + prior := map[string]map[string]int64{"consumer": mi.FileMtimes("consumer"), "provider": mi.FileMtimes("provider")} + restarted := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, restarted.Close(context.Background())) }) + restarted.BeginBatch() + var output struct { + Files []string `json:"cross_repo_resolve_files"` + } + for _, entry := range gc.Repos { + force := "0" + if entry.Name == "provider" { + force = "1" + } + t.Setenv("GORTEX_WARMUP_FULL_RETRACK", force) + result, restoreErr := restarted.ReconcileRepoCtx(context.Background(), entry, prior[entry.Name]) + require.NoError(t, restoreErr) + if entry.Name == "provider" { + require.True(t, result.FullRetrack) + } + data, marshalErr := json.Marshal(result) + require.NoError(t, marshalErr) + var current struct { + Files []string `json:"cross_repo_resolve_files"` + } + require.NoError(t, json.Unmarshal(data, ¤t)) + output.Files = appendUniqueSorted(output.Files, current.Files...) + if entry.Name == "provider" { + require.NotContains(t, result.DerivedInvalidation.Files, "consumer/app.py") + } + } + if backend == "sqlite" { + require.Contains(t, output.Files, "consumer/app.py") + } else { + require.Empty(t, output.Files, "memory replacement preserved the existing import") + } + err = restarted.RunPreEnrichResolveFiles(context.Background(), output.Files, nil) + restarted.EndBatch() + case "no_change": + result, noChangeErr := mi.IncrementalReindexRepo("provider", nil) + require.NoError(t, noChangeErr) + require.Zero(t, result.StaleFileCount) + case "untrack_retrack": + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") + require.NoError(t, err) + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + } + require.NoError(t, err) + missingUtil := operation == "provider_delete_util_batched" || operation == "forced_evict_util" + if missingUtil { + var missingImport, missingCall []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports && edge.Line == 4 { + missingImport = append(missingImport, edge) + } + } + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 10 { + missingCall = append(missingCall, edge) + } + } + require.Len(t, missingImport, 1) + require.Equal(t, "module::pypi:PIL", missingImport[0].To) + require.Equal(t, "PIL.util", missingImport[0].Meta["python_import"]) + require.False(t, missingImport[0].CrossRepo) + require.Len(t, missingCall, 1) + require.Equal(t, "unresolved::extern::PIL.util.helper::helper", missingCall[0].To) + require.Equal(t, missingCall[0].To, missingCall[0].Meta["python_call_stub"]) + require.False(t, missingCall[0].CrossRepo) + } + var cold graph.Store = graph.New() + if backend == "sqlite" { + db, openErr := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, openErr) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + cold = db + } + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, pythonLifecycleBindings(cold, missingUtil), pythonLifecycleBindings(g, missingUtil), "after %s", operation) + require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) + require.Empty(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveAll").All()) + }) + } + } + } +} + +const pythonLifecycleProvider = "class Image:\n def show(self): pass\ndef f(): pass\n" +const pythonLifecycleConsumer = "from PIL.image import Image\nfrom PIL.image import f\nimport PIL.image as image\nfrom PIL.util import helper\ndef run():\n x = Image()\n x.show()\n f()\n image.f()\n helper()\n" + +func pythonLifecycleBindings(g graph.Store, missingUtil bool) []string { + var bindings []string + for _, node := range g.GetFileNodes("consumer/app.py") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind == graph.EdgeImports || edge.Kind == graph.EdgeCalls { + if missingUtil && (edge.Kind == graph.EdgeImports && edge.Line == 4 || edge.Kind == graph.EdgeCalls && edge.Line == 10) { + continue // These exact parked sites are asserted in the deletion case. + } + var stub any + if edge.Kind == graph.EdgeCalls { + stub = edge.Meta["python_call_stub"] + } + bindings = append(bindings, fmt.Sprintf("%s -%s-> %s cross=%v line=%d stub=%v", edge.From, edge.Kind, edge.To, edge.CrossRepo, edge.Line, stub)) + } + } + } + slices.Sort(bindings) + return bindings +} diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go new file mode 100644 index 000000000..b24f2dea6 --- /dev/null +++ b/internal/indexer/python_import_restub.go @@ -0,0 +1,113 @@ +package indexer + +import ( + "strings" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +// Preserve importer-owned facts before their provider's files disappear. +func restubTrackedPythonRepoIncoming(g graph.Store, repo string) []string { + if repo == "" { + return nil + } + var files, ids []string + flush := func() { + incoming := g.GetInEdgesByNodeIDs(ids) + evicted := make(map[string]struct{}) + for _, edges := range incoming { + for _, edge := range edges { + if edge != nil && graph.RepoPrefixOfID(edge.From) == repo { + evicted[edge.From] = struct{}{} + } + } + } + files = appendUniqueSorted(files, restubTrackedPythonImportsFromView(g, evicted, incoming, true)...) + ids = ids[:0] + } + for node := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFile, graph.KindFunction, graph.KindMethod, graph.KindType) { + file, _, _ := strings.Cut(node.ID, "::") + if !strings.HasSuffix(file, ".py") { + continue + } + ids = append(ids, node.ID) + if len(ids) == 512 { + flush() + } + } + if len(ids) > 0 { + flush() + } + return files +} + +func restubTrackedPythonImportsFromView(g graph.Store, evicted map[string]struct{}, incoming map[string][]*graph.Edge, parkCalls bool) []string { + consumers := make(map[string]struct{}) + var batch []graph.EdgeReindex + for id, edges := range incoming { + if _, changing := evicted[id]; !parkCalls && !changing { + continue + } + for _, edge := range edges { + if edge == nil || !edge.CrossRepo { + continue + } + key := resolver.PythonImportStubMeta + if edge.Kind == graph.EdgeCalls { + key = resolver.PythonCallStubMeta + } else if edge.Kind != graph.EdgeImports { + continue + } + stub, _ := edge.Meta[key].(string) + if _, gone := evicted[edge.From]; stub == "" || gone || edge.To == stub { + continue + } + consumers[edge.FilePath] = struct{}{} + // Save paths keep the existing carry/restub decision and its dataflow sites. + if edge.Kind == graph.EdgeCalls && !parkCalls { + continue + } + old := edge.To + if edge.Kind == graph.EdgeCalls { + graph.StashRestubProvenance(edge) + edge.ConfidenceLabel = "" + } + edge.To, edge.CrossRepo = stub, false + batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: old}) + } + } + if len(batch) > 0 { + g.ReindexEdges(batch) + } + return sortedStringKeys(consumers) +} + +// Retry only parked imports claimed by this provider, or owned by this repo. +func (mi *MultiIndexer) pythonConsumerFilesForRepo(repo string) []string { + lookup := mi.pythonPackageLookup() + if lookup == nil { + return nil + } + files := make(map[string]struct{}) + for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { + if !strings.HasSuffix(edge.FilePath, ".py") { + continue + } + top := "" + for _, prefix := range []string{"module::pypi:", "external::", "unresolved::import::"} { + if spec, ok := strings.CutPrefix(edge.To, prefix); ok { + top, _, _ = strings.Cut(spec, ".") + break + } + } + caller := graph.RepoPrefixOfID(edge.From) + for provider := range lookup.Providers(caller, top) { + if provider == repo || caller == repo { + files[edge.FilePath] = struct{}{} + break + } + } + } + return sortedStringKeys(files) +} diff --git a/internal/indexer/python_import_restub_test.go b/internal/indexer/python_import_restub_test.go new file mode 100644 index 000000000..404c7c674 --- /dev/null +++ b/internal/indexer/python_import_restub_test.go @@ -0,0 +1,60 @@ +package indexer + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +func TestPythonRestubLeavesUnmarkedImportsUnchanged(t *testing.T) { + for _, file := range []string{"provider/module.py", "provider/main.go"} { + t.Run(file, func(t *testing.T) { + g := graph.New() + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "provider"}) + edge := &graph.Edge{From: "consumer/source", FilePath: "consumer/source", To: file, Kind: graph.EdgeImports, CrossRepo: true} + if file == "provider/main.go" { + edge.Meta = map[string]any{resolver.PythonImportStubMeta: "external::PIL.image"} + } + g.AddEdge(edge) + require.Empty(t, restubTrackedPythonRepoIncoming(g, "provider")) + require.Equal(t, file, edge.To) + require.True(t, edge.CrossRepo) + }) + } +} + +func TestPythonRestubKeepsExactImportAndCallStubs(t *testing.T) { + g := graph.New() + file, caller, target := "consumer/app.py", "consumer/app.py::run", "provider/src/PIL/image.py" + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "consumer"}) + g.AddNode(&graph.Node{ID: caller, Kind: graph.KindFunction, FilePath: file, RepoPrefix: "consumer"}) + g.AddNode(&graph.Node{ID: target, Kind: graph.KindFile, FilePath: target, RepoPrefix: "provider"}) + importEdge := &graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{ + resolver.PythonImportStubMeta: "module::pypi:PIL", "python_import": "PIL.image", + }} + g.AddNode(&graph.Node{ID: target + "::f", Kind: graph.KindFunction, FilePath: target, RepoPrefix: "provider"}) + call := &graph.Edge{From: caller, FilePath: file, To: target + "::f", Kind: graph.EdgeCalls, CrossRepo: true, + Meta: map[string]any{resolver.PythonCallStubMeta: "unresolved::extern::PIL.image::f"}} + g.AddEdge(importEdge) + g.AddEdge(call) + require.Equal(t, []string{file}, restubTrackedPythonRepoIncoming(g, "provider")) + require.Equal(t, "module::pypi:PIL", importEdge.To) + require.Equal(t, "PIL.image", importEdge.Meta["python_import"]) + require.False(t, importEdge.CrossRepo) + require.Equal(t, "unresolved::extern::PIL.image::f", call.To) + require.False(t, call.CrossRepo) +} + +func TestPythonRestubIgnoresMetadataOnlyTargets(t *testing.T) { + g := graph.New() + target := "provider/src/PIL/image.py" + edge := &graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: target, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{resolver.PythonImportStubMeta: "module::pypi:PIL"}} + g.AddEdge(edge) + files := restubTrackedPythonImportsFromView(g, map[string]struct{}{"provider/src/PIL/changed.py": {}}, map[string][]*graph.Edge{target: {edge}}, false) + require.Empty(t, files) + require.Equal(t, target, edge.To) + require.True(t, edge.CrossRepo) +} diff --git a/internal/indexer/python_import_reuse_test.go b/internal/indexer/python_import_reuse_test.go new file mode 100644 index 000000000..091f3448e --- /dev/null +++ b/internal/indexer/python_import_reuse_test.go @@ -0,0 +1,25 @@ +package indexer + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" +) + +func TestPythonReuseKeepsFreshCallStubWithoutConfirmation(t *testing.T) { + prior := &graph.Edge{Meta: map[string]any{ + "confirmed_from_origin": "ast", + "semantic_source": "python-lsp", + "python_call_stub": "unresolved::extern::PIL.image.f::f", + }} + captured := reuseSemanticMeta(prior) + values := map[reuseKey]*reuseVal{{}: {semanticMeta: captured}} + stripReuseSemanticMeta(values) + require.Equal(t, map[string]any{"python_call_stub": "unresolved::extern::PIL.image.f::f"}, captured) + fresh := &graph.Edge{To: "provider/PIL/image.py::f"} + applyReuseSemanticMeta(fresh, captured, "unresolved::extern::PIL.image::f") + require.Equal(t, "unresolved::extern::PIL.image::f", fresh.Meta["python_call_stub"]) + require.NotContains(t, fresh.Meta, "semantic_source") + require.NotContains(t, fresh.Meta, "confirmed_from_origin") +} diff --git a/internal/indexer/repo_eviction_scope.go b/internal/indexer/repo_eviction_scope.go index 131a6f5c3..a181486a5 100644 --- a/internal/indexer/repo_eviction_scope.go +++ b/internal/indexer/repo_eviction_scope.go @@ -12,6 +12,7 @@ import ( func evictRepoCurrentGeneration(target graph.Store, repoPrefix string) (nodesRemoved, edgesRemoved int) { restubTrackedNpmRepoIncoming(target, repoPrefix) restubTrackedCargoRepoIncoming(target, repoPrefix) + restubTrackedPythonRepoIncoming(target, repoPrefix) if scoped, ok := target.(graph.CurrentGenerationRepoEvicter); ok { return scoped.EvictRepoCurrentGeneration(repoPrefix) } diff --git a/internal/indexer/repository_untrack.go b/internal/indexer/repository_untrack.go index 9622f74d5..12835a506 100644 --- a/internal/indexer/repository_untrack.go +++ b/internal/indexer/repository_untrack.go @@ -292,6 +292,7 @@ func (mi *MultiIndexer) purgeRepositoryPayload( restubTrackedNpmRepoIncoming(mi.graph, repoPrefix) parked := restubGoModuleRepoIncoming(mi.graph, repoPrefix) restubTrackedCargoRepoIncoming(mi.graph, repoPrefix) + restubTrackedPythonRepoIncoming(mi.graph, repoPrefix) if purger, ok := mi.graph.(interface{ PurgeRepo(string) error }); ok { if err := purger.PurgeRepo(repoPrefix); err != nil { return 0, 0, fmt.Errorf("purge repository payload: %w", err) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 3ec0f5686..d31efefe5 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -435,7 +435,7 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS if err := cr.resolveCargoImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { return stats, err } - if err := cr.resolveTrackedPythonImports(ctx, cr.trackedPythonImportEdges(), stats); err != nil { + if _, err := cr.resolveTrackedPythonImports(ctx, cr.trackedPythonImportEdges(), stats, false); err != nil { return stats, err } // Share the master resolver's stable high-water/keyset stream. The cold @@ -724,7 +724,19 @@ func (cr *CrossRepoResolver) resolveScopedLocked(edges []*graph.Edge) *CrossRepo stats := &CrossRepoStats{ByRepo: make(map[string]int)} _ = cr.resolveTrackedNpmImports(context.Background(), slices.Values(edges), stats) _ = cr.resolveCargoImports(context.Background(), slices.Values(edges), stats) - _ = cr.resolveTrackedPythonImports(context.Background(), slices.Values(edges), stats) + reopened, _ := cr.resolveTrackedPythonImports(context.Background(), slices.Values(edges), stats, true) + if len(reopened) > 0 { + seen := make(map[*graph.Edge]bool, len(edges)) + for _, edge := range edges { + seen[edge] = true + } + for _, edge := range reopened { + if !seen[edge] { + edges = append(edges, edge) + seen[edge] = true + } + } + } pending := make([]*graph.Edge, 0, len(edges)) for _, edge := range edges { if edge != nil && graph.IsUnresolvedTarget(edge.To) { @@ -1270,6 +1282,30 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b cr.resolveImport(e, strings.TrimPrefix(target, "import::"), stats) case strings.HasPrefix(target, "*."): cr.resolveMethodCall(e, strings.TrimPrefix(target, "*."), stats) + case strings.HasPrefix(target, "extern::") && e.Kind == graph.EdgeCalls && cr.pythonPackages != nil && e.Meta[PythonCallStubMeta] == oldTo: + module, symbol, _ := strings.Cut(strings.TrimPrefix(target, "extern::"), "::") + callerRepo, callerWS := cr.callerRepoPrefix(e), cr.callerWorkspaceID(e) + reachable := cr.reachabilityChecker(e) + importedDirs := cr.importedDirsByFile[cr.callerFileID(e)] + find := func(name string) ([]*graph.Node, error) { + var eligible []*graph.Node + for _, node := range cr.scopedCandidates(e, name) { + _, imported := importedDirs[filePathDir(node.FilePath)] + if imported && reachable(node.RepoPrefix) && cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(node), module) { + eligible = append(eligible, node) + } + } + return eligible, nil + } + if resolvePythonModuleExtern(e, module, symbol, callerRepo, &ResolveStats{}, find) { + stats.Resolved++ + if e.CrossRepo { + stats.CrossRepoEdges++ + stats.ByRepo[graph.RepoPrefixOfID(e.To)]++ + } + } else { + stats.Unresolved++ + } case e.Kind == graph.EdgeExtends || e.Kind == graph.EdgeImplements || e.Kind == graph.EdgeComposes: // Type-hierarchy edges never resolve to a function/method. // CrossRepoResolver has no type-only resolution path, and a @@ -1297,6 +1333,14 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b } e.Meta[CargoCallStubMeta] = oldTo } + if e.Kind == graph.EdgeCalls && e.CrossRepo && cr.pythonPackages != nil && isPythonSourcePath(e.FilePath) { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + if original, _ := e.Meta[PythonCallStubMeta].(string); original == "" { + e.Meta[PythonCallStubMeta] = oldTo + } + } *batch = append(*batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) cr.noteRetargetedCall(e) } diff --git a/internal/resolver/python_absolute_imports.go b/internal/resolver/python_absolute_imports.go index 2767aa2da..3793299ec 100644 --- a/internal/resolver/python_absolute_imports.go +++ b/internal/resolver/python_absolute_imports.go @@ -66,6 +66,12 @@ func pythonModuleFileMatches(filePath, module string) bool { // ambiguous layout keeps the previous behaviour. Methods never match: an // import binds module attributes, not class members. func (r *Resolver) resolvePythonModuleExtern(e *graph.Edge, importPath, symbol, callerRepo string, stats *ResolveStats) bool { + return resolvePythonModuleExtern(e, importPath, symbol, callerRepo, stats, func(name string) ([]*graph.Node, error) { + return r.cachedFindExternNodesByName(name, e) + }) +} + +func resolvePythonModuleExtern(e *graph.Edge, importPath, symbol, callerRepo string, stats *ResolveStats, find func(string) ([]*graph.Node, error)) bool { if !isPythonSourcePath(e.FilePath) || !isDottedPythonModule(importPath) { return false } @@ -75,7 +81,7 @@ func (r *Resolver) resolvePythonModuleExtern(e *graph.Edge, importPath, symbol, readings = append(readings, reading{importPath[:len(importPath)-len(name)-1], name}) } for _, rd := range readings { - candidates, err := r.cachedFindExternNodesByName(rd.name, e) + candidates, err := find(rd.name) if err != nil { return false } diff --git a/internal/resolver/python_import_call_retry_test.go b/internal/resolver/python_import_call_retry_test.go new file mode 100644 index 000000000..c7bcefa3b --- /dev/null +++ b/internal/resolver/python_import_call_retry_test.go @@ -0,0 +1,172 @@ +package resolver + +import ( + "context" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestTrackedPythonImportReopensOnlyNewlyLinkedCalls(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file := "consumer/app.py" + for _, path := range []string{file, "consumer/other.py", "outsider/app.py"} { + g.AddNode(&graph.Node{ID: path, Kind: graph.KindFile, FilePath: path}) + } + eligible := "external-call::dep::PIL.image.f" + for i, tc := range []struct { + file, target string + kind graph.EdgeKind + }{ + {file, eligible, graph.EdgeCalls}, + {file, "external-call::dep::other.f", graph.EdgeCalls}, + {file, "external-call::dep::PIL.image_extra.f", graph.EdgeCalls}, + {file, "provider/src/PIL/image.py::f", graph.EdgeCalls}, + {file, eligible, graph.EdgeReferences}, + {"consumer/other.py", eligible, graph.EdgeCalls}, + {"outsider/app.py", eligible, graph.EdgeCalls}, + } { + g.AddEdge(&graph.Edge{From: tc.file, FilePath: tc.file, To: tc.target, Kind: tc.kind, Line: i + 1, Meta: map[string]any{"external_call": true, PythonCallStubMeta: "unresolved::extern::PIL.image::f"}}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: eligible, Kind: graph.EdgeCalls, Line: 8}) + linked := &graph.Edge{From: file, FilePath: file, To: "provider/src/PIL/image.py", Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{PythonImportStubMeta: "module::pypi:PIL", "python_import": "PIL.image"}} + unmarked := &graph.Edge{From: "consumer/other.py", FilePath: "consumer/other.py", To: linked.To, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{"python_import": "PIL.image"}} + unlinked := &graph.Edge{From: "outsider/app.py", FilePath: "outsider/app.py", To: "module::pypi:PIL", Kind: graph.EdgeImports, + Meta: linked.Meta} + cr := NewCrossRepo(g) + reopened := cr.restubCallsForLinkedPythonImports([]graph.EdgeReindex{{Edge: linked}, {Edge: unmarked}, {Edge: unlinked}}) + require.Len(t, reopened, 1) + require.Equal(t, file, reopened[0].FilePath) + require.Equal(t, "unresolved::extern::PIL.image::f", reopened[0].To) + require.NotContains(t, reopened[0].Meta, "external_call") + for _, edges := range g.GetOutEdgesByNodeIDs([]string{file, "consumer/other.py", "outsider/app.py"}) { + for _, edge := range edges { + if edge.Line != 1 { + require.NotEqual(t, "unresolved::extern::PIL.image::f", edge.To) + } + } + } + }) + } +} + +func TestTrackedPythonImportFullPassStreamsReopenedCalls(t *testing.T) { + g := graph.New() + for _, file := range []string{"consumer/app.py", "provider/PIL/__init__.py", "provider/PIL/image.py"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), WorkspaceID: "shared"}) + } + importEdge := &graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeImports, To: "external::PIL.image", Line: 1} + g.AddEdge(importEdge) + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeCalls, To: "external-call::dep::PIL.image.f", Line: 2, Meta: map[string]any{PythonCallStubMeta: "unresolved::extern::PIL.image::f"}}) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + return &PythonPackageLookup{HasTop: func(top string) bool { return top == "PIL" }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", RepoPrefix: "provider", WorkspaceID: "shared"}}} + }} + }) + calls, err := cr.resolveTrackedPythonImports(context.Background(), slices.Values([]*graph.Edge{importEdge}), &CrossRepoStats{ByRepo: map[string]int{}}, false) + require.NoError(t, err) + require.Empty(t, calls, "the full pass must not retain call metadata across import pages") + out := outEdgesOfKind(g, "consumer/app.py", graph.EdgeCalls) + require.Len(t, out, 1) + require.Equal(t, "unresolved::extern::PIL.image::f", out[0].To, "the full pass's pending stream will admit this reopened call") +} + +func TestTrackedPythonModuleCallKeepsImportAndWorkspaceEvidence(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scenario := range []string{"module", "from_alias", "unreachable", "foreign_workspace", "wrong_module", "method", "other_provider"} { + t.Run(backend+"/"+scenario, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file, providerFile := "consumer/app.py", "provider/PIL/image.py" + workspace := "shared" + if scenario == "foreign_workspace" { + workspace = "foreign" + } + for _, node := range []*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared", Language: "python"}, + {ID: file + "::run", FilePath: file, Name: "run", Kind: graph.KindFunction, RepoPrefix: "consumer", WorkspaceID: "shared", Language: "python"}, + {ID: providerFile, FilePath: providerFile, Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: workspace, Language: "python"}, + } { + g.AddNode(node) + } + if scenario != "unreachable" { + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeImports, To: providerFile, CrossRepo: true, Line: 1, + Meta: map[string]any{PythonImportStubMeta: "external::PIL.image", "python_import": "PIL.image"}}) + } + candidate := &graph.Node{ID: providerFile + "::f", FilePath: providerFile, Name: "f", Kind: graph.KindFunction, RepoPrefix: "provider", WorkspaceID: workspace, Language: "python"} + switch scenario { + case "wrong_module": + candidate.FilePath = "provider/PIL/unrelated.py" + candidate.ID = candidate.FilePath + "::f" + case "method": + candidate.Kind = graph.KindMethod + case "other_provider": + candidate.FilePath, candidate.RepoPrefix = "other/PIL/image.py", "other" + candidate.ID = candidate.FilePath + "::f" + g.AddNode(&graph.Node{ID: "other/unrelated.py", FilePath: "other/unrelated.py", Kind: graph.KindFile, RepoPrefix: "other", WorkspaceID: "shared"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeImports, To: "other/unrelated.py", Line: 2, CrossRepo: true}) + } + g.AddNode(candidate) + original := "unresolved::extern::PIL.image::f" + if scenario == "from_alias" { + original = "unresolved::extern::PIL.image.f::g" + } + g.AddEdge(&graph.Edge{From: file + "::run", FilePath: file, Kind: graph.EdgeCalls, To: original, Line: 3, + Meta: map[string]any{PythonCallStubMeta: original}}) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { return nil }) + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { return nil }) + cr.ResolveForFile("consumer", "app.py") + calls := outEdgesOfKind(g, file+"::run", graph.EdgeCalls) + require.Len(t, calls, 1) + if scenario == "module" || scenario == "from_alias" { + require.Equal(t, candidate.ID, calls[0].To) + require.True(t, calls[0].CrossRepo) + } else { + require.Equal(t, original, calls[0].To) + require.False(t, calls[0].CrossRepo) + } + }) + } + } +} + +func TestTrackedPythonScopedRetryVisitsEachCallOnce(t *testing.T) { + g := graph.New() + for _, file := range []string{"consumer/app.py", "provider/PIL/__init__.py", "provider/PIL/image.py"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), WorkspaceID: "shared"}) + } + g.AddNode(&graph.Node{ID: "provider/PIL/image.py::f", FilePath: "provider/PIL/image.py", Name: "f", Kind: graph.KindFunction, RepoPrefix: "provider", WorkspaceID: "shared"}) + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeImports, To: "external::PIL.image", Line: 1}) + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeCalls, To: "external-call::dep::PIL.image.f", Line: 2, + Meta: map[string]any{PythonCallStubMeta: "unresolved::extern::PIL.image::f"}}) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + return &PythonPackageLookup{HasTop: func(top string) bool { return top == "PIL" }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", RepoPrefix: "provider", WorkspaceID: "shared"}}} + }} + }) + stats := cr.ResolveForFile("consumer", "app.py") + require.Equal(t, 2, stats.Resolved) + require.Zero(t, stats.Unresolved) +} diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go index 2b2223bc3..fdab01dcc 100644 --- a/internal/resolver/python_tracked_import.go +++ b/internal/resolver/python_tracked_import.go @@ -9,6 +9,12 @@ import ( "github.com/zzet/gortex/internal/graph" ) +// PythonImportStubMeta preserves the import's exact target before a tracked bind. +const PythonImportStubMeta = "python_import_stub" + +// PythonCallStubMeta retains the exact function, method or module-qualified call target. +const PythonCallStubMeta = "python_call_stub" + // PythonPackageLookup exposes only indexed tops from declared tracked // distributions. The indexer owns manifest I/O and consumer-local checks. type PythonPackageLookup struct { @@ -123,10 +129,11 @@ func (cr *CrossRepoResolver) trackedPythonImportEdges() iter.Seq[*graph.Edge] { return slices.Values(edges) } -// Retry only import/file edges; never reconsider working links. -func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats) error { +// Retry only import/file edges; never reconsider working links. Full passes +// stream reopened calls later; scoped passes collect them for their pending list. +func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats, captureCalls bool) (reopened []*graph.Edge, err error) { if cr.pythonPackages == nil { - return ctx.Err() + return reopened, ctx.Err() } next, stop := iter.Pull(edges) defer stop() @@ -135,7 +142,7 @@ func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, ed page := make([]*graph.Edge, 0, resolvePendingPageRows) for len(page) < resolvePendingPageRows { if err := ctx.Err(); err != nil { - return err + return reopened, err } e, ok := next() if !ok { @@ -148,7 +155,7 @@ func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, ed if lookup == nil { lookup = cr.pythonPackages() if lookup == nil { - return ctx.Err() + return reopened, ctx.Err() } } if lookup.HasTop(top) { @@ -156,10 +163,10 @@ func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, ed } } if err := ctx.Err(); err != nil { - return err + return reopened, err } if len(page) == 0 { - return nil + return reopened, nil } nodes := map[string]*graph.Node{} sites := make([]graph.EdgeSite, 0, len(page)) @@ -193,6 +200,10 @@ func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, ed continue } old := e.To + if e.Meta == nil { + e.Meta = make(map[string]any) + } + e.Meta[PythonImportStubMeta], e.Meta["python_import"] = old, spec e.To, e.CrossRepo = match.ID, true batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: old}) projected.To, projected.CrossRepo = e.To, e.CrossRepo @@ -203,6 +214,56 @@ func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, ed } if len(batch) > 0 { cr.graph.ReindexEdges(batch) + calls := cr.restubCallsForLinkedPythonImports(batch) + if captureCalls { + reopened = append(reopened, calls...) + } + } + } +} + +// Reopen only synthesized calls backed by an import this bridge just linked. +func (cr *CrossRepoResolver) restubCallsForLinkedPythonImports(changes []graph.EdgeReindex) []*graph.Edge { + specs := make(map[string][]string) + var files []string + for _, change := range changes { + spec, _ := change.Edge.Meta["python_import"].(string) + if stub, _ := change.Edge.Meta[PythonImportStubMeta].(string); stub == "" || spec == "" || !change.Edge.CrossRepo { + continue + } + file := change.Edge.FilePath + if _, seen := specs[file]; !seen { + files = append(files, file) } + specs[file] = append(specs[file], spec) + } + if len(files) == 0 { + return nil + } + var reopened []*graph.Edge + var batch []graph.EdgeReindex + for row := range graph.EdgesInScopeSeq(cr.graph, nil, files, graph.EdgeCalls) { + call := row.Edge + if call == nil { + continue + } + for _, spec := range specs[call.FilePath] { + original, _ := call.Meta[PythonCallStubMeta].(string) + target := externalCallPrefix + "dep::" + spec + if original == "" || (call.To != target && !strings.HasPrefix(call.To, target+".")) { + continue + } + old := call.To + graph.StashRestubProvenance(call) + delete(call.Meta, "external_call") + call.To, call.CrossRepo, call.ConfidenceLabel = original, false, "" + reopened = append(reopened, call) + batch = append(batch, graph.EdgeReindex{Edge: call, OldTo: old}) + break + } + } + if len(batch) > 0 { + cr.graph.ReindexEdges(batch) } + return reopened } diff --git a/internal/resolver/python_tracked_import_test.go b/internal/resolver/python_tracked_import_test.go index a1f17b0a5..82c33c07e 100644 --- a/internal/resolver/python_tracked_import_test.go +++ b/internal/resolver/python_tracked_import_test.go @@ -38,11 +38,14 @@ func TestTrackedPythonImportRetry(t *testing.T) { require.Len(t, imports, 1) if target == "local.py" { require.Equal(t, target, imports[0].To) + require.NotContains(t, imports[0].Meta, PythonImportStubMeta) return } require.Equal(t, "provider/src/PIL/image.py", imports[0].To) require.True(t, imports[0].CrossRepo) require.Equal(t, "metadata", imports[0].Meta["keep"]) + require.Equal(t, target, imports[0].Meta[PythonImportStubMeta]) + require.Equal(t, "PIL.image", imports[0].Meta["python_import"]) }) } } @@ -118,7 +121,9 @@ func (s *pythonImportPageStore) GetNode(id string) *graph.Node { } func (s *pythonImportPageStore) GetNodesByIDs(ids []string) map[string]*graph.Node { - s.callerBatches = append(s.callerBatches, len(ids)) + if len(ids) > 0 { // Empty scoped-call projections perform no node hydration. + s.callerBatches = append(s.callerBatches, len(ids)) + } return s.Store.GetNodesByIDs(ids) } diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 7a34f7aba..34e402c20 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -4112,6 +4112,14 @@ func (r *Resolver) resolveEdge(e *graph.Edge, stats *ResolveStats) (oldTo string } } + if e.To != oldTo && e.Kind == graph.EdgeCalls && isPythonSourcePath(e.FilePath) { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + if original, _ := e.Meta[PythonCallStubMeta].(string); original == "" { + e.Meta[PythonCallStubMeta] = oldTo + } + } return oldTo, e.To != oldTo } From 2845f4f76222c39f99ee4a79b0367b44875ec187 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 03:47:55 -0700 Subject: [PATCH 022/111] fix(indexer): keep local Python calls on the native restub path --- internal/indexer/incremental_batch.go | 2 +- internal/indexer/indexer.go | 2 +- .../indexer/python_local_lifecycle_test.go | 88 +++++++++++++++++++ internal/resolver/resolver.go | 3 +- 4 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 internal/indexer/python_local_lifecycle_test.go diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index bf3b6be62..5f6ccc5a9 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -1560,7 +1560,7 @@ func restubIncomingRefsFromView( if original, _ := edge.Meta[resolver.CargoCallStubMeta].(string); edge.Kind == graph.EdgeCalls && edge.CrossRepo && original != "" { edge.To, edge.CrossRepo, edge.ConfidenceLabel = original, false, "" } - if original, _ := edge.Meta[resolver.PythonCallStubMeta].(string); edge.Kind == graph.EdgeCalls && original != "" { + if original, _ := edge.Meta[resolver.PythonCallStubMeta].(string); edge.Kind == graph.EdgeCalls && edge.CrossRepo && original != "" { edge.To, edge.CrossRepo, edge.ConfidenceLabel = original, false, "" } reindexes = append(reindexes, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 60cde9dd4..c0d3a6e73 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -6017,7 +6017,7 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string, [] if original, _ := e.Meta[resolver.CargoCallStubMeta].(string); e.Kind == graph.EdgeCalls && e.CrossRepo && original != "" { e.To, e.CrossRepo, e.ConfidenceLabel = original, false, "" } - if original, _ := e.Meta[resolver.PythonCallStubMeta].(string); e.Kind == graph.EdgeCalls && original != "" { + if original, _ := e.Meta[resolver.PythonCallStubMeta].(string); e.Kind == graph.EdgeCalls && e.CrossRepo && original != "" { e.To, e.CrossRepo, e.ConfidenceLabel = original, false, "" } batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) diff --git a/internal/indexer/python_local_lifecycle_test.go b/internal/indexer/python_local_lifecycle_test.go new file mode 100644 index 000000000..cbc1b6912 --- /dev/null +++ b/internal/indexer/python_local_lifecycle_test.go @@ -0,0 +1,88 @@ +package indexer + +import ( + "context" + "fmt" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestPythonLocalRenameAndRestore(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, repos := range []string{"solo", "duo"} { + t.Run(backend+"/"+repos, func(t *testing.T) { + dir := t.TempDir() + original := "def foo(): pass\nclass K:\n def m(self): pass\n" + writeWorkspaceFile(t, dir, "a.py", original) + writeWorkspaceFile(t, dir, "b.py", "from a import foo, K\nimport a\ndef run():\n foo()\n a.foo()\n k = K()\n k.m()\n") + entries := []config.RepoEntry{{Path: dir, Name: "solo", Workspace: "shared"}} + if repos == "duo" { + other := t.TempDir() + writeWorkspaceFile(t, other, "z.py", "def foo(): pass\n") + entries = append(entries, config.RepoEntry{Path: other, Name: "other", Workspace: "shared"}) + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + g := pythonLocalStore(t, backend) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + _, err = mi.IndexAll() + require.NoError(t, err) + before := pythonLocalBindings(g) + writeWorkspaceFile(t, dir, "a.py", "def foo2(): pass\nclass K:\n def m2(self): pass\n") + _, err = mi.IncrementalReindexRepo("solo", []string{filepath.Join(dir, "a.py")}) + require.NoError(t, err) + writeWorkspaceFile(t, dir, "a.py", original) + _, err = mi.IncrementalReindexRepo("solo", []string{filepath.Join(dir, "a.py")}) + require.NoError(t, err) + require.Equal(t, before, pythonLocalBindings(g)) + cold := pythonLocalStore(t, backend) + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, pythonLocalBindings(cold), pythonLocalBindings(g)) + }) + } + } +} + +func pythonLocalStore(t *testing.T, backend string) graph.Store { + t.Helper() + if backend == "memory" { + return graph.New() + } + disk, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, disk.Close()) }) + return disk +} + +func pythonLocalBindings(g graph.Store) []string { + var bindings []string + for _, node := range g.GetFileNodes("solo/b.py") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind == graph.EdgeImports || edge.Kind == graph.EdgeCalls { + bindings = append(bindings, fmt.Sprintf("%s -%s-> %s cross=%v line=%d stub=%v", edge.From, edge.Kind, edge.To, edge.CrossRepo, edge.Line, edge.Meta["python_call_stub"])) + } + } + } + slices.Sort(bindings) + return bindings +} diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 34e402c20..719735700 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -4112,7 +4112,8 @@ func (r *Resolver) resolveEdge(e *graph.Edge, stats *ResolveStats) (oldTo string } } - if e.To != oldTo && e.Kind == graph.EdgeCalls && isPythonSourcePath(e.FilePath) { + // External terminals need their spelling before a later tracked import retry. + if e.To != oldTo && e.Kind == graph.EdgeCalls && isPythonSourcePath(e.FilePath) && (e.CrossRepo || isExternalCandidateTarget(e.To)) { if e.Meta == nil { e.Meta = make(map[string]any) } From 4fb0247248eef3a075ac1bb222b8fa28a936fa48 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 05:04:14 -0700 Subject: [PATCH 023/111] fix(indexer): preserve Python provider discovery across warm restarts Read manifests, requirements and package layout through the owning indexer's manifest tree. Carry added-file importer candidates in CrossRepoResolveFiles so deferred and provider-first warmup can resolve after all repositories register. Keep the existing matcher and avoid consumer-frontier scans on ordinary saves. A file-only addition cannot trigger a whole-graph fallback. --- internal/indexer/file_meta.go | 1 + internal/indexer/incremental_batch.go | 3 + internal/indexer/incremental_watcher_batch.go | 12 +-- internal/indexer/multi.go | 3 + .../indexer/python_import_lifecycle_test.go | 28 ++++- internal/indexer/python_import_restub.go | 32 ++++++ .../python_import_warm_recovery_test.go | 102 ++++++++++++++++++ internal/indexer/python_tracked_import.go | 31 +++--- .../indexer/python_tracked_import_test.go | 75 +++++++++++++ 9 files changed, 259 insertions(+), 28 deletions(-) create mode 100644 internal/indexer/python_import_warm_recovery_test.go diff --git a/internal/indexer/file_meta.go b/internal/indexer/file_meta.go index 2bfeceec8..d49c0079a 100644 --- a/internal/indexer/file_meta.go +++ b/internal/indexer/file_meta.go @@ -200,6 +200,7 @@ type reparsePendingEnrichmentBatch struct { cargoAddedFile bool cargoConsumerFiles []string pythonConsumerFiles []string + pythonAddedFiles []string byFile map[string]bool deferResolverCatchup bool deferredAffectedFiles map[string]struct{} diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 5f6ccc5a9..1a2da839c 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -1084,6 +1084,9 @@ func (idx *Indexer) commitStructuralIncrementalBatch( var edges []*graph.Edge var priorPending []*graph.Edge for _, stage := range stages { + if markerBatch != nil && len(stage.priorNodes) == 0 && strings.HasSuffix(stage.graphPath, ".py") { + markerBatch.pythonAddedFiles = append(markerBatch.pythonAddedFiles, stage.graphPath) + } paths = append(paths, stage.graphPath) legacyIDs = append(legacyIDs, stage.legacyIDs...) if stage.legacyPath != "" { diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index 3b7cac7d1..009db2ce7 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -537,13 +537,8 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( batch.dropDeferredPriorBindings() } crossRepoFiles := appendUniqueSorted(nil, files...) - if needed { - for _, file := range files { - if strings.HasSuffix(file, ".py") { - crossRepoFiles = appendUniqueSorted(crossRepoFiles, mi.pythonConsumerFilesForRepo(repoPrefix)...) - break - } - } + if result != nil && batch != nil { + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, pythonConsumerFilesForAddedFiles(mi.graph, batch.pythonAddedFiles)...) } cargoRetargetRepo := "" if batch != nil { @@ -556,6 +551,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.pythonConsumerFiles...) } if result != nil { + crossRepoFiles = appendUniqueSorted(crossRepoFiles, result.CrossRepoResolveFiles...) crossRepoFiles = appendUniqueSorted(crossRepoFiles, result.DerivedInvalidation.Files...) } idx := mi.GetIndexer(repoPrefix) @@ -603,7 +599,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } crossRepoFiles = appendUniqueSorted(crossRepoFiles, semanticFiles...) } - if needed || len(semanticFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || len(batch.pythonConsumerFiles) > 0 || len(crossRepoFiles) > 0 && (batch.cargoAddedFile || len(batch.cargoConsumerFiles) > 0)) { + if needed || len(semanticFiles) > 0 || result != nil && len(result.CrossRepoResolveFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || len(batch.pythonConsumerFiles) > 0 || len(crossRepoFiles) > 0 && (batch.cargoAddedFile || len(batch.cargoConsumerFiles) > 0)) { if idx != nil { idx.observeIncrementalCatchup("cross_repo", crossRepoFiles) } diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 8b0778bed..898159e2f 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -3742,6 +3742,9 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE // Parallel warmup deliberately defers both tails to its later workspace // resolve/global phases. A full retrack already ran its own resolve/derived // pipeline and only needs the existing cross-repository contract reconcile. + if batchMode.deferResolve && batch != nil { + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, pythonConsumerFilesForAddedFiles(mi.graph, batch.pythonAddedFiles)...) + } if !result.FullRetrack && !batchMode.deferResolve { mi.resolveIncrementalRepoMutation(prefix, result, receipt, batch) if !batchMode.deferGlobalPasses { diff --git a/internal/indexer/python_import_lifecycle_test.go b/internal/indexer/python_import_lifecycle_test.go index 5e18215bb..f04b2fe3e 100644 --- a/internal/indexer/python_import_lifecycle_test.go +++ b/internal/indexer/python_import_lifecycle_test.go @@ -24,7 +24,7 @@ import ( func TestTrackedPythonImportLifecycle(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { for _, providerFirst := range []bool{false, true} { - for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track", "deferred_track"} { + for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "provider_gains_empty_module", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track", "deferred_track"} { if operation == "deferred_track" && providerFirst { continue // This case starts with an already tracked consumer. } @@ -42,6 +42,8 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { consumerSource := pythonLifecycleConsumer if operation == "provider_gains_new_module" { consumerSource += "\nfrom PIL.extra import later\ndef after(): return later()\n" + } else if operation == "provider_gains_empty_module" { + consumerSource += "\nimport PIL.extra\n" } writeWorkspaceFile(t, consumer, "app.py", consumerSource) entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} @@ -119,6 +121,11 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { } assertImport() logs.TakeAll() + var scans *pythonLookupScanStore + if operation == "provider_save" { + scans = &pythonLookupScanStore{Store: g} + mi.graph = scans + } switch operation { case "consumer_save": writeWorkspaceFile(t, consumer, "app.py", pythonLifecycleConsumer+"def edited(): pass\n") @@ -136,6 +143,9 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { require.NoError(t, os.Remove(filepath.Join(provider, "src/PIL/util.py"))) writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider+"def added(): pass\n") _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py"), filepath.Join(provider, "src/PIL/image.py")}) + case "provider_gains_empty_module": + writeWorkspaceFile(t, provider, "src/PIL/extra.py", "") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/extra.py")}) case "provider_gains_new_module": writeWorkspaceFile(t, provider, "src/PIL/extra.py", "def later(): pass\n") _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/extra.py")}) @@ -223,6 +233,10 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { _, err = mi.TrackRepoCtx(context.Background(), entries[1]) } require.NoError(t, err) + if scans != nil { + // The existing cross-repo pass scans imports once for reachability. + require.Equal(t, 1, scans.importScans, "ordinary saves must not add a Python consumer-frontier scan") + } missingUtil := operation == "provider_delete_util_batched" || operation == "forced_evict_util" if missingUtil { var missingImport, missingCall []*graph.Edge @@ -287,3 +301,15 @@ func pythonLifecycleBindings(g graph.Store, missingUtil bool) []string { slices.Sort(bindings) return bindings } + +func TestPythonAddedFileWithoutConsumersDoesNotRunGlobalResolve(t *testing.T) { + core, logs := observer.New(zap.InfoLevel) + g := graph.New() + g.AddNode(&graph.Node{ID: "consumer/app.py::run", Kind: graph.KindFunction, Name: "run", FilePath: "consumer/app.py", RepoPrefix: "consumer", Language: "python"}) + g.AddNode(&graph.Node{ID: "provider/new.py::f", Kind: graph.KindFunction, Name: "f", FilePath: "provider/new.py", RepoPrefix: "provider", Language: "python"}) + g.AddEdge(&graph.Edge{From: "consumer/app.py::run", To: "unresolved::f", Kind: graph.EdgeCalls, FilePath: "consumer/app.py"}) + mi := &MultiIndexer{graph: g, logger: zap.New(core)} + mi.resolveIncrementalRepoMutation("provider", &IndexResult{}, &graph.MutationReceipt{Complete: true}, + &reparsePendingEnrichmentBatch{pythonAddedFiles: []string{"provider/empty.py"}}) + require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) +} diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index b24f2dea6..492efce9d 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -111,3 +111,35 @@ func (mi *MultiIndexer) pythonConsumerFilesForRepo(repo string) []string { } return sortedStringKeys(files) } + +// Keep candidate importers before warmup has registered every repository. +// The existing cross-repo matcher still checks providers and declarations. +func pythonConsumerFilesForAddedFiles(g graph.Store, files []string) []string { + if len(files) == 0 { + return nil + } + tops := map[string]bool{} + for _, file := range files { + rel := strings.TrimPrefix(file, graph.RepoPrefixOfID(file)+"/") + for _, path := range []string{rel, strings.TrimPrefix(rel, "src/")} { + top, _, _ := strings.Cut(path, "/") + tops[strings.TrimSuffix(top, ".py")] = true + } + } + consumers := map[string]struct{}{} + for edge := range graph.EdgesLightSeq(g, graph.EdgeImports) { + if !strings.HasSuffix(edge.FilePath, ".py") { + continue + } + for _, prefix := range []string{"module::pypi:", "external::", "unresolved::import::"} { + if spec, ok := strings.CutPrefix(edge.To, prefix); ok { + top, _, _ := strings.Cut(spec, ".") + if tops[top] { + consumers[edge.FilePath] = struct{}{} + } + break + } + } + } + return sortedStringKeys(consumers) +} diff --git a/internal/indexer/python_import_warm_recovery_test.go b/internal/indexer/python_import_warm_recovery_test.go new file mode 100644 index 000000000..195b8db46 --- /dev/null +++ b/internal/indexer/python_import_warm_recovery_test.go @@ -0,0 +1,102 @@ +package indexer + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedPythonWarmFileRecovery(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, deferred := range []bool{false, true} { + for _, restore := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/provider_first=%v/deferred=%v/restore=%v", backend, providerFirst, deferred, restore), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider) + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + consumerSource := pythonLifecycleConsumer + if !restore { + consumerSource += "\nimport PIL.extra\n" + } + writeWorkspaceFile(t, consumer, "app.py", consumerSource) + for i := 0; i < 6; i++ { + writeWorkspaceFile(t, provider, fmt.Sprintf("pad%d.py", i), "def pad(): pass\n") + writeWorkspaceFile(t, consumer, fmt.Sprintf("pad%d.py", i), "def pad(): pass\n") + } + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + if providerFirst { + entries[0], entries[1] = entries[1], entries[0] + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + newMI := func(g graph.Store) *MultiIndexer { + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + return mi + } + g := pythonLocalStore(t, backend) + mi := newMI(g) + _, err = mi.IndexAll() + require.NoError(t, err) + if restore { + file := filepath.Join(provider, "src/PIL/util.py") + require.NoError(t, os.Remove(file)) + _, err = mi.IncrementalReindexRepo("provider", []string{file}) + require.NoError(t, err) + } + prior := map[string]map[string]int64{"consumer": mi.FileMtimes("consumer"), "provider": mi.FileMtimes("provider")} + require.NoError(t, mi.Close(context.Background())) + if restore { + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + } else { + writeWorkspaceFile(t, provider, "src/PIL/extra.py", "") + } + restarted := newMI(g) + if deferred { + restarted.BeginParallelBatch() + } else { + restarted.BeginBatch() + } + var files []string + for _, entry := range entries { + result, err := restarted.ReconcileRepoCtx(context.Background(), entry, prior[entry.Name]) + require.NoError(t, err) + require.False(t, result.FullRetrack, "must exercise scoped warm reconcile") + if entry.Name == "provider" { + require.Contains(t, result.CrossRepoResolveFiles, "consumer/app.py") + } + files = appendUniqueSorted(files, result.DerivedInvalidation.Files...) + files = appendUniqueSorted(files, result.CrossRepoResolveFiles...) + } + require.NoError(t, restarted.RunPreEnrichResolveFiles(context.Background(), files, nil)) + restarted.EndBatch() + cold := pythonLocalStore(t, backend) + _, err = newMI(cold).IndexAll() + require.NoError(t, err) + require.Equal(t, pythonLifecycleBindings(cold, false), pythonLifecycleBindings(g, false)) + }) + } + } + } + } +} diff --git a/internal/indexer/python_tracked_import.go b/internal/indexer/python_tracked_import.go index 7ee1c4c0d..2e9295931 100644 --- a/internal/indexer/python_tracked_import.go +++ b/internal/indexer/python_tracked_import.go @@ -1,8 +1,7 @@ package indexer import ( - "os" - "path/filepath" + "path" "regexp" "strings" @@ -35,7 +34,8 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { if meta == nil || meta.RootPath == "" { continue } - src, _ := readDiskFile(filepath.Join(meta.RootPath, "pyproject.toml")) + tree := mi.manifestTreeFor(repo, meta.RootPath) + src, _ := tree.readFile("pyproject.toml") var manifest struct { Project struct{ Name string } `toml:"project"` Tool struct { @@ -62,9 +62,8 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { } } } - requirements, _ := filepath.Glob(filepath.Join(meta.RootPath, "requirements*.txt")) - for _, file := range requirements { - src, _ := readDiskFile(file) + for _, file := range tree.matchFiles("requirements*.txt") { + src, _ := tree.readFile(file) specs = append(specs, modules.ParseRequirementsTxt(src)...) } for _, dep := range specs { @@ -86,25 +85,19 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { if !declared { continue } + tree := mi.manifestTreeFor(repo, roots[repo].RootPath) for _, dir := range []string{"", "src"} { - children, err := os.ReadDir(filepath.Join(roots[repo].RootPath, dir)) - if err != nil { - continue + files := tree.matchFiles(path.Join(dir, "*.py")) + for _, pkg := range tree.matchDirs(path.Join(dir, "*")) { + files = append(files, pkg+"/__init__.py") } - for _, child := range children { - name := child.Name() - if !child.IsDir() && !strings.HasSuffix(name, ".py") { - continue - } + for _, file := range files { + name := strings.TrimSuffix(strings.TrimPrefix(file, dir+"/"), "/__init__.py") if dir == "" && pythonRootToolingFile(name) { continue } top := strings.TrimSuffix(name, ".py") - file := filepath.ToSlash(filepath.Join(repo, dir, name)) - if child.IsDir() { - file += "/__init__.py" - } - node := mi.graph.GetNode(file) + node := mi.graph.GetNode(repo + "/" + file) if node == nil || node.Kind != graph.KindFile || node.RepoPrefix != repo { continue } diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go index 185cb8526..ee33983d5 100644 --- a/internal/indexer/python_tracked_import_test.go +++ b/internal/indexer/python_tracked_import_test.go @@ -141,6 +141,7 @@ func TestPythonPackageLookupNoProviders(t *testing.T) { type pythonLookupScanStore struct { graph.Store globalScans int + importScans int } func (s *pythonLookupScanStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { @@ -165,3 +166,77 @@ func TestPythonPackageLookupAvoidsGlobalFileScan(t *testing.T) { require.Len(t, lookup.Providers("consumer", "PIL"), 1) require.Zero(t, g.globalScans) } + +func TestPythonPackageLookupUsesInstalledManifestTree(t *testing.T) { + for _, entry := range []string{"PIL.py", "PIL/__init__.py", "src/PIL.py", "src/PIL/__init__.py"} { + for _, mode := range []string{"pyproject", "requirements", "layout", "undeclared", "missing_layout"} { + t.Run(entry+"/"+mode, func(t *testing.T) { + provider, consumer, providerSnapshot, consumerSnapshot := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir() + providerName, checkoutDeps := "pillow", "[project]\ndependencies = ['pillow']\n" + if mode == "pyproject" { + providerName, checkoutDeps = "wrong-checkout", "" + } + snapshotDeps := "[project]\ndependencies = ['pillow']\n" + if mode == "requirements" || mode == "undeclared" { + snapshotDeps = "" + } + if mode == "requirements" { + checkoutDeps = "" + writeWorkspaceFile(t, consumer, "requirements-dev.txt", "other\n") + writeWorkspaceFile(t, consumerSnapshot, "requirements-dev.txt", "pillow >= 1\n") + } + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = '"+providerName+"'\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", checkoutDeps) + writeWorkspaceFile(t, providerSnapshot, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, consumerSnapshot, "pyproject.toml", snapshotDeps) + if mode != "layout" { + writeWorkspaceFile(t, provider, entry, "") + } + if mode != "missing_layout" { + writeWorkspaceFile(t, providerSnapshot, entry, "") + } + g := graph.New() + id := "provider/" + entry + g.AddNode(&graph.Node{ID: id, FilePath: id, Kind: graph.KindFile, RepoPrefix: "provider"}) + providerIdx, consumerIdx := sideChannelIndexer(t, g, provider), sideChannelIndexer(t, g, consumer) + providerIdx.SetContentSource(snapshotOf(t, providerSnapshot)) + consumerIdx.SetContentSource(snapshotOf(t, consumerSnapshot)) + mi := &MultiIndexer{graph: g, + repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}, + indexers: map[string]*Indexer{"provider": providerIdx, "consumer": consumerIdx}} + lookup := mi.pythonPackageLookup() + if mode == "undeclared" || mode == "missing_layout" { + require.Nil(t, lookup, "checkout-only declaration/layout must not supply a provider") + return + } + require.NotNil(t, lookup) + require.Equal(t, []graph.FileNodeIdentity{{ID: id, FilePath: id, RepoPrefix: "provider"}}, lookup.Providers("consumer", "PIL")["provider"]) + }) + } + } +} + +func (s *pythonLookupScanStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + if len(kinds) == 1 && kinds[0] == graph.EdgeImports { + s.importScans++ + } + return graph.EdgesLightSeq(s.Store, kinds...) +} + +func TestPythonAddedFileConsumerFrontier(t *testing.T) { + g := &pythonLookupScanStore{Store: graph.New()} + for _, row := range []struct{ file, target string }{ + {"consumer/a.py", "module::pypi:PIL"}, + {"consumer/b.py", "external::PIL.extra"}, + {"consumer/c.py", "unresolved::import::PIL.extra"}, + {"consumer/d.py", "module::pypi:other"}, + {"consumer/e.py", "provider/src/PIL/image.py"}, + {"consumer/f.rs", "external::PIL.extra"}, + } { + g.AddEdge(&graph.Edge{From: row.file, FilePath: row.file, To: row.target, Kind: graph.EdgeImports}) + } + require.Empty(t, pythonConsumerFilesForAddedFiles(g, nil)) + require.Zero(t, g.importScans) + require.Equal(t, []string{"consumer/a.py", "consumer/b.py", "consumer/c.py"}, pythonConsumerFilesForAddedFiles(g, []string{"provider/src/PIL/extra.py"})) + require.Equal(t, 1, g.importScans) +} From d79c100eced360bde2390c0e359ad814477c1ab7 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 06:00:43 -0700 Subject: [PATCH 024/111] fix(indexer): retain Python module additions through fallback Record successful new Python files on the fallback path, matching the structural batch path. This preserves consumer retry when a refresh read changes after the probe. Cover the fallback with a deterministic transform on memory and SQLite, then compare import and call bindings with a cold index. --- internal/indexer/incremental_batch.go | 3 + .../indexer/python_import_fallback_test.go | 77 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 internal/indexer/python_import_fallback_test.go diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 1a2da839c..f3be650c0 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -522,6 +522,9 @@ func (idx *Indexer) reindexIncrementalFallback( if markerBatch != nil && len(fallback.priorNodes) == 0 && strings.HasSuffix(fallback.graphPath, ".rs") { markerBatch.cargoAddedFile = true } + if markerBatch != nil && len(fallback.priorNodes) == 0 && strings.HasSuffix(fallback.graphPath, ".py") { + markerBatch.pythonAddedFiles = append(markerBatch.pythonAddedFiles, fallback.graphPath) + } freshNodes := idx.graph.GetFileNodesByPaths([]string{fallback.graphPath})[fallback.graphPath] freshDerived := storedDerivedFingerprints(freshNodes) if fallback.probeOK && fallback.probe.derived.complete() { diff --git a/internal/indexer/python_import_fallback_test.go b/internal/indexer/python_import_fallback_test.go new file mode 100644 index 000000000..2bdaf7bec --- /dev/null +++ b/internal/indexer/python_import_fallback_test.go @@ -0,0 +1,77 @@ +package indexer + +import ( + "context" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +// pythonFallbackRefreshTransform changes extra.py's bytes on exactly its second +// application: the probe read sees call 1, takePreparedRefresh sees call 2 +// (mismatch, so the stage falls back), and the fallback indexFile sees call 3. +type pythonFallbackRefreshTransform struct{ calls *atomic.Int32 } + +func (pythonFallbackRefreshTransform) name() string { return "python-fallback-refresh" } +func (pythonFallbackRefreshTransform) matches(p string) bool { return strings.HasSuffix(p, "extra.py") } +func (pythonFallbackRefreshTransform) asLanguage() string { return "" } +func (f pythonFallbackRefreshTransform) apply(p string, src []byte) ([]byte, error) { + if f.calls.Add(1) == 2 { + return append(append([]byte(nil), src...), []byte("\n# concurrent write\n")...), nil + } + return src, nil +} + +func TestTrackedPythonNewModuleThroughFallback(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider) + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + writeWorkspaceFile(t, consumer, "app.py", pythonLifecycleConsumer+"\nfrom PIL.extra import later\ndef after(): return later()\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + g := pythonLocalStore(t, backend) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + _, err = mi.IndexAll() + require.NoError(t, err) + + idx := mi.GetIndexer("provider") + require.NotNil(t, idx) + var calls atomic.Int32 + idx.transforms.transforms = append(idx.transforms.transforms, pythonFallbackRefreshTransform{calls: &calls}) + writeWorkspaceFile(t, provider, "src/PIL/extra.py", "def later(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/extra.py")}) + require.NoError(t, err) + t.Logf("transform calls=%d", calls.Load()) + require.GreaterOrEqual(t, calls.Load(), int32(3), "fallback path must run") + require.NotNil(t, g.GetNode("provider/src/PIL/extra.py")) + + cold := pythonLocalStore(t, backend) + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, pythonLifecycleBindings(cold, false), pythonLifecycleBindings(g, false)) + }) + } +} From b31696ad5469c1a43e2493c8b347e3b300774472 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 12:35:13 -0700 Subject: [PATCH 025/111] fix(resolver): follow tracked Python package re-exports --- docs/multi-repo.md | 8 ++ internal/indexer/extractor_version.go | 1 + internal/indexer/extractor_version_test.go | 8 +- internal/indexer/python_import_restub.go | 19 +++ .../indexer/python_package_reexport_test.go | 119 ++++++++++++++++++ internal/parser/languages/python.go | 20 ++- .../languages/python_a22_relative_test.go | 31 +++++ internal/resolver/cross_repo.go | 2 +- internal/resolver/python_package_reexport.go | 84 +++++++++++++ .../resolver/python_package_reexport_test.go | 64 ++++++++++ 10 files changed, 352 insertions(+), 4 deletions(-) create mode 100644 internal/indexer/python_package_reexport_test.go create mode 100644 internal/resolver/python_package_reexport.go create mode 100644 internal/resolver/python_package_reexport_test.go diff --git a/docs/multi-repo.md b/docs/multi-repo.md index 259f5d57d..85e547ff9 100644 --- a/docs/multi-repo.md +++ b/docs/multi-repo.md @@ -126,6 +126,14 @@ the participating repositories' allow-lists. This keeps a repository from silently losing framework edges requested by a sibling while allowing a workspace where every repository opts out to avoid the scans entirely. +### Python package re-exports + +Calls through a declared, tracked Python distribution can follow one level of +relative `from .module import Name` or `from .module import Name as Alias` in its +package `__init__.py`. The bridge requires an exact imported binding and a unique +function or class in the provider. It does not follow wildcard imports, +conditional or function-local imports, or multi-level re-export chains. + ## Daemon tuning (optional) The background daemon writes JSON lines to `daemon.log` and rotates it while diff --git a/internal/indexer/extractor_version.go b/internal/indexer/extractor_version.go index fb33b938e..340e45c7a 100644 --- a/internal/indexer/extractor_version.go +++ b/internal/indexer/extractor_version.go @@ -49,6 +49,7 @@ var extractorVersions = map[string]int{ "julia": 3, // callee and macro decoding from CST children (chained/parametric callees, Base.@time, Base.:(==)), macro/operator exports, member_of for module consts and nested modules, @doc and public metadata (was: bespoke tree-sitter extractor replaced the regex extractor) "javascript": 2, // renamed per-binding imports persist import_alias Meta; older rows keyed a renamed import by its original name "typescript": 2, // renamed per-binding imports persist import_alias Meta; older rows keyed a renamed import by its original name + "python": 2, // package relative from-imports retain exact re-export bindings } // extractorSaltExtLang maps a lower-case file extension to the language diff --git a/internal/indexer/extractor_version_test.go b/internal/indexer/extractor_version_test.go index e409f6e91..966d98db8 100644 --- a/internal/indexer/extractor_version_test.go +++ b/internal/indexer/extractor_version_test.go @@ -227,8 +227,14 @@ func TestStaleLangsDetection(t *testing.T) { t.Errorf("persisted policy epoch = %d, want %d", got, postExtractionPolicyVersion) } + if got := ExtractorVersionStaleLangs(snapshotJSON(t, nil, "python")); !reflect.DeepEqual(got, []string{"python"}) { + t.Errorf("snapshot without Python re-export evidence = %v, want [python]", got) + } policySalt := "_post_extraction_policy@2" - for _, path := range []string{"model.py", "Model.java", "model.rb", "schema.ex"} { + if got, want := merkleSaltFor("model.py"), policySalt+"|python@2"; got != want { + t.Errorf("Python extractor salt = %q, want %q", got, want) + } + for _, path := range []string{"Model.java", "model.rb", "schema.ex"} { if got := merkleSaltFor(path); got != policySalt { t.Errorf("global policy salt for %s = %q, want %q", path, got, policySalt) } diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index 492efce9d..2bafcb3cc 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -69,6 +69,25 @@ func restubTrackedPythonImportsFromView(g graph.Store, evicted map[string]struct continue } old := edge.To + // A package edit can change a re-export without changing the old + // definition. Reopen its dependent calls when the import is restubbed. + if edge.Kind == graph.EdgeImports && strings.HasSuffix(old, "/__init__.py") { + for row := range graph.EdgesInScopeSeq(g, nil, []string{edge.FilePath}, graph.EdgeCalls) { + call := row.Edge + if call == nil || !call.CrossRepo || call.Meta[resolver.PythonReExportFileMeta] != old { + continue + } + original, _ := call.Meta[resolver.PythonCallStubMeta].(string) + if original == "" { + continue + } + previous := call.To + graph.StashRestubProvenance(call) + call.To, call.CrossRepo, call.ConfidenceLabel = original, false, "" + delete(call.Meta, resolver.PythonReExportFileMeta) + batch = append(batch, graph.EdgeReindex{Edge: call, OldTo: previous}) + } + } if edge.Kind == graph.EdgeCalls { graph.StashRestubProvenance(edge) edge.ConfidenceLabel = "" diff --git a/internal/indexer/python_package_reexport_test.go b/internal/indexer/python_package_reexport_test.go new file mode 100644 index 000000000..8678f766f --- /dev/null +++ b/internal/indexer/python_package_reexport_test.go @@ -0,0 +1,119 @@ +package indexer + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedPythonPackageReExport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, tc := range []struct { + name, exports string + resolves bool + }{ + {"explicit_alias", "from .url_safe import URLSafeTimedSerializer as URLSafeTimedSerializer\n", true}, + {"plain", "from .url_safe import URLSafeTimedSerializer\n", true}, + {"ambiguous", "from .url_safe import URLSafeTimedSerializer\nfrom .other import URLSafeTimedSerializer\n", false}, + {"not_exported", "from .other import unrelated\n", false}, + } { + t.Run(fmt.Sprintf("%s/provider_first=%v/%s", backend, providerFirst, tc.name), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'itsdangerous'\n") + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", tc.exports) + writeWorkspaceFile(t, provider, "src/itsdangerous/url_safe.py", "class URLSafeTimedSerializer: pass\n") + writeWorkspaceFile(t, provider, "src/itsdangerous/other.py", "class URLSafeTimedSerializer: pass\ndef unrelated(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['itsdangerous']\n") + writeWorkspaceFile(t, consumer, "app.py", "from itsdangerous import URLSafeTimedSerializer\ndef run(): return URLSafeTimedSerializer('secret')\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + if providerFirst { + entries[0], entries[1] = entries[1], entries[0] + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + g := pythonLocalStore(t, backend) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + for _, entry := range entries { + _, err = mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + var imports, calls []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports { + imports = append(imports, edge) + } + } + require.Len(t, imports, 1) + require.Equal(t, "provider/src/itsdangerous/__init__.py", imports[0].To) + require.True(t, imports[0].CrossRepo) + assertCall := func(target string, linked bool) { + t.Helper() + calls = nil + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls { + calls = append(calls, edge) + } + } + require.Len(t, calls, 1) + require.Equal(t, linked, calls[0].CrossRepo, "%#v", calls[0]) + require.Equal(t, target, calls[0].To) + require.NotEmpty(t, calls[0].Meta["python_call_stub"]) + require.NotContains(t, calls[0].Meta, "external_call") + } + stub := "unresolved::extern::itsdangerous.URLSafeTimedSerializer::URLSafeTimedSerializer" + target := "provider/src/itsdangerous/url_safe.py::URLSafeTimedSerializer" + if !tc.resolves { + assertCall(stub, false) + return + } + assertCall(target, true) + if tc.name != "explicit_alias" { + return + } + writeWorkspaceFile(t, provider, "src/itsdangerous/url_safe.py", "class URLSafeTimedSerializer:\n def added(self): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/itsdangerous/url_safe.py")}) + require.NoError(t, err) + assertCall(target, true) + _, err = mi.IndexRepo("provider") + require.NoError(t, err) + assertCall(target, true) + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") + require.NoError(t, err) + assertCall(stub, false) + _, err = mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: provider, Name: "provider", Workspace: "shared"}) + require.NoError(t, err) + assertCall(target, true) + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", "from .other import URLSafeTimedSerializer\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/itsdangerous/__init__.py")}) + require.NoError(t, err) + assertCall("provider/src/itsdangerous/other.py::URLSafeTimedSerializer", true) + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", "from .url_safe import URLSafeTimedSerializer\nfrom .other import URLSafeTimedSerializer\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/itsdangerous/__init__.py")}) + require.NoError(t, err) + assertCall(stub, false) + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", "") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/itsdangerous/__init__.py")}) + require.NoError(t, err) + assertCall(stub, false) + }) + } + } + } +} diff --git a/internal/parser/languages/python.go b/internal/parser/languages/python.go index 4926d8b37..03c4ba8a9 100644 --- a/internal/parser/languages/python.go +++ b/internal/parser/languages/python.go @@ -1041,12 +1041,14 @@ func (e *PythonExtractor) emitImportFromRelative(m parser.QueryResult, filePath, // `from numpy import array` path uses (one import edge to the // originating module) — the only difference is that for `from . // import X` the module name comes from the import list. + var moduleImport *graph.Edge if modPath != "" { pyEmitImportNode(filePath, fileID, modPath, "", importLine, result) - result.Edges = append(result.Edges, &graph.Edge{ + moduleImport = &graph.Edge{ From: fileID, To: "unresolved::pyrel::" + stem, Kind: graph.EdgeImports, FilePath: filePath, Line: importLine, - }) + } + result.Edges = append(result.Edges, moduleImport) } def, ok := m.Captures["importfrom_rel.def"] @@ -1054,9 +1056,23 @@ func (e *PythonExtractor) emitImportFromRelative(m parser.QueryResult, filePath, return } stmt := def.Node + // Keep exact package bindings for the tracked-import call bridge. Ordinary + // imports and function-local bindings do not expose package attributes. + var exports map[string]any + if moduleImport != nil && strings.HasSuffix(filePath, "/__init__.py") && stmt.Parent().Type() == "module" { + exports = make(map[string]any) + moduleImport.Meta = map[string]any{"python_reexports": exports, "python_reexport_stem": stem} + } moduleSeen := false emittedSub := map[string]bool{} emitSubmodule := func(name, alias string) { + if exports != nil && name != "" { + binding := alias + if binding == "" { + binding = name + } + exports[binding] = name + } if modPath != "" || name == "" { return } diff --git a/internal/parser/languages/python_a22_relative_test.go b/internal/parser/languages/python_a22_relative_test.go index 60864906b..bdcd2a13c 100644 --- a/internal/parser/languages/python_a22_relative_test.go +++ b/internal/parser/languages/python_a22_relative_test.go @@ -120,3 +120,34 @@ func TestPyResolveRelativeStem_CornerCases(t *testing.T) { "filePath=%q dots=%d mod=%q", c.filePath, c.dots, c.mod) } } + +func TestPythonPackageReExportMetadata(t *testing.T) { + for _, tc := range []struct { + name, file, source string + want map[string]any + }{ + {"plain and alias", "pkg/__init__.py", "from .impl import Plain, Original as Public\n", map[string]any{"Plain": "Plain", "Public": "Original"}}, + {"ordinary module", "pkg/app.py", "from .impl import Plain\n", nil}, + {"function local", "pkg/__init__.py", "def local():\n from .impl import Plain\n", nil}, + {"conditional", "pkg/__init__.py", "if flag:\n from .impl import Plain\n", nil}, + {"wildcard", "pkg/__init__.py", "from .impl import *\n", map[string]any{}}, + } { + t.Run(tc.name, func(t *testing.T) { + result, err := NewPythonExtractor().Extract(tc.file, []byte(tc.source)) + require.NoError(t, err) + var imports []*graph.Edge + for _, edge := range result.Edges { + if edge.Kind == graph.EdgeImports { + imports = append(imports, edge) + } + } + require.Len(t, imports, 1) + if tc.want == nil { + require.NotContains(t, imports[0].Meta, "python_reexports") + } else { + require.Equal(t, tc.want, imports[0].Meta["python_reexports"]) + require.Equal(t, "pkg/impl", imports[0].Meta["python_reexport_stem"]) + } + }) + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index d31efefe5..bacab5dab 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -1297,7 +1297,7 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b } return eligible, nil } - if resolvePythonModuleExtern(e, module, symbol, callerRepo, &ResolveStats{}, find) { + if resolvePythonModuleExtern(e, module, symbol, callerRepo, &ResolveStats{}, find) || cr.resolvePythonPackageReExport(e, module, symbol) { stats.Resolved++ if e.CrossRepo { stats.CrossRepoEdges++ diff --git a/internal/resolver/python_package_reexport.go b/internal/resolver/python_package_reexport.go new file mode 100644 index 000000000..c4196822e --- /dev/null +++ b/internal/resolver/python_package_reexport.go @@ -0,0 +1,84 @@ +package resolver + +import ( + "strings" + + "github.com/zzet/gortex/internal/graph" +) + +// PythonReExportFileMeta retains the package whose binding selected the call. +const PythonReExportFileMeta = "python_reexport_file" + +// resolvePythonPackageReExport follows one exact relative from-import in a +// linked package __init__.py. A directory import alone is not symbol evidence. +func (cr *CrossRepoResolver) resolvePythonPackageReExport(e *graph.Edge, module, symbol string) bool { + callerWS := cr.callerWorkspaceID(e) + var match *graph.Node + packageFile := "" + for _, imp := range cr.graph.GetOutEdges(cr.callerFileID(e)) { + if imp.Kind != graph.EdgeImports || !imp.CrossRepo || !strings.HasSuffix(imp.To, "/__init__.py") { + continue + } + stub, _ := imp.Meta[PythonImportStubMeta].(string) + spec, _ := imp.Meta["python_import"].(string) + if stub == "" { + continue + } + pkg := cr.cachedGetNode(imp.To) + if pkg == nil || !cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(pkg), spec) { + continue + } + name := symbol + if module != spec { + name = pythonImportedName(module) + if name == "" || module != spec+"."+name { + continue + } + } + if !pythonModuleFileMatches(pkg.FilePath, spec) { + continue + } + for _, reexport := range cr.graph.GetOutEdges(pkg.ID) { + if reexport.Kind != graph.EdgeImports { + continue + } + exports, _ := reexport.Meta["python_reexports"].(map[string]any) + name, _ := exports[name].(string) + if name == "" { + continue + } + // The extractor's stem is repo-relative even when the per-repo pass + // has parked the relative import before multi-repo prefixing. + stem, _ := reexport.Meta["python_reexport_stem"].(string) + if stem == "" { + return false + } + found := false + for _, suffix := range []string{".py", "/__init__.py"} { + candidate := cr.cachedGetNode(pkg.RepoPrefix + "/" + stem + suffix + "::" + name) + if candidate == nil { + continue + } + if candidate.RepoPrefix != pkg.RepoPrefix || + (candidate.Kind != graph.KindFunction && candidate.Kind != graph.KindType) || + !cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(candidate), spec) { + return false + } + if match != nil && match.ID != candidate.ID { + return false + } + match, found = candidate, true + packageFile = pkg.ID + } + if !found { + return false + } + } + } + if match == nil { + return false + } + e.To, e.CrossRepo = match.ID, true + e.Meta[PythonReExportFileMeta] = packageFile + return true +} diff --git a/internal/resolver/python_package_reexport_test.go b/internal/resolver/python_package_reexport_test.go new file mode 100644 index 000000000..061e1f2e8 --- /dev/null +++ b/internal/resolver/python_package_reexport_test.go @@ -0,0 +1,64 @@ +package resolver + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" +) + +func TestTrackedPythonPackageReExportEvidence(t *testing.T) { + for _, scenario := range []string{"module_call", "from_alias", "no_import", "unmarked_import", "wrong_package", "package_workspace", "symbol_workspace", "missing_symbol", "ambiguous_layout"} { + t.Run(scenario, func(t *testing.T) { + g := graph.New() + file, pkg, impl := "consumer/app.py", "provider/src/pkg/__init__.py", "provider/src/pkg/impl.py" + for _, node := range []*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared"}, + {ID: pkg, FilePath: pkg, Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: "shared"}, + {ID: impl + "::Original", FilePath: impl, Name: "Original", Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}, + } { + if scenario == "package_workspace" && node.ID == pkg || scenario == "symbol_workspace" && node.ID == impl+"::Original" { + node.WorkspaceID = "foreign" + } + if scenario != "missing_symbol" || node.ID != impl+"::Original" { + g.AddNode(node) + } + } + imp := &graph.Edge{From: file, FilePath: file, To: pkg, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{PythonImportStubMeta: "external::pkg", "python_import": "pkg"}} + if scenario == "unmarked_import" { + delete(imp.Meta, PythonImportStubMeta) + } + if scenario == "wrong_package" { + imp.Meta["python_import"] = "other" + } + if scenario != "no_import" { + g.AddEdge(imp) + } + g.AddEdge(&graph.Edge{From: pkg, FilePath: pkg, To: impl, Kind: graph.EdgeImports, + Meta: map[string]any{"python_reexports": map[string]any{"Public": "Original"}, "python_reexport_stem": "src/pkg/impl"}}) + if scenario == "ambiguous_layout" { + path := "provider/src/pkg/impl/__init__.py" + g.AddNode(&graph.Node{ID: path + "::Original", FilePath: path, Name: "Original", Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}) + } + stub := "unresolved::extern::pkg::Public" + if scenario == "from_alias" { + stub = "unresolved::extern::pkg.Public::Alias" + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: stub, Kind: graph.EdgeCalls, Line: 2, Meta: map[string]any{PythonCallStubMeta: stub}}) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { return nil }) + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { return nil }) + cr.ResolveForFile("consumer", "app.py") + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 1) + if scenario == "module_call" || scenario == "from_alias" { + require.Equal(t, impl+"::Original", calls[0].To) + require.True(t, calls[0].CrossRepo) + } else { + require.Equal(t, stub, calls[0].To) + require.False(t, calls[0].CrossRepo) + } + }) + } +} From 618f87b6270934e99d80c5a9758f93a25460b8dd Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 13:16:42 -0700 Subject: [PATCH 026/111] fix(resolver): preserve Python re-export links across cold and mixed saves --- internal/indexer/python_import_restub.go | 7 ++ .../indexer/python_package_reexport_test.go | 94 ++++++++++++++++++- internal/resolver/cross_repo.go | 2 +- .../resolver/python_import_call_retry_test.go | 19 ++-- internal/resolver/python_tracked_import.go | 8 +- 5 files changed, 121 insertions(+), 9 deletions(-) diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index 2bafcb3cc..7b6c91f81 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -86,6 +86,13 @@ func restubTrackedPythonImportsFromView(g graph.Store, evicted map[string]struct call.To, call.CrossRepo, call.ConfidenceLabel = original, false, "" delete(call.Meta, resolver.PythonReExportFileMeta) batch = append(batch, graph.EdgeReindex{Edge: call, OldTo: previous}) + // SQLite reads return separate objects. Update the prior view too, + // so the generic carry cannot re-state the old call target. + for _, prior := range incoming[previous] { + if prior != nil && prior.From == call.From && prior.Kind == call.Kind && prior.Line == call.Line && prior.To == previous { + prior.To, prior.CrossRepo, prior.ConfidenceLabel = call.To, false, "" + } + } } } if edge.Kind == graph.EdgeCalls { diff --git a/internal/indexer/python_package_reexport_test.go b/internal/indexer/python_package_reexport_test.go index 8678f766f..3a8b2d371 100644 --- a/internal/indexer/python_package_reexport_test.go +++ b/internal/indexer/python_package_reexport_test.go @@ -100,8 +100,14 @@ func TestTrackedPythonPackageReExport(t *testing.T) { _, err = mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: provider, Name: "provider", Workspace: "shared"}) require.NoError(t, err) assertCall(target, true) + // Both files change structurally, but the old class keeps its shape. + // A stale incoming view must not carry its old call target. writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", "from .other import URLSafeTimedSerializer\n") - _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/itsdangerous/__init__.py")}) + writeWorkspaceFile(t, provider, "src/itsdangerous/url_safe.py", "class URLSafeTimedSerializer:\n def added(self): pass\ndef extra(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{ + filepath.Join(provider, "src/itsdangerous/__init__.py"), + filepath.Join(provider, "src/itsdangerous/url_safe.py"), + }) require.NoError(t, err) assertCall("provider/src/itsdangerous/other.py::URLSafeTimedSerializer", true) writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", "from .url_safe import URLSafeTimedSerializer\nfrom .other import URLSafeTimedSerializer\n") @@ -117,3 +123,89 @@ func TestTrackedPythonPackageReExport(t *testing.T) { } } } + +func TestTrackedPythonPackageReExportColdPaths(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, ambiguous := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/provider_first=%v/ambiguous=%v", backend, providerFirst, ambiguous), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + exports := "from .url_safe import URLSafeTimedSerializer\n" + if ambiguous { + exports += "from .other import URLSafeTimedSerializer\n" + } + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'itsdangerous'\n") + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", exports) + writeWorkspaceFile(t, provider, "src/itsdangerous/url_safe.py", "class URLSafeTimedSerializer: pass\n") + writeWorkspaceFile(t, provider, "src/itsdangerous/other.py", "class URLSafeTimedSerializer: pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['itsdangerous']\n") + writeWorkspaceFile(t, consumer, "app.py", "from itsdangerous import URLSafeTimedSerializer\ndef run(): return URLSafeTimedSerializer('secret')\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + if providerFirst { + entries[0], entries[1] = entries[1], entries[0] + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + expected := "provider/src/itsdangerous/url_safe.py::URLSafeTimedSerializer" + if ambiguous { + expected = "" + } + incrementalTarget := "" + for _, mode := range []string{"incremental", "index_all", "parallel_full"} { + t.Run(mode, func(t *testing.T) { + g := pythonLocalStore(t, backend) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + switch mode { + case "index_all": + _, err := mi.IndexAll() + require.NoError(t, err) + case "parallel_full": + // The daemon's cold-start sequence defers resolution until + // every repository is registered. No daemon is started here. + mi.BeginParallelBatch() + defer mi.EndBatch() + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + require.NoError(t, mi.RunPreEnrichResolve(context.Background(), nil, nil)) + case "incremental": + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + } + var calls []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls { + calls = append(calls, edge) + } + } + require.Len(t, calls, 1) + call := calls[0] + target := "" + if call.CrossRepo { + target = call.To + require.Equal(t, "unresolved::extern::itsdangerous.URLSafeTimedSerializer::URLSafeTimedSerializer", call.Meta["python_call_stub"]) + require.Equal(t, "provider/src/itsdangerous/__init__.py", call.Meta["python_reexport_file"]) + } + require.Equal(t, expected, target, "call: %#v", call) + if mode == "incremental" { + incrementalTarget = target + } else { + require.Equal(t, incrementalTarget, target) + } + }) + } + }) + } + } + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index bacab5dab..68fd1bfea 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -157,7 +157,7 @@ type CrossRepoResolver struct { cargoCrates CargoCrateLookup cargoRetargetRepos map[string]bool - pythonPackages func() *PythonPackageLookup + pythonPackages func() *PythonPackageLookup // pathAlias expands a JS/TS tsconfig/jsconfig path-alias / baseUrl // import specifier to the repo-prefixed file stem it targets. Same // contract as the field of the same name on Resolver — see diff --git a/internal/resolver/python_import_call_retry_test.go b/internal/resolver/python_import_call_retry_test.go index c7bcefa3b..0629218ab 100644 --- a/internal/resolver/python_import_call_retry_test.go +++ b/internal/resolver/python_import_call_retry_test.go @@ -37,10 +37,15 @@ func TestTrackedPythonImportReopensOnlyNewlyLinkedCalls(t *testing.T) { {file, eligible, graph.EdgeReferences}, {"consumer/other.py", eligible, graph.EdgeCalls}, {"outsider/app.py", eligible, graph.EdgeCalls}, + {file, "dep::PIL.image::f", graph.EdgeCalls}, + {file, "dep::other::f", graph.EdgeCalls}, + {file, "dep::PIL.image_extra::f", graph.EdgeCalls}, + {file, "dep::PIL.image::f", graph.EdgeReferences}, } { g.AddEdge(&graph.Edge{From: tc.file, FilePath: tc.file, To: tc.target, Kind: tc.kind, Line: i + 1, Meta: map[string]any{"external_call": true, PythonCallStubMeta: "unresolved::extern::PIL.image::f"}}) } - g.AddEdge(&graph.Edge{From: file, FilePath: file, To: eligible, Kind: graph.EdgeCalls, Line: 8}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: eligible, Kind: graph.EdgeCalls, Line: 12}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "dep::PIL.image::f", Kind: graph.EdgeCalls, Line: 13}) linked := &graph.Edge{From: file, FilePath: file, To: "provider/src/PIL/image.py", Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "module::pypi:PIL", "python_import": "PIL.image"}} unmarked := &graph.Edge{From: "consumer/other.py", FilePath: "consumer/other.py", To: linked.To, Kind: graph.EdgeImports, CrossRepo: true, @@ -49,13 +54,15 @@ func TestTrackedPythonImportReopensOnlyNewlyLinkedCalls(t *testing.T) { Meta: linked.Meta} cr := NewCrossRepo(g) reopened := cr.restubCallsForLinkedPythonImports([]graph.EdgeReindex{{Edge: linked}, {Edge: unmarked}, {Edge: unlinked}}) - require.Len(t, reopened, 1) - require.Equal(t, file, reopened[0].FilePath) - require.Equal(t, "unresolved::extern::PIL.image::f", reopened[0].To) - require.NotContains(t, reopened[0].Meta, "external_call") + require.Len(t, reopened, 2) + for _, call := range reopened { + require.Equal(t, file, call.FilePath) + require.Equal(t, "unresolved::extern::PIL.image::f", call.To) + require.NotContains(t, call.Meta, "external_call") + } for _, edges := range g.GetOutEdgesByNodeIDs([]string{file, "consumer/other.py", "outsider/app.py"}) { for _, edge := range edges { - if edge.Line != 1 { + if edge.Line != 1 && edge.Line != 8 { require.NotEqual(t, "unresolved::extern::PIL.image::f", edge.To) } } diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go index fdab01dcc..c48b90af1 100644 --- a/internal/resolver/python_tracked_import.go +++ b/internal/resolver/python_tracked_import.go @@ -249,8 +249,14 @@ func (cr *CrossRepoResolver) restubCallsForLinkedPythonImports(changes []graph.E } for _, spec := range specs[call.FilePath] { original, _ := call.Meta[PythonCallStubMeta].(string) + // Cold master resolution leaves dep:: terminals until externalization; + // incremental tracking has already synthesized external-call:: nodes. + callTarget := call.To + if ecosystem, module, ok := parseExternalCallTarget(callTarget); ok && ecosystem == "dep" { + callTarget = externalCallNodeID(ecosystem, module) + } target := externalCallPrefix + "dep::" + spec - if original == "" || (call.To != target && !strings.HasPrefix(call.To, target+".")) { + if original == "" || (callTarget != target && !strings.HasPrefix(callTarget, target+".")) { continue } old := call.To From d0193d0e6f159a1ead5f0ba17d65ff1e6e94ce80 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 19:18:57 -0700 Subject: [PATCH 027/111] test: keep the popular Go provider over the incoming-row budget The filler consumers had no workspace, so the full resolve correctly demoted all 16,385 filler imports and the provider save no longer ran over MaxIncomingSourceCandidateRows. Give the fillers the tracked caller's workspace and assert the budget is exceeded before the save. This also removes the slow path: with SQLite and the provider tracked first, the demotion went through a generic identity refresh with a poor query plan (180 s locally, past the 45 min CI timeout under -race). The test now takes about 14 s. --- internal/indexer/go_module_import_recovery_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/indexer/go_module_import_recovery_test.go b/internal/indexer/go_module_import_recovery_test.go index 381f3e336..17067ff7a 100644 --- a/internal/indexer/go_module_import_recovery_test.go +++ b/internal/indexer/go_module_import_recovery_test.go @@ -120,7 +120,7 @@ func TestGoModuleImportPopularProviderSave(t *testing.T) { var edges []*graph.Edge for i := 0; i < graph.MaxIncomingSourceCandidateRows+1; i++ { f := fmt.Sprintf("a/filler%d.go", i) - nodes = append(nodes, &graph.Node{ID: f, Kind: graph.KindFile, FilePath: f, RepoPrefix: "a", Language: "go"}, &graph.Node{ID: f + "::import::example.org/library", Kind: graph.KindImport, FilePath: f, RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/library"}}) + nodes = append(nodes, &graph.Node{ID: f, Kind: graph.KindFile, FilePath: f, RepoPrefix: "a", WorkspaceID: "shared", Language: "go"}, &graph.Node{ID: f + "::import::example.org/library", Kind: graph.KindImport, FilePath: f, RepoPrefix: "a", WorkspaceID: "shared", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/library"}}) edges = append(edges, &graph.Edge{From: f, To: "b/lib.go", Kind: graph.EdgeImports, FilePath: f, Line: 2, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/library", "go_import_fallback": "dep::example.org/library"}}) } store.AddBatch(nodes, edges) @@ -134,6 +134,9 @@ func TestGoModuleImportPopularProviderSave(t *testing.T) { require.NoError(t, err) } restored.RunGlobalResolve() + // Filler consumers share the tracked caller's workspace, so the full + // pass must keep their imports bound and the save over budget. + require.Greater(t, len(g.GetInEdges("b/lib.go")), graph.MaxIncomingSourceCandidateRows) f := filepath.Join(b, "new.go") writeFile(t, f, "package library\nfunc New() {}\n") require.NoError(t, restored.GetIndexer("b").IndexFile(f)) From d5d29251cce039440acba3be85d785c8dcef3a67 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 19:19:08 -0700 Subject: [PATCH 028/111] style: apply staticcheck QF1001 and QF1003 to the Go import code Use De Morgan's form for the Go import restub condition (same logic) and a tagged switch for the eviction route in its lifecycle test. --- internal/indexer/go_module_import_lifecycle_test.go | 7 ++++--- internal/indexer/indexer.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/indexer/go_module_import_lifecycle_test.go b/internal/indexer/go_module_import_lifecycle_test.go index 0871a7229..a9a082654 100644 --- a/internal/indexer/go_module_import_lifecycle_test.go +++ b/internal/indexer/go_module_import_lifecycle_test.go @@ -145,15 +145,16 @@ func TestGoModuleImportParkBeforeEviction(t *testing.T) { {ID: "a/main.go", Name: "main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, {ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}, }, []*graph.Edge{{From: "a/main.go", To: "b/lib.go", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2, CrossRepo: true, Confidence: 0.6, ConfidenceLabel: "EXTRACTED", Origin: graph.OriginASTInferred, Meta: map[string]any{"go_import": "example.org/lib"}}}) - if route == "repo" { + switch route { + case "repo": restubGoModuleRepoIncoming(g, "b") - } else if route == "deleted_batch" { + case "deleted_batch": provider := g.GetNode("b/lib.go") carried, _, _, _ := restubIncomingRefsFromView(g, []*incrementalBatchStage{{graphPath: "b/lib.go", priorNodes: []*graph.Node{provider}}}, incrementalPriorView{inByNode: g.GetInEdgesByNodeIDs([]string{provider.ID})}) assert.Empty(t, carried, "a deleted file cannot carry a resolved target") - } else { + default: idx := newTestIndexer(g) idx.SetRepoPrefix("b") t.Cleanup(idx.Close) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index c0d3a6e73..6e8983aae 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -5994,7 +5994,7 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string, [] } stub := graph.UnresolvedMarker + n.Name for _, e := range inEdges[n.ID] { - if e == nil || !graph.IsResolvableRefEdge(e.Kind) && !(e.Kind == graph.EdgeImports && e.Meta["go_import"] != nil) { + if e == nil || !graph.IsResolvableRefEdge(e.Kind) && (e.Kind != graph.EdgeImports || e.Meta["go_import"] == nil) { continue } if _, fromEvicted := evicted[e.From]; fromEvicted { From 7885639eeca348488c39387ae4ebee76744d9bdc Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 19:24:29 -0700 Subject: [PATCH 029/111] style: apply staticcheck to the Cargo and Python import code Use tagged switches in the Cargo and Python lifecycle tests and the Cargo retarget test (QF1003). Remove runCrossRepoResolveMutationFiles: its only caller, runCrossRepoResolveFiles, now calls runCrossRepoResolveMutationFrontiers directly with the same files. --- internal/indexer/cargo_import_lifecycle_test.go | 12 +++++++----- internal/indexer/python_import_lifecycle_test.go | 5 +++-- internal/indexer/workspace_resolve.go | 4 ---- internal/resolver/cargo_import_retarget_test.go | 7 ++++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/indexer/cargo_import_lifecycle_test.go b/internal/indexer/cargo_import_lifecycle_test.go index 3abb8a031..7cc2d752f 100644 --- a/internal/indexer/cargo_import_lifecycle_test.go +++ b/internal/indexer/cargo_import_lifecycle_test.go @@ -46,9 +46,10 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { } writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") consumerSource := cargoLifecycleConsumer - if operation == "provider_gains_new_module" { + switch operation { + case "provider_gains_new_module": consumerSource += "use other_crate::newmod::later;\npub fn after() { later(); }\n" - } else if operation == "provider_gains_empty_module" { + case "provider_gains_empty_module": consumerSource += "use other_crate::newmod::*;\n" } writeWorkspaceFile(t, consumer, "src/lib.rs", consumerSource) @@ -75,7 +76,8 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { logger := zap.New(core) mi := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) - if operation == "deferred_track" { + switch operation { + case "deferred_track": _, err = mi.TrackRepoCtx(context.Background(), entries[0]) require.NoError(t, err) mi.BeginParallelBatch() @@ -88,12 +90,12 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { } require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs"}, nil)) mi.EndBatch() - } else if operation == "track" { + case "track": for _, entry := range gc.Repos { _, err = mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) } - } else { + default: _, err = mi.IndexAll() require.NoError(t, err) } diff --git a/internal/indexer/python_import_lifecycle_test.go b/internal/indexer/python_import_lifecycle_test.go index f04b2fe3e..3b38bbd58 100644 --- a/internal/indexer/python_import_lifecycle_test.go +++ b/internal/indexer/python_import_lifecycle_test.go @@ -40,9 +40,10 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") consumerSource := pythonLifecycleConsumer - if operation == "provider_gains_new_module" { + switch operation { + case "provider_gains_new_module": consumerSource += "\nfrom PIL.extra import later\ndef after(): return later()\n" - } else if operation == "provider_gains_empty_module" { + case "provider_gains_empty_module": consumerSource += "\nimport PIL.extra\n" } writeWorkspaceFile(t, consumer, "app.py", consumerSource) diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 85dfd4e99..7c28acf74 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -553,10 +553,6 @@ func (mi *MultiIndexer) runCrossRepoResolveFiles(filePaths []string, cargoRetarg mi.runCrossRepoResolveMutationFrontiers(filePaths, filePaths, filePaths, cargoRetargetRepo...) } -func (mi *MultiIndexer) runCrossRepoResolveMutationFiles(resolutionFiles, materializationFiles []string) { - mi.runCrossRepoResolveMutationFrontiers(resolutionFiles, materializationFiles, materializationFiles) -} - func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles []string, cargoRetargetRepo ...string) { if len(resolutionFiles) == 0 && len(edgeSourceFiles) == 0 && len(definitionFiles) == 0 { return diff --git a/internal/resolver/cargo_import_retarget_test.go b/internal/resolver/cargo_import_retarget_test.go index e88307dfd..15eb295ac 100644 --- a/internal/resolver/cargo_import_retarget_test.go +++ b/internal/resolver/cargo_import_retarget_test.go @@ -63,11 +63,12 @@ func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { } stats := &CrossRepoStats{ByRepo: map[string]int{}} edges := slices.Values([]*graph.Edge{edge}) - if scenario == "light_projection" { + switch scenario { + case "light_projection": edges = graph.EdgesLightSeq(g, graph.EdgeImports) - } else if scenario == "nil_input" { + case "nil_input": edges = slices.Values([]*graph.Edge{nil}) - } else if scenario == "wrong_kind" { + case "wrong_kind": projected := *edge projected.Kind = graph.EdgeCalls edges = slices.Values([]*graph.Edge{&projected}) From 013ef090450b481bd4bc7cc059a00bb804bfd9d3 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 19:27:32 -0700 Subject: [PATCH 030/111] style: use a tagged switch for the Python lifecycle track operation Staticcheck QF1003; golangci-lint had capped this repeat in the file. --- internal/indexer/python_import_lifecycle_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/indexer/python_import_lifecycle_test.go b/internal/indexer/python_import_lifecycle_test.go index 3b38bbd58..0acecbcb5 100644 --- a/internal/indexer/python_import_lifecycle_test.go +++ b/internal/indexer/python_import_lifecycle_test.go @@ -70,7 +70,8 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { logger := zap.New(core) mi := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) - if operation == "deferred_track" { + switch operation { + case "deferred_track": _, err = mi.TrackRepoCtx(context.Background(), entries[0]) require.NoError(t, err) mi.BeginParallelBatch() @@ -83,12 +84,12 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { } require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/app.py", "provider/src/PIL/image.py", "provider/src/PIL/util.py"}, nil)) mi.EndBatch() - } else if operation == "track" { + case "track": for _, entry := range gc.Repos { _, err = mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) } - } else { + default: _, err = mi.IndexAll() require.NoError(t, err) } From 948263168f690e59ef887b0d6ca325e2f78fe838 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:17:21 -0700 Subject: [PATCH 031/111] fix(resolver): bind bridge-linked calls only through the linked import file The Go, Cargo and Python bridges point a consumer's import at a provider file with CrossRepo set. buildReachableReposIndex then made the whole provider repo reachable for that file, so the name-only tiers could bind any same-named provider function or method, in map order. A call such as resp.Body.Close() could land on an unrelated provider Close. Keep bridge-linked imports out of repo-wide reachability, as npm links already are. Bind Cargo calls and typed Python methods only to a unique candidate in the exact linked file, with the imported item or receiver type; qualified Python calls also use the linked file. Generic non-bridge name tiers are unchanged. --- .../bridge_import_reachability_test.go | 148 ++++++++++++++++++ internal/resolver/cargo_crate.go | 55 +++++++ internal/resolver/cross_repo.go | 27 +++- internal/resolver/python_tracked_import.go | 46 ++++++ 4 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 internal/resolver/bridge_import_reachability_test.go diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go new file mode 100644 index 000000000..e33c3fc4e --- /dev/null +++ b/internal/resolver/bridge_import_reachability_test.go @@ -0,0 +1,148 @@ +package resolver + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestTrackedBridgeImportsDoNotAuthorizeBareNames(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, tc := range []struct{ language, file, provider, marker, spec string }{ + {"go", "main.go", "lib.go", "go_import", "example.org/lib"}, + {"rust", "src/lib.rs", "src/lib.rs", CargoImportStubMeta, "external::library/action"}, + {"python", "app.py", "pkg/image.py", PythonImportStubMeta, "external::pkg.image"}, + } { + t.Run(backend+"/"+tc.language, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file, provider := "consumer/"+tc.file, "provider/"+tc.provider + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "consumer", Language: tc.language}) + g.AddNode(&graph.Node{ID: file + "::run", Name: "run", Kind: graph.KindFunction, FilePath: file, RepoPrefix: "consumer", Language: tc.language}) + g.AddNode(&graph.Node{ID: provider, Kind: graph.KindFile, FilePath: provider, RepoPrefix: "provider", Language: tc.language}) + // Neither imported module exports hidden. Two other modules do. + for _, decoy := range []string{"provider/other", "provider/decoy"} { + g.AddNode(&graph.Node{ID: decoy + "::hidden", Name: "hidden", Kind: graph.KindFunction, FilePath: decoy, RepoPrefix: "provider", Language: tc.language}) + g.AddNode(&graph.Node{ID: decoy + "::Client.Close", Name: "Close", Kind: graph.KindMethod, FilePath: decoy, RepoPrefix: "provider", Language: tc.language, Meta: map[string]any{"receiver": "Client"}}) + } + g.AddEdge(&graph.Edge{From: file, To: provider, Kind: graph.EdgeImports, FilePath: file, Line: 1, CrossRepo: true, Meta: map[string]any{tc.marker: tc.spec}}) + // A foreign-typed resp.Body.Close() has no evidence for Client.Close. + for i, target := range []string{"unresolved::hidden", "unresolved::*.Close"} { + g.AddEdge(&graph.Edge{From: file + "::run", To: target, Kind: graph.EdgeCalls, FilePath: file, Line: i + 2, Meta: map[string]any{"receiver_type": "io.ReadCloser"}}) + } + cr := NewCrossRepo(g) + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + calls := outEdgesOfKind(g, file+"::run", graph.EdgeCalls) + require.Len(t, calls, 2) + for _, call := range calls { + assert.True(t, graph.IsUnresolvedTarget(call.To), "unimported symbol must stay unresolved: %s", call.To) + assert.False(t, call.CrossRepo) + } + }) + } + } +} + +func TestCargoCallsStayWithinImportedItem(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scenario := range []string{"function", "constructor", "method", "foreign_type", "ambiguous"} { + t.Run(backend+"/"+scenario, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file, imported := "consumer/src/lib.rs", "provider/src/util.rs" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "rust"}) + kind, name, target, item := graph.KindFunction, "action", "unresolved::action", "action" + meta := map[string]any{} + if scenario == "constructor" { + kind, name, target, item = graph.KindMethod, "new", "unresolved::new", "Foo" + meta["rust_path"] = "Foo::new" + } + if scenario == "method" || scenario == "foreign_type" { + kind, name, target, item = graph.KindMethod, "close", "unresolved::*.close", "Foo" + meta["receiver_type"] = "Foo" + if scenario == "foreign_type" { + meta["receiver_type"] = "Foreign" + } + } + // A same-named decoy in another module must never compete with util. + for _, path := range []string{"provider/src/aaa.rs", imported} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: "provider", Language: "rust"}) + g.AddNode(&graph.Node{ID: path + "::" + name, FilePath: path, Name: name, Kind: kind, RepoPrefix: "provider", Language: "rust", Meta: map[string]any{"receiver": "Foo"}}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: imported, Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::library/util/" + item}}) + if scenario == "ambiguous" { + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/src/aaa.rs", Kind: graph.EdgeImports, Line: 2, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::library/aaa/" + item}}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeCalls, Line: 3, Meta: meta}) + _, err := NewCrossRepo(g).ResolveAllContext(t.Context()) + require.NoError(t, err) + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 1) + if scenario == "foreign_type" || scenario == "ambiguous" { + require.Equal(t, target, calls[0].To) + require.False(t, calls[0].CrossRepo) + } else { + require.Equal(t, imported+"::"+name, calls[0].To) + require.True(t, calls[0].CrossRepo) + } + }) + } + } +} + +func TestPythonMethodsStayWithinImportedType(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scenario := range []string{"method", "foreign_type", "ambiguous"} { + t.Run(backend+"/"+scenario, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file := "consumer/app.py" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "python"}) + for _, provider := range []string{"provider/pkg/aaa.py", "provider/pkg/image.py"} { + g.AddNode(&graph.Node{ID: provider, FilePath: provider, Kind: graph.KindFile, RepoPrefix: "provider", Language: "python"}) + g.AddNode(&graph.Node{ID: provider + "::Image.show", FilePath: provider, Name: "show", Kind: graph.KindMethod, RepoPrefix: "provider", Language: "python", Meta: map[string]any{"receiver": "Image"}}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/pkg/image.py", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image"}}) + if scenario == "ambiguous" { + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/pkg/aaa.py", Kind: graph.EdgeImports, Line: 2, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.aaa", "python_import": "pkg.aaa"}}) + } + receiver := "Image" + if scenario == "foreign_type" { + receiver = "Foreign" + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::*.show", Kind: graph.EdgeCalls, Line: 3, Meta: map[string]any{"receiver_type": receiver}}) + _, err := NewCrossRepo(g).ResolveAllContext(t.Context()) + require.NoError(t, err) + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 1) + if scenario == "method" { + require.Equal(t, "provider/pkg/image.py::Image.show", calls[0].To) + require.True(t, calls[0].CrossRepo) + } else { + require.Equal(t, "unresolved::*.show", calls[0].To) + require.False(t, calls[0].CrossRepo) + } + }) + } + } +} diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index ff07551c1..3cfc2268d 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -191,3 +191,58 @@ func (cr *CrossRepoResolver) bindCargoImport(e *graph.Edge, node *graph.Node, st stats.CrossRepoEdges++ stats.ByRepo[node.RepoPrefix]++ } + +// A tracked use authorizes only its item or receiver in the linked module file. +func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target string) bool { + if e.Kind != graph.EdgeCalls || !strings.HasSuffix(e.FilePath, ".rs") { + return false + } + name := strings.TrimPrefix(target, "*.") + receiver := edgeReceiverType(e) + scoped, _ := e.Meta["rust_path"].(string) + if owner, _, ok := strings.Cut(scoped, "::"); ok { + receiver = owner + } + var match *graph.Node + for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { + stub, _ := imp.Meta[CargoImportStubMeta].(string) + if stub == "" { + continue + } + spec := strings.TrimPrefix(strings.TrimPrefix(stub, "external::"), "unresolved::import::") + item := spec[strings.LastIndex(spec, "/")+1:] + local := item + if imp.Alias != "" { + local = imp.Alias + } + for _, candidate := range cr.cachedFindNodesByName(name) { + if candidate.FilePath != imp.To || candidate.Language != "rust" || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { + continue + } + switch candidate.Kind { + case graph.KindFunction: + if strings.HasPrefix(target, "*.") || receiver != "" || (item != "*" && local != name) { + continue + } + case graph.KindMethod: + if receiver == "" || receiver != local || nodeReceiverType(candidate) != item { + continue + } + default: + continue + } + if match != nil && match.ID != candidate.ID { + return false + } + match = candidate + } + } + if match == nil { + return false + } + e.To, e.CrossRepo = match.ID, true + if strings.HasPrefix(target, "*.") { + e.Confidence, e.Origin = 0.85, graph.OriginASTResolved + } + return true +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 68fd1bfea..c1faedc3d 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -111,8 +111,9 @@ type CrossRepoResolver struct { // importedDirsByFile retains the package placement from those same import // rows. A Go call the per-repo package guard refused must not regain a // same-repo name-only binding just because the repository is reachable. - importedDirsByFile map[string]map[string]struct{} - jsTSBindingsByFile map[string]map[string]string + importedDirsByFile map[string]map[string]struct{} + jsTSBindingsByFile map[string]map[string]string + bridgeImportsByFile map[string][]*graph.Edge // depModuleIndex bridges Go imports to dep:: contract // nodes from the caller's go.mod. Same shape and rationale as // the field of the same name on Resolver — see resolver.go for @@ -861,6 +862,7 @@ func (cr *CrossRepoResolver) buildReachableReposIndex() { idx := make(map[string]map[string]struct{}) dirs := make(map[string]map[string]struct{}) cr.jsTSBindingsByFile = make(map[string]map[string]string) + cr.bridgeImportsByFile = make(map[string][]*graph.Edge) // Materialise metadata-free import projections and batch-load only target // placement fields. A per-edge point lookup here is a query round-trip per // import on a disk backend, which under the cross-repo pass's import @@ -887,12 +889,12 @@ func (cr *CrossRepoResolver) buildReachableReposIndex() { page := imports[start:min(start+resolvePendingPageRows, len(imports))] var sites []graph.EdgeSite for _, e := range page { - if e.CrossRepo && isJSTSPath(e.From) { + if e.CrossRepo && (isJSTSPath(e.From) || strings.HasSuffix(e.FilePath, ".go") || strings.HasSuffix(e.FilePath, ".rs") || isPythonSourcePath(e.FilePath)) { sites = append(sites, graph.EdgeSite{From: e.From, Line: e.Line, Kind: graph.EdgeImports}) } } // Markers belong to the persisted import, including imports outside a - // scoped resolution frontier. Hydrate only cross-repo JS/TS imports. + // scoped resolution frontier. Hydrate only cross-repo imports. full := cr.graph.GetEdgeCandidates(nil, sites) for _, e := range page { for _, candidate := range full.Site(e.From, e.Line, graph.EdgeImports) { @@ -941,7 +943,11 @@ func (cr *CrossRepoResolver) buildReachableReposIndex() { } } } - if npmLinked { + if e.Meta[CargoImportStubMeta] != nil || e.Meta[PythonImportStubMeta] != nil { + cr.bridgeImportsByFile[e.From] = append(cr.bridgeImportsByFile[e.From], e) + } + // A linked module file is not evidence for every symbol in its repo. + if npmLinked || e.Meta["go_import"] != nil || e.Meta[CargoImportStubMeta] != nil || e.Meta[PythonImportStubMeta] != nil { continue } set := idx[e.From] @@ -960,6 +966,7 @@ func (cr *CrossRepoResolver) clearReachableReposIndex() { cr.reachableReposByFile = nil cr.importedDirsByFile = nil cr.jsTSBindingsByFile = nil + cr.bridgeImportsByFile = nil } // A package-guard refusal is stronger than the cross-repo pass's same-repo @@ -1276,6 +1283,10 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b } switch { + case cr.resolveCargoImportedCall(e, target) || cr.resolvePythonImportedMethod(e, target): + stats.Resolved++ + stats.CrossRepoEdges++ + stats.ByRepo[graph.RepoPrefixOfID(e.To)]++ case strings.HasSuffix(e.FilePath, ".go") && strings.HasPrefix(target, "extern::"): cr.resolveGoExtern(e, strings.TrimPrefix(target, "extern::"), stats) case strings.HasPrefix(target, "import::"): @@ -1289,7 +1300,11 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b importedDirs := cr.importedDirsByFile[cr.callerFileID(e)] find := func(name string) ([]*graph.Node, error) { var eligible []*graph.Node - for _, node := range cr.scopedCandidates(e, name) { + for _, node := range cr.cachedFindNodesByName(name) { + if cr.linkedPythonCallCandidate(e, node, module) { + eligible = append(eligible, node) + continue + } _, imported := importedDirs[filePathDir(node.FilePath)] if imported && reachable(node.RepoPrefix) && cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(node), module) { eligible = append(eligible, node) diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go index c48b90af1..1ed03c3ba 100644 --- a/internal/resolver/python_tracked_import.go +++ b/internal/resolver/python_tracked_import.go @@ -273,3 +273,49 @@ func (cr *CrossRepoResolver) restubCallsForLinkedPythonImports(changes []graph.E } return reopened } + +// Qualified Python calls use the exact linked import, not repo-wide reachability. +func (cr *CrossRepoResolver) linkedPythonCallCandidate(e *graph.Edge, node *graph.Node, module string) bool { + for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { + spec, _ := imp.Meta["python_import"].(string) + if imp.Meta[PythonImportStubMeta] == nil || spec == "" || (module != spec && !strings.HasPrefix(module, spec+".")) { + continue + } + if node.FilePath == imp.To && cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(node), spec) { + return true + } + } + return false +} + +func (cr *CrossRepoResolver) resolvePythonImportedMethod(e *graph.Edge, target string) bool { + if e.Kind != graph.EdgeCalls || !isPythonSourcePath(e.FilePath) || !strings.HasPrefix(target, "*.") { + return false + } + receiver := edgeReceiverType(e) + if receiver == "" { + return false + } + var match *graph.Node + for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { + spec, _ := imp.Meta["python_import"].(string) + if imp.Meta[PythonImportStubMeta] == nil || !pythonModuleFileMatches(imp.To, spec) { + continue + } + for _, candidate := range cr.cachedFindNodesByName(strings.TrimPrefix(target, "*.")) { + if candidate.Kind != graph.KindMethod || candidate.FilePath != imp.To || nodeReceiverType(candidate) != receiver || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { + continue + } + if match != nil && match.ID != candidate.ID { + return false + } + match = candidate + } + } + if match == nil { + return false + } + e.To, e.CrossRepo = match.ID, true + e.Confidence, e.Origin = 0.85, graph.OriginASTResolved + return true +} From f225c3238bfeb9d81bb997688cb7eafce5713bb9 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:17:31 -0700 Subject: [PATCH 032/111] fix(indexer): close the import scan before the Python package lookup pythonConsumerFilesForRepo called the package lookup, which runs its own store query, while the light import scan still held its rows open. With a one-connection read pool the second query waits forever. Collect the caller, top and file tuples first, close the scan, then look up providers, as trackedPythonImportEdges does. --- internal/indexer/python_import_restub.go | 13 +++-- .../indexer/python_tracked_import_test.go | 49 +++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index 7b6c91f81..52b3dfc6b 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -116,6 +116,8 @@ func (mi *MultiIndexer) pythonConsumerFilesForRepo(repo string) []string { return nil } files := make(map[string]struct{}) + type candidate struct{ caller, top, file string } + var candidates []candidate for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { if !strings.HasSuffix(edge.FilePath, ".py") { continue @@ -127,10 +129,13 @@ func (mi *MultiIndexer) pythonConsumerFilesForRepo(repo string) []string { break } } - caller := graph.RepoPrefixOfID(edge.From) - for provider := range lookup.Providers(caller, top) { - if provider == repo || caller == repo { - files[edge.FilePath] = struct{}{} + candidates = append(candidates, candidate{graph.RepoPrefixOfID(edge.From), top, edge.FilePath}) + } + // Providers may query the store; release the import cursor first. + for _, item := range candidates { + for provider := range lookup.Providers(item.caller, item.top) { + if provider == repo || item.caller == repo { + files[item.file] = struct{}{} break } } diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go index ee33983d5..9b2698ecf 100644 --- a/internal/indexer/python_tracked_import_test.go +++ b/internal/indexer/python_tracked_import_test.go @@ -240,3 +240,52 @@ func TestPythonAddedFileConsumerFrontier(t *testing.T) { require.Equal(t, []string{"consumer/a.py", "consumer/b.py", "consumer/c.py"}, pythonConsumerFilesForAddedFiles(g, []string{"provider/src/PIL/extra.py"})) require.Equal(t, 1, g.importScans) } + +// A one-connection store cannot hydrate package identities under an open cursor. +type pythonCursorGuardStore struct { + graph.Store + t *testing.T + cursorOpen bool +} + +func (s *pythonCursorGuardStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + return func(yield func(*graph.Edge) bool) { + s.cursorOpen = true + defer func() { s.cursorOpen = false }() + for edge := range graph.EdgesLightSeq(s.Store, kinds...) { + if !yield(edge) { + return + } + } + } +} + +func (s *pythonCursorGuardStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { + require.False(s.t, s.cursorOpen, "package lookup must not query while the import cursor is open") + return graph.FileNodeIdentitiesSeq(s.Store, repos) +} + +func TestPythonConsumerLookupClosesImportCursor(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var store graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store = db + } + g := &pythonCursorGuardStore{Store: store, t: t} + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'library'\n") + writeWorkspaceFile(t, provider, "mylib.py", "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['library']\n") + for _, file := range []string{"provider/mylib.py", "consumer/app.py"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "python"}) + } + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: "external::mylib", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + require.Equal(t, []string{"consumer/app.py"}, mi.pythonConsumerFilesForRepo("provider")) + }) + } +} From 7ea03278c946c05ed80bf33b9d87fe30c00b61ff Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:17:44 -0700 Subject: [PATCH 033/111] fix(indexer): keep same-repo Go replacement imports on single-file saves The single-file restub turned a go_import-marked import into unresolved:: when the import stayed in the caller's repo (a tracked local replace), and nothing rebinds that form. It also admitted every reference kind into file nodes. The batch path carries the import and leaves file references to eviction. Keep the package identity for marked imports regardless of CrossRepo, carry marked same-repo imports whose target file survives, and stop restubbing other edge kinds into file nodes, so single-file, batch and cold indexing agree. --- .../indexer/go_module_import_recovery_test.go | 90 +++++++++++++++++++ internal/indexer/go_module_restub.go | 6 +- internal/indexer/indexer.go | 19 +++- 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/internal/indexer/go_module_import_recovery_test.go b/internal/indexer/go_module_import_recovery_test.go index 17067ff7a..dcbb2b6d1 100644 --- a/internal/indexer/go_module_import_recovery_test.go +++ b/internal/indexer/go_module_import_recovery_test.go @@ -316,3 +316,93 @@ func TestGoModuleImportDuplicateAliases(t *testing.T) { } } } + +func TestGoModuleReplacementFileSaveColdParity(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"file", "batch", "fallback"} { + t.Run(backend+"/"+route, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + replacement := filepath.Join(repos[0].Path, "third_party", "library", "lib.go") + require.NoError(t, os.MkdirAll(filepath.Dir(replacement), 0o755)) + writeFile(t, filepath.Join(repos[0].Path, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\nreplace example.org/library => ./third_party/library\n") + writeFile(t, filepath.Join(repos[0].Path, "main.go"), "package caller\nimport library \"example.org/library\"\nfunc Caller() { library.Hello() }\n") + writeFile(t, replacement, "package library\nfunc Hello() {}\n") + for _, entry := range repos { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + // Reproduce a persisted same-repo binding emitted by the module bridge. + for _, imp := range g.GetOutEdges("a/main.go") { + if imp.Kind != graph.EdgeImports { + continue + } + imp.Meta = map[string]any{"go_import": "example.org/library", "go_import_fallback": "dep::example.org/library"} + g.ReindexEdges([]graph.EdgeReindex{{Edge: imp, OldTo: imp.To}}) + } + rows := func(store graph.Store) []string { + var result []string + for _, imp := range store.GetOutEdges("a/main.go") { + if imp.Kind == graph.EdgeImports { + result = append(result, fmt.Sprintf("L%d %s cross=%v", imp.Line, imp.To, imp.CrossRepo)) + } + } + sort.Strings(result) + return result + } + before := rows(g) + require.Len(t, before, 1) + require.Contains(t, before[0], "a/third_party/library/lib.go cross=false") + writeFile(t, replacement, "package library\nfunc Hello() { println(1) }\n") + if route == "fallback" { + require.NoError(t, mi.GetIndexer("a").indexFile(replacement, true, nil)) + } else if route == "file" { + require.NoError(t, mi.GetIndexer("a").IndexFile(replacement)) + } else { + _, err := mi.IncrementalReindexRepo("a", []string{replacement}) + require.NoError(t, err) + } + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "replacement-cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + for _, entry := range repos { + _, err := cold.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + require.Equal(t, before, rows(coldGraph)) + require.Equal(t, rows(coldGraph), rows(g), "provider save must match batch and cold imports") + }) + } + } +} + +func TestSingleFileRestubLeavesFileReferencesForEviction(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file, page := "app/+page.server.ts", "app/+page.svelte" + g.AddNode(&graph.Node{ID: file, Name: "+page.server.ts", FilePath: file, Kind: graph.KindFile}) + g.AddNode(&graph.Node{ID: page, Name: "+page.svelte", FilePath: page, Kind: graph.KindFile}) + // SvelteKit load synthesis references the server file, not a named symbol. + g.AddEdge(&graph.Edge{From: page, To: file, FilePath: page, Kind: graph.EdgeReferences}) + idx := &Indexer{graph: g} + idx.restubIncomingRefs(file) + refs := g.GetOutEdges(page) + require.Len(t, refs, 1) + require.Equal(t, file, refs[0].To, "file references must not become unresolvable name stubs") + g.EvictFile(file) + require.Empty(t, g.GetOutEdges(page)) + }) + } +} diff --git a/internal/indexer/go_module_restub.go b/internal/indexer/go_module_restub.go index 9c65ab770..fb2b193d9 100644 --- a/internal/indexer/go_module_restub.go +++ b/internal/indexer/go_module_restub.go @@ -7,11 +7,13 @@ import ( ) func goModuleRestubTarget(e *graph.Edge, fallback string) string { - if spec, ok := e.Meta["go_import"].(string); ok && spec != "" && e.CrossRepo && strings.HasSuffix(e.FilePath, ".go") { + if spec, ok := e.Meta["go_import"].(string); ok && spec != "" && strings.HasSuffix(e.FilePath, ".go") { if e.Kind == graph.EdgeImports { return graph.UnresolvedMarker + "import::" + spec } - return graph.UnresolvedMarker + "extern::" + spec + if e.CrossRepo { + return graph.UnresolvedMarker + "extern::" + spec + } } return fallback } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 6e8983aae..c45f3dace 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -5157,8 +5157,16 @@ func (idx *Indexer) indexFile( priorNodes, legacyIDs, legacyPath := idx.filePriorWithLegacy(graphPath, idx.graph.GetFileNodes(graphPath)) var legacyCarried []*graph.Edge var oldFuncIDs []string + var carriedGoImports []*graph.Edge evictExisting := func(result *parser.ExtractionResult) { idx.invalidateGoModuleSourceStamp(relPath) + if strings.HasSuffix(graphPath, ".go") { + for _, e := range idx.graph.GetInEdges(graphPath) { + if e.Kind == graph.EdgeImports && !e.CrossRepo && e.Meta["go_import"] != nil && e.FilePath != graphPath { + carriedGoImports = append(carriedGoImports, e) + } + } + } var oldFTSNodeIDs []string for _, n := range priorNodes { if n == nil { @@ -5448,7 +5456,10 @@ func (idx *Indexer) indexFile( // the node. idx.replaceContentSections(graphPath, result.Nodes, false) - idx.graph.AddBatch(result.Nodes, append(result.Edges, legacyCarried...)) + if _, survives := newNodeIDs[graphPath]; !survives { + carriedGoImports = nil + } + idx.graph.AddBatch(result.Nodes, append(append(result.Edges, legacyCarried...), carriedGoImports...)) idx.persistFileMeta(relPath, src, result) idx.persistConstValues(result) // No subsequent stage reads source bytes or the parse tree. Release both @@ -5994,12 +6005,18 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string, [] } stub := graph.UnresolvedMarker + n.Name for _, e := range inEdges[n.ID] { + if n.Kind == graph.KindFile && (e == nil || e.Kind != graph.EdgeImports || e.Meta["go_import"] == nil) { + continue + } if e == nil || !graph.IsResolvableRefEdge(e.Kind) && (e.Kind != graph.EdgeImports || e.Meta["go_import"] == nil) { continue } if _, fromEvicted := evicted[e.From]; fromEvicted { continue // intra-file edge: the source is evicted too } + if e.Kind == graph.EdgeImports && !e.CrossRepo && e.Meta["go_import"] != nil { + continue // The single-file save carries surviving local file imports. + } if graph.IsUnresolvedTarget(e.To) { continue // already a pending stub } From 2e12eafe0e955d5576985cf015bf80f245f22c39 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:17:51 -0700 Subject: [PATCH 034/111] fix(resolver): skip an oversized provider without starving later import recovery IncomingSourceBudget.Charge adds the rows before it compares them with the ceiling, so one refused charge left the count above it and every later repo's charge failed too. Check Remaining first in both import-recovery loops, so one oversized registry is skipped and the next small provider still restores its imports. --- internal/resolver/cross_repo_incremental.go | 4 +++ .../resolver/go_module_import_seed_test.go | 33 +++++++++++++++++++ internal/resolver/go_modules_cross_repo.go | 4 +++ 3 files changed, 41 insertions(+) diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 0772a0f62..0f812cfe9 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -186,6 +186,10 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed } repos[repo] = true ids := cr.goModuleImportIDs(repo) + if len(ids) > importBudget.Remaining() { + cr.logger.Warn("cross-repo resolve: Go module imports deferred to full pass", zap.Int("imports", len(ids)), zap.Int("remaining", importBudget.Remaining())) + continue + } if err := importBudget.Charge(len(ids)); err != nil { cr.logger.Warn("cross-repo resolve: Go module imports deferred to full pass", zap.Error(err)) continue diff --git a/internal/resolver/go_module_import_seed_test.go b/internal/resolver/go_module_import_seed_test.go index c3606c377..799314522 100644 --- a/internal/resolver/go_module_import_seed_test.go +++ b/internal/resolver/go_module_import_seed_test.go @@ -87,3 +87,36 @@ func TestGoModuleImportLargeRestartSeed(t *testing.T) { } } } + +func TestGoModuleImportOversizedProviderDoesNotStarveNext(t *testing.T) { + for _, route := range []string{"frontier", "identity"} { + t.Run(route, func(t *testing.T) { + g := graph.New() + for _, file := range []string{"a-big/lib.go", "b-small/lib.go", "consumer/main.go"} { + g.AddNode(&graph.Node{ID: file, Name: "lib.go", FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "go"}) + } + const id = "consumer/main.go::import::example.org/small" + g.AddNode(&graph.Node{ID: id, Name: "small", FilePath: "consumer/main.go", StartLine: 2, Kind: graph.KindImport, RepoPrefix: "consumer", Language: "go", Meta: map[string]any{"path": "example.org/small"}}) + cr := NewCrossRepo(g) + modules := map[string]string{"a-big": "example.org/big", "b-small": "example.org/small", "consumer": "example.org/app"} + cr.SetGoModules(modules) + state := cr.goModuleImportState() + state.importsSeeded, state.complete = true, true + state.modules = modules + state.imports["a-big"] = make(map[string]struct{}) + for i := 0; i <= graph.MaxIncomingSourceCandidateRows; i++ { + state.imports["a-big"][fmt.Sprintf("missing/%d", i)] = struct{}{} + } + state.imports["b-small"] = map[string]struct{}{id: {}} + if route == "frontier" { + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"a-big/lib.go", "b-small/lib.go"}) + require.False(t, admission.Refused) + } else { + require.NoError(t, cr.recheckGoModuleBindings(t.Context(), nil, &CrossRepoStats{ByRepo: map[string]int{}})) + } + imports := outEdgesOfKind(g, "consumer/main.go", graph.EdgeImports) + require.Len(t, imports, 1, "the small provider must still restore its import") + require.Equal(t, "b-small/lib.go", imports[0].To) + }) + } +} diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index b69febe17..fd59d2211 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -665,6 +665,10 @@ func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previo continue } importIDs := cr.goModuleImportIDs(repo) + if len(importIDs) > importBudget.Remaining() { + cr.logger.Warn("Go module import identity recheck deferred to full pass", zap.Int("imports", len(importIDs)), zap.Int("remaining", importBudget.Remaining())) + continue + } if err := importBudget.Charge(len(importIDs)); err != nil { cr.logger.Warn("Go module import identity recheck deferred to full pass", zap.Error(err)) continue From c3d84987f1cfd0ea5ac37db3dde23b0bce0958ca Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:18:01 -0700 Subject: [PATCH 035/111] fix(resolver): let the longest tracked Go module own nested imports The full scan and the restart seed skipped any import under the caller's own module path, even when a nested module tracked by another repo owns it. The warm path linked such imports, so cold and warm graphs differed. Skip only when the longest tracked module is the caller's own. --- .../resolver/go_module_import_seed_test.go | 35 +++++++++++++++++++ internal/resolver/go_module_imports.go | 4 +-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/internal/resolver/go_module_import_seed_test.go b/internal/resolver/go_module_import_seed_test.go index 799314522..443cb3cfb 100644 --- a/internal/resolver/go_module_import_seed_test.go +++ b/internal/resolver/go_module_import_seed_test.go @@ -120,3 +120,38 @@ func TestGoModuleImportOversizedProviderDoesNotStarveNext(t *testing.T) { }) } } + +func TestGoModuleNestedProviderFullAndWarmParity(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"full", "frontier", "seed"} { + t.Run(backend+"/"+route, func(t *testing.T) { + g := goImportReviewStore(t, backend) + const file = "a/main.go" + const spec = "example.org/root/child/sub" + const id = file + "::import::" + spec + g.AddBatch([]*graph.Node{ + {ID: file, Name: "main.go", FilePath: file, Kind: graph.KindFile, RepoPrefix: "a", Language: "go"}, + {ID: id, Name: "sub", FilePath: file, StartLine: 2, Kind: graph.KindImport, RepoPrefix: "a", Language: "go", Meta: map[string]any{"path": spec}}, + {ID: "b/sub/lib.go", Name: "lib.go", FilePath: "b/sub/lib.go", Kind: graph.KindFile, RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{{From: file, To: "dep::example.org/root/child", FilePath: file, Line: 2, Kind: graph.EdgeImports}}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"a": "example.org/root", "b": "example.org/root/child"}) + if route == "seed" { + require.NoError(t, cr.seedGoModuleImportRegistry(t.Context())) + require.Contains(t, cr.goModuleRetry.imports["b"], id) + return + } + if route == "full" { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + } else { + cr.ResolveMutationFiles([]string{file}, nil) + } + imports := outEdgesOfKind(g, file, graph.EdgeImports) + require.Len(t, imports, 1) + require.Equal(t, "b/sub/lib.go", imports[0].To) + require.True(t, imports[0].CrossRepo) + }) + } + } +} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 01aa77bec..4e7ee3392 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -234,7 +234,7 @@ func (cr *CrossRepoResolver) goModuleImportFileBatches(ctx context.Context, repo if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModulePaths, path) || seen[file] { continue } - if module := cr.goModules[n.RepoPrefix]; module != "" && (path == module || strings.HasPrefix(path, module+"/")) { + if module := cr.goModules[n.RepoPrefix]; module != "" && cr.goModulePaths.longest(path) == module { continue } seen[file] = true @@ -300,7 +300,7 @@ func (cr *CrossRepoResolver) seedGoModuleImportRegistry(ctx context.Context) err if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModulePaths, path) { continue } - if module := cr.goModules[n.RepoPrefix]; module != "" && (path == module || strings.HasPrefix(path, module+"/")) { + if module := cr.goModules[n.RepoPrefix]; module != "" && cr.goModulePaths.longest(path) == module { continue } cr.rememberGoModuleImports([]*graph.Node{{ID: n.ID, RepoPrefix: n.RepoPrefix, Kind: graph.KindImport, Language: "go"}}) From a77b5f36d0abb3bee62d9f8e5343d28ddee82b6a Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:18:10 -0700 Subject: [PATCH 036/111] fix(resolver): restore a revisited Cargo import when its target is refused A bound Rust import was skipped by the declaration prefilter, and when it was revisited a nil target left the old binding in place. A cold index refuses the same import. Revisit bound imports past the prefilter and restore CargoImportStubMeta when the target is now undeclared or ambiguous. --- internal/resolver/cargo_crate.go | 14 +++++++-- .../resolver/cargo_import_retarget_test.go | 29 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index 3cfc2268d..57b3c5558 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -118,7 +118,7 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter } if spec != "" { name, _, _ := strings.Cut(spec, "/") - if len(cr.cargoCrates(e.FilePath, name)) == 0 { + if len(cr.cargoCrates(e.FilePath, name)) == 0 && !e.CrossRepo { continue } } @@ -142,7 +142,17 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter if e.To != projected.To { continue } - if node := cr.cargoImportTarget(e, cr.cargoImportSpec(e)); node != nil && node.ID != e.To { + node := cr.cargoImportTarget(e, cr.cargoImportSpec(e)) + if node == nil && e.CrossRepo { + if stub, _ := e.Meta[CargoImportStubMeta].(string); stub != "" { + oldTo := e.To + e.To, e.CrossRepo = stub, false + clearEdgeTerminal(e) + batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) + projected.To, projected.CrossRepo = e.To, false + stats.Unresolved++ + } + } else if node != nil && node.ID != e.To { oldTo := e.To cr.bindCargoImport(e, node, stats) batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) diff --git a/internal/resolver/cargo_import_retarget_test.go b/internal/resolver/cargo_import_retarget_test.go index 15eb295ac..9e60e9d2b 100644 --- a/internal/resolver/cargo_import_retarget_test.go +++ b/internal/resolver/cargo_import_retarget_test.go @@ -98,3 +98,32 @@ func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { } } } + +func TestCargoRevisitedImportRestoresStubOnRefusal(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, reason := range []string{"undeclared", "ambiguous"} { + for _, stub := range []string{"external::other_crate/util", "unresolved::import::other_crate/util"} { + t.Run(backend+"/"+reason+"/"+stub, func(t *testing.T) { + g := goImportReviewStore(t, backend) + for _, file := range []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs", "fork/src/lib.rs"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "rust"}) + } + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "provider/src/util.rs", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: stub}}) + cr := NewCrossRepo(g) + cr.SetCargoImportRetargetRepo("provider") + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { + if reason == "undeclared" { + return nil + } + return []CargoCrate{{Root: "provider/src/lib.rs", Name: "other_crate"}, {Root: "fork/src/lib.rs", Name: "other_crate"}} + }) + require.NoError(t, cr.resolveCargoImports(t.Context(), graph.EdgesLightSeq(g, graph.EdgeImports), &CrossRepoStats{ByRepo: map[string]int{}})) + imports := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeImports) + require.Len(t, imports, 1) + require.Equal(t, stub, imports[0].To) + require.False(t, imports[0].CrossRepo) + }) + } + } + } +} From 8240ed025882299ecfeb83321119168b97c7dcab Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:18:28 -0700 Subject: [PATCH 037/111] fix(resolver): reopen top-level Python calls parked as stdlib For `import mylib; mylib.f()` resolved before the provider was tracked, the call takes a stdlib terminal, because the Go-style stdlib test treats a dotless name as standard library. Reopening after the import links matched only dep terminals, so the call stayed parked. Accept stdlib terminals too, still keyed by the exact linked module and the call stub. --- .../resolver/python_import_call_retry_test.go | 28 +++++++++++++++++++ internal/resolver/python_tracked_import.go | 5 ++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/internal/resolver/python_import_call_retry_test.go b/internal/resolver/python_import_call_retry_test.go index 0629218ab..5ca8ac059 100644 --- a/internal/resolver/python_import_call_retry_test.go +++ b/internal/resolver/python_import_call_retry_test.go @@ -177,3 +177,31 @@ func TestTrackedPythonScopedRetryVisitsEachCallOnce(t *testing.T) { require.Equal(t, 2, stats.Resolved) require.Zero(t, stats.Unresolved) } + +func TestTrackedPythonTopLevelCallReopensStdlibTerminal(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, terminal := range []string{"stdlib::mylib::f", "external-call::stdlib::mylib"} { + t.Run(backend+"/"+terminal, func(t *testing.T) { + g := goImportReviewStore(t, backend) + const file = "consumer/app.py" + const stub = "unresolved::extern::mylib::f" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, To: terminal, Line: 2, Meta: map[string]any{PythonCallStubMeta: stub, "external_call": true}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, To: "external-call::stdlib::other", Line: 3, Meta: map[string]any{PythonCallStubMeta: "unresolved::extern::other::f"}}) + imp := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeImports, To: "provider/mylib.py", CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::mylib", "python_import": "mylib"}} + cr := NewCrossRepo(g) + reopened := cr.restubCallsForLinkedPythonImports([]graph.EdgeReindex{{Edge: imp}}) + require.Len(t, reopened, 1) + require.Equal(t, stub, reopened[0].To) + require.NotContains(t, reopened[0].Meta, "external_call") + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 2) + for _, call := range calls { + if call.Line == 3 { + require.Equal(t, "external-call::stdlib::other", call.To) + } + } + }) + } + } +} diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go index 1ed03c3ba..a33b1b815 100644 --- a/internal/resolver/python_tracked_import.go +++ b/internal/resolver/python_tracked_import.go @@ -252,9 +252,10 @@ func (cr *CrossRepoResolver) restubCallsForLinkedPythonImports(changes []graph.E // Cold master resolution leaves dep:: terminals until externalization; // incremental tracking has already synthesized external-call:: nodes. callTarget := call.To - if ecosystem, module, ok := parseExternalCallTarget(callTarget); ok && ecosystem == "dep" { - callTarget = externalCallNodeID(ecosystem, module) + if ecosystem, module, ok := parseExternalCallTarget(callTarget); ok && (ecosystem == "dep" || ecosystem == "stdlib") { + callTarget = externalCallNodeID("dep", module) } + callTarget = strings.Replace(callTarget, externalCallPrefix+"stdlib::", externalCallPrefix+"dep::", 1) target := externalCallPrefix + "dep::" + spec if original == "" || (callTarget != target && !strings.HasPrefix(callTarget, target+".")) { continue From 81e7b502e9e92819d1cdf9ec079a918bf9d355a3 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:18:38 -0700 Subject: [PATCH 038/111] fix(indexer): rewrite bridge call stubs when carrying prior bindings The prior-binding carry copied the old edge's resolution metadata as is, so cargo_call_stub and python_call_stub kept the old spelling on a fresh edge, and a later provider eviction restored the stale target. Rewrite those keys to the fresh unresolved target, as applyReuseSemanticMeta already does. --- internal/indexer/edit_delta_prior_bindings.go | 8 ++++++- .../indexer/edit_delta_prior_bindings_test.go | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/internal/indexer/edit_delta_prior_bindings.go b/internal/indexer/edit_delta_prior_bindings.go index 051e0ac43..90eb64b35 100644 --- a/internal/indexer/edit_delta_prior_bindings.go +++ b/internal/indexer/edit_delta_prior_bindings.go @@ -169,11 +169,17 @@ func stagePriorBindings(g graph.Store, stages []*incrementalBatchStage, view inc if ambiguous || chosen == nil { continue } + meta := resolutionMetaAdded(chosen.edge.Meta, edge.Meta) + for _, key := range reuseBindingMetaKeys { + if _, kept := meta[key]; kept { + meta[key] = edge.To + } + } out[resolver.PriorBindingKey(edge.From, edge.Kind, edge.Line, edge.To)] = resolver.PriorBinding{ To: chosen.target, Origin: chosen.edge.Origin, Confidence: chosen.edge.Confidence, - Meta: resolutionMetaAdded(chosen.edge.Meta, edge.Meta), + Meta: meta, } } } diff --git a/internal/indexer/edit_delta_prior_bindings_test.go b/internal/indexer/edit_delta_prior_bindings_test.go index ad7137319..71754b1d7 100644 --- a/internal/indexer/edit_delta_prior_bindings_test.go +++ b/internal/indexer/edit_delta_prior_bindings_test.go @@ -13,6 +13,8 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/resolver" ) // A save that shifts every later line of the file keeps the file's unchanged @@ -81,3 +83,24 @@ func TestPriorBindingsCarryUnchangedReferencesAcrossAReparse(t *testing.T) { require.Equal(t, rows(fresh), rows(g), "the carried rows must equal a whole index of the edited tree") require.Contains(t, strings.Join(rows(g), "\n"), "-calls-> "+builderRepoPrefix+"/p/a.go::Later", "the reference the edit made resolvable is bound") } + +func TestPriorBindingsKeepFreshBridgeCallSpelling(t *testing.T) { + for _, key := range []string{"cargo_call_stub", "python_call_stub"} { + t.Run(key, func(t *testing.T) { + g := graph.New() + caller := &graph.Node{ID: "app.py::run", Name: "run", FilePath: "app.py", Kind: graph.KindFunction, StartLine: 1} + provider := &graph.Node{ID: "provider.py::f", Name: "f", FilePath: "provider.py", Kind: graph.KindFunction} + prior := &graph.Edge{From: caller.ID, To: provider.ID, Kind: graph.EdgeCalls, FilePath: "app.py", Line: 2, CrossRepo: true, Meta: map[string]any{key: "unresolved::old.f", "keep": "metadata"}} + g.AddBatch([]*graph.Node{caller, provider}, []*graph.Edge{prior}) + fresh := &graph.Edge{From: caller.ID, To: "unresolved::new.f", Kind: graph.EdgeCalls, FilePath: "app.py", Line: 2} + stage := &incrementalBatchStage{graphPath: "app.py", priorNodes: []*graph.Node{caller}, result: &parser.ExtractionResult{Nodes: []*graph.Node{caller}, Edges: []*graph.Edge{fresh}}, reuse: map[reuseKey]*reuseVal{}} + stages := []*incrementalBatchStage{stage} + carried := stagePriorBindings(g, stages, loadIncrementalPriorView(g, stages)) + binding, ok := carried[resolver.PriorBindingKey(fresh.From, fresh.Kind, fresh.Line, fresh.To)] + require.True(t, ok, "the prior binding must carry") + require.Equal(t, fresh.To, binding.Meta[key]) + require.Equal(t, "metadata", binding.Meta["keep"]) + require.Equal(t, "unresolved::old.f", prior.Meta[key], "carry must not change prior metadata") + }) + } +} From e065aff51ee050729d424f56722b7a6630d40afc Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:18:48 -0700 Subject: [PATCH 039/111] fix(indexer): recover Cargo consumers whose import stub is unresolved::import Provider recovery collected Cargo consumers only for external:: targets, but the cross-repo import path can leave CargoImportStubMeta as unresolved::import::. Those imports waited for a full pass. Accept both forms, as the Python collector does. --- internal/indexer/cargo_import_restub.go | 3 +++ internal/indexer/cargo_import_restub_test.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go index 3952c7adf..4d0267093 100644 --- a/internal/indexer/cargo_import_restub.go +++ b/internal/indexer/cargo_import_restub.go @@ -98,6 +98,9 @@ func (mi *MultiIndexer) cargoConsumerFilesForRepo(repo string, retargetBound boo } for edge := range edges { spec, external := strings.CutPrefix(edge.To, "external::") + if !external { + spec, external = strings.CutPrefix(edge.To, "unresolved::import::") + } if !external || !strings.HasSuffix(edge.FilePath, ".rs") { stub, _ := edge.Meta[resolver.CargoImportStubMeta].(string) if retargetBound && stub != "" && edge.CrossRepo && strings.HasSuffix(edge.FilePath, ".rs") && graph.RepoPrefixOfID(edge.To) == repo { diff --git a/internal/indexer/cargo_import_restub_test.go b/internal/indexer/cargo_import_restub_test.go index 5b2c99cd4..1e5332da1 100644 --- a/internal/indexer/cargo_import_restub_test.go +++ b/internal/indexer/cargo_import_restub_test.go @@ -49,3 +49,20 @@ func TestCargoRestubKeepsSameRepoCallsOnNativePath(t *testing.T) { }) } } + +func TestCargoConsumerFrontierIncludesUnresolvedImportStubs(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname='library'\nversion='1.0.0'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname='app'\nversion='1.0.0'\n[dependencies]\nlibrary='1'\n") + g := graph.New() + g.AddNode(&graph.Node{ID: "provider/src/lib.rs", FilePath: "provider/src/lib.rs", Kind: graph.KindFile, RepoPrefix: "provider", Language: "rust"}) + for _, tc := range []struct{ file, target string }{ + {"consumer/a.rs", "external::library/action"}, + {"consumer/b.rs", "unresolved::import::library/action"}, + {"consumer/c.rs", "unresolved::import::unknown/action"}, + } { + g.AddEdge(&graph.Edge{From: tc.file, FilePath: tc.file, To: tc.target, Kind: graph.EdgeImports}) + } + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + require.Equal(t, []string{"consumer/a.rs", "consumer/b.rs"}, mi.cargoConsumerFilesForRepo("provider", false)) +} From 60e1dfbb27ba225ac70031014b8bd237c345c823 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:18:56 -0700 Subject: [PATCH 040/111] fix(resolver): restore every duplicate-alias Go import statement Recovery checked and restored the import only at the shared import node's last line. With the same package imported under two aliases, a deleted first-alias edge was not recreated until the consumer was re-parsed. Use the file's containment lines for that import node to check and restore each statement. --- .../go_module_import_recovery_test.go | 32 ++++++++ internal/resolver/go_module_imports.go | 73 +++++++++++-------- 2 files changed, 73 insertions(+), 32 deletions(-) diff --git a/internal/resolver/go_module_import_recovery_test.go b/internal/resolver/go_module_import_recovery_test.go index 92286fe8a..71d663ab2 100644 --- a/internal/resolver/go_module_import_recovery_test.go +++ b/internal/resolver/go_module_import_recovery_test.go @@ -173,3 +173,35 @@ func TestGoModuleImportSeedCompletionAndCancellation(t *testing.T) { }) } } + +func TestGoModuleImportRestoresEveryAliasStatement(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + const file = "a/main.go" + const id = file + "::import::example.org/lib" + g.AddBatch([]*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "a", Language: "go"}, + {ID: id, FilePath: file, StartLine: 3, Kind: graph.KindImport, RepoPrefix: "a", Language: "go", Meta: map[string]any{"path": "example.org/lib"}}, + {ID: "b/lib.go", FilePath: "b/lib.go", Kind: graph.KindFile, RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{ + {From: file, To: id, Kind: graph.EdgeContains, FilePath: file, Line: 2}, + {From: file, To: id, Kind: graph.EdgeContains, FilePath: file, Line: 3}, + {From: "other.go", To: id, Kind: graph.EdgeContains, FilePath: "other.go", Line: 4}, + {From: file, To: "b/lib.go", Kind: graph.EdgeImports, FilePath: file, Line: 3, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/lib"}}, + }) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"a": "example.org/app", "b": "example.org/lib"}) + for range 2 { + require.NoError(t, cr.restoreGoModuleImportBatch(t.Context(), "b", []string{id}, &CrossRepoStats{ByRepo: map[string]int{}})) + } + var lines []int + for _, edge := range outEdgesOfKind(g, file, graph.EdgeImports) { + require.Equal(t, "b/lib.go", edge.To) + lines = append(lines, edge.Line) + } + sort.Ints(lines) + require.Equal(t, []int{2, 3}, lines, "one import per retained statement, with no duplicates") + }) + } +} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 4e7ee3392..37394692a 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -420,42 +420,51 @@ func (cr *CrossRepoResolver) restoreGoModuleImportBatch(ctx context.Context, rep delete(state.imports[repo], id) continue } - present := false - for _, e := range outgoing[n.FilePath] { - if e.Kind != graph.EdgeImports || e.Line != n.StartLine { - continue - } - target := graph.UnresolvedName(e.To) - if target == "" { - target = e.To - } - if strings.HasPrefix(strings.TrimPrefix(target, externalCallPrefix), "dep::") && fileNodes == nil { - fileNodes = cr.graph.GetFileNodesByPaths(files) - } - if goModuleImportPath(e, fileNodes[n.FilePath], outgoing[n.FilePath]) == path { - present = true - break + lines := map[int]bool{n.StartLine: true} + for _, edge := range outgoing[n.FilePath] { + if edge.Kind == graph.EdgeContains && edge.From == n.FilePath && edge.To == n.ID { + lines[edge.Line] = true } - replacement, _ := goModuleReplacementFor(cr.goModuleReplacements, n.RepoPrefix, path) - if replacement.Directory != "" && graph.RepoPrefixOfID(e.To) == n.RepoPrefix && - strings.HasSuffix(e.To, ".go") && filePathDir(e.To) == replacement.Directory { - present = true - break + } + for line := range lines { + present := false + for _, e := range outgoing[n.FilePath] { + if e.Kind != graph.EdgeImports || e.Line != line { + continue + } + target := graph.UnresolvedName(e.To) + if target == "" { + target = e.To + } + if strings.HasPrefix(strings.TrimPrefix(target, externalCallPrefix), "dep::") && fileNodes == nil { + fileNodes = cr.graph.GetFileNodesByPaths(files) + } + if goModuleImportPath(e, fileNodes[n.FilePath], outgoing[n.FilePath]) == path { + present = true + break + } + replacement, _ := goModuleReplacementFor(cr.goModuleReplacements, n.RepoPrefix, path) + if replacement.Directory != "" && graph.RepoPrefixOfID(e.To) == n.RepoPrefix && + strings.HasSuffix(e.To, ".go") && filePathDir(e.To) == replacement.Directory { + present = true + break + } + // A longer module may own the bound package, even when this + // statement is also registered under the edited provider. + targetRepo := graph.RepoPrefixOfID(e.To) + targetModule := cr.goModules[targetRepo] + if strings.HasSuffix(e.To, ".go") && targetModule != "" && + (path == targetModule || strings.HasPrefix(path, targetModule+"/")) && + filePathDir(e.To) == targetRepo+strings.TrimPrefix(path, targetModule) { + present = true + break + } } - // A longer module may own the bound package, even when this - // statement is also registered under the edited provider. - targetRepo := graph.RepoPrefixOfID(e.To) - targetModule := cr.goModules[targetRepo] - if strings.HasSuffix(e.To, ".go") && targetModule != "" && - (path == targetModule || strings.HasPrefix(path, targetModule+"/")) && - filePathDir(e.To) == targetRepo+strings.TrimPrefix(path, targetModule) { - present = true - break + if !present { + cr.graph.AddEdge(&graph.Edge{From: n.FilePath, To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: n.FilePath, Line: line}) } } - if !present { - cr.graph.AddEdge(&graph.Edge{From: n.FilePath, To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: n.FilePath, Line: n.StartLine}) - } + } return cr.resolveGoModuleImports(ctx, files, stats, nil) } From fc789a9cdecb3be27b5790fbeb689c29eb56f0e1 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 12:42:28 -0700 Subject: [PATCH 041/111] fix(resolver): bump ResolutionSemanticsVersion for Python bridge metadata Python calls and imports now store python_call_stub and python_import metadata that the bridge reads back. Stores written before it cannot be reused as resolved, so raise the version to 6 (#948 landed at 4 and #953 takes 5) and pin the new value. --- internal/resolver/version.go | 3 ++- internal/resolver/version_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/resolver/version.go b/internal/resolver/version.go index 568677412..7e569ef9d 100644 --- a/internal/resolver/version.go +++ b/internal/resolver/version.go @@ -40,7 +40,8 @@ import ( // Convergence winner payloads and surviving outgoing order change stored edges // for an unchanged corpus. // Go module binding changes candidate order, cross-repo provenance and import identity. -const ResolutionSemanticsVersion = 5 +// Python import and call stubs add stored metadata required by tracked-package linking. +const ResolutionSemanticsVersion = 6 // resolverVersionDomain versions the ENCODING below, not the contract. Two // fingerprints that disagree about how the preimage is rendered must not be diff --git a/internal/resolver/version_test.go b/internal/resolver/version_test.go index f90af9239..a31bbbcbb 100644 --- a/internal/resolver/version_test.go +++ b/internal/resolver/version_test.go @@ -22,7 +22,7 @@ import ( // and say which in the commit message. A diff that changes this line and // nothing else in internal/resolver is the signal that something re-keyed // every cached generation by accident. -const resolverVersionGolden = "resolver-5:7c5e80643f2ba32b450b7a56735b7e8a" +const resolverVersionGolden = "resolver-6:1d041ee489712f11544c6b6f7ce62982" func TestResolverVersionIsPinned(t *testing.T) { if got := Version(); got != resolverVersionGolden { @@ -38,7 +38,7 @@ func TestResolverVersionIsStable(t *testing.T) { if first != second { t.Fatalf("two reads of the resolution contract disagree: %q then %q", first, second) } - if !strings.HasPrefix(first, "resolver-5:") { + if !strings.HasPrefix(first, "resolver-6:") { t.Fatalf("the version %q does not name the semantics version it was produced under", first) } } From 76da94be4eb77107110fad3de33e72db7683af32 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:19:12 -0700 Subject: [PATCH 042/111] refactor(daemon): drop the duplicate warm-frontier recording loop The new loop records the frontier files before any early return, which made the later loop record the same files again. Delete the later one. --- cmd/gortex/daemon_state.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/gortex/daemon_state.go b/cmd/gortex/daemon_state.go index a338b3b4b..86f8adff0 100644 --- a/cmd/gortex/daemon_state.go +++ b/cmd/gortex/daemon_state.go @@ -381,11 +381,6 @@ func (frontier *warmupDeltaFrontier) record(result *indexer.IndexResult) { plan := frontier.plans[result.RepoPrefix] plan.Merge(result.DerivedInvalidation) frontier.plans[result.RepoPrefix] = plan - for _, file := range result.CrossRepoResolveFiles { - if file != "" { - frontier.crossRepoFiles[file] = struct{}{} - } - } } func (frontier *warmupDeltaFrontier) snapshot(changedRepos int, scopeUnknown bool) (map[string]indexer.DerivedInvalidationPlan, []string, bool) { From e21bb408f96f57983aba56a21e0d1b5a775b80ab Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:19:33 -0700 Subject: [PATCH 043/111] test: replace leftover diagnostic logs with assertions Remove t.Logf calls already backed by nearby assertions, and turn the lifecycle call logs into binding and cold-parity checks. Rename go_module_import_review_test.go to go_module_import_retry_state_test.go after its behavior; its content is unchanged. --- internal/indexer/cargo_crate_test.go | 1 - internal/indexer/cargo_import_fallback_test.go | 1 - .../indexer/go_module_import_recovery_test.go | 18 +++++++++--------- internal/indexer/go_module_import_test.go | 1 - internal/indexer/go_module_import_tie_test.go | 6 +++--- .../indexer/python_import_fallback_test.go | 1 - ...go => go_module_import_retry_state_test.go} | 0 internal/resolver/go_module_imports_test.go | 1 - 8 files changed, 12 insertions(+), 17 deletions(-) rename internal/resolver/{go_module_import_review_test.go => go_module_import_retry_state_test.go} (100%) diff --git a/internal/indexer/cargo_crate_test.go b/internal/indexer/cargo_crate_test.go index 1dd21442e..df4781b4c 100644 --- a/internal/indexer/cargo_crate_test.go +++ b/internal/indexer/cargo_crate_test.go @@ -151,7 +151,6 @@ func TestMultiIndexer_CargoCrateImport(t *testing.T) { continue } imports++ - t.Logf("import %s -> %s crossRepo=%v", e.From, e.To, e.CrossRepo) assert.Equal(t, resolves, e.CrossRepo) if resolves { assert.Equal(t, want, e.To) diff --git a/internal/indexer/cargo_import_fallback_test.go b/internal/indexer/cargo_import_fallback_test.go index 6bba9e321..51b00fc4e 100644 --- a/internal/indexer/cargo_import_fallback_test.go +++ b/internal/indexer/cargo_import_fallback_test.go @@ -61,7 +61,6 @@ func TestTrackedCargoNewModuleThroughFallback(t *testing.T) { writeWorkspaceFile(t, provider, "src/extra.rs", "pub fn later() {}\n") _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/extra.rs")}) require.NoError(t, err) - t.Logf("transform calls=%d", calls.Load()) require.GreaterOrEqual(t, calls.Load(), int32(3), "fallback path must run") require.NotNil(t, g.GetNode("provider/src/extra.rs")) diff --git a/internal/indexer/go_module_import_recovery_test.go b/internal/indexer/go_module_import_recovery_test.go index dcbb2b6d1..5f003b85d 100644 --- a/internal/indexer/go_module_import_recovery_test.go +++ b/internal/indexer/go_module_import_recovery_test.go @@ -179,16 +179,16 @@ func TestGoModuleImportReplaceAddedLater(t *testing.T) { _, err := mi.TrackRepoCtx(t.Context(), e) require.NoError(t, err) } - t.Logf("linked: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) require.NoError(t, os.MkdirAll(filepath.Join(a, "third_party", "library"), 0o755)) vendored := filepath.Join(a, "third_party", "library", "lib.go") writeFile(t, vendored, "package library\nfunc Hello() {}\n") writeFile(t, manifest, "module example.org/caller\nrequire example.org/library v1.0.0\nreplace example.org/library => ./third_party/library\n") _, err := mi.IncrementalReindexRepo("a", []string{manifest, vendored}) require.NoError(t, err) - t.Logf("after replace save: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + assert.Equal(t, "a/third_party/library/lib.go::Hello", goModuleLifecycleCall(t, g).To) mi.RunGlobalResolve() - t.Logf("after global: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + assert.Equal(t, "a/third_party/library/lib.go::Hello", goModuleLifecycleCall(t, g).To) var coldGraph graph.Store = graph.New() if backend == "sqlite" { db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) @@ -204,7 +204,7 @@ func TestGoModuleImportReplaceAddedLater(t *testing.T) { } cold.RunGlobalResolve() require.Len(t, goImportRecoveryRows(coldGraph, "a/main.go"), 1) - t.Logf("cold: %v call=%s", goImportRecoveryRows(coldGraph, "a/main.go"), goModuleLifecycleCall(t, coldGraph).To) + assert.Equal(t, goModuleLifecycleCall(t, coldGraph).To, goModuleLifecycleCall(t, g).To) // Retain retry metadata on the local row so removing replace can rebind it. assert.Equal(t, goImportBindingRows(coldGraph, "a/main.go"), goImportBindingRows(g, "a/main.go")) }) @@ -224,20 +224,20 @@ func TestGoModuleImportReplaceAddedThenRemoved(t *testing.T) { _, err := mi.TrackRepoCtx(t.Context(), e) require.NoError(t, err) } - t.Logf("linked: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) require.NoError(t, os.MkdirAll(filepath.Join(a, "third_party", "library"), 0o755)) vendored := filepath.Join(a, "third_party", "library", "lib.go") writeFile(t, vendored, "package library\nfunc Hello() {}\n") writeFile(t, manifest, plain+"replace example.org/library => ./third_party/library\n") _, err := mi.IncrementalReindexRepo("a", []string{manifest, vendored}) require.NoError(t, err) - t.Logf("replace added: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + assert.Equal(t, "a/third_party/library/lib.go::Hello", goModuleLifecycleCall(t, g).To) writeFile(t, manifest, plain) _, err = mi.IncrementalReindexRepo("a", []string{manifest}) require.NoError(t, err) - t.Logf("replace removed: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) mi.RunGlobalResolve() - t.Logf("after global: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) var coldGraph graph.Store = graph.New() if backend == "sqlite" { db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) @@ -252,7 +252,7 @@ func TestGoModuleImportReplaceAddedThenRemoved(t *testing.T) { require.NoError(t, err) } cold.RunGlobalResolve() - t.Logf("cold: %v call=%s", goImportRecoveryRows(coldGraph, "a/main.go"), goModuleLifecycleCall(t, coldGraph).To) + assert.Equal(t, goModuleLifecycleCall(t, coldGraph).To, goModuleLifecycleCall(t, g).To) assert.Equal(t, goImportRecoveryRows(coldGraph, "a/main.go"), goImportRecoveryRows(g, "a/main.go")) }) } diff --git a/internal/indexer/go_module_import_test.go b/internal/indexer/go_module_import_test.go index 33fe69f70..e57d4b1a1 100644 --- a/internal/indexer/go_module_import_test.go +++ b/internal/indexer/go_module_import_test.go @@ -70,7 +70,6 @@ func main() { lib.F() } } } require.Len(t, imports, 1) - t.Logf("import target after track + global resolve: %s", imports[0].To) assert.Equal(t, "provider/lib.go", imports[0].To) assert.True(t, imports[0].CrossRepo) contractID := "dep::example.com/lib" diff --git a/internal/indexer/go_module_import_tie_test.go b/internal/indexer/go_module_import_tie_test.go index 4020d7e49..686ae65e5 100644 --- a/internal/indexer/go_module_import_tie_test.go +++ b/internal/indexer/go_module_import_tie_test.go @@ -27,11 +27,11 @@ func TestGoModuleImportTieBrokenByRename(t *testing.T) { require.NoError(t, err) } mi.RunGlobalResolve() - t.Logf("tie: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + assert.False(t, goModuleLifecycleCall(t, g).CrossRepo, "ambiguous modules must not bind") manifest := filepath.Join(repos[2].Path, "go.mod") writeFile(t, manifest, "module example.org/forked\n") require.NoError(t, mi.GetIndexer("fork").IndexFile(manifest)) - t.Logf("after rename: %v call=%s", goImportRecoveryRows(g, "a/main.go"), goModuleLifecycleCall(t, g).To) + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) var coldGraph graph.Store = graph.New() if backend == "sqlite" { db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) @@ -46,7 +46,7 @@ func TestGoModuleImportTieBrokenByRename(t *testing.T) { require.NoError(t, err) } cold.RunGlobalResolve() - t.Logf("cold: %v call=%s", goImportRecoveryRows(coldGraph, "a/main.go"), goModuleLifecycleCall(t, coldGraph).To) + assert.Equal(t, goModuleLifecycleCall(t, coldGraph).To, goModuleLifecycleCall(t, g).To) assert.Equal(t, goImportRecoveryRows(coldGraph, "a/main.go"), goImportRecoveryRows(g, "a/main.go")) }) } diff --git a/internal/indexer/python_import_fallback_test.go b/internal/indexer/python_import_fallback_test.go index 2bdaf7bec..2ccf239b8 100644 --- a/internal/indexer/python_import_fallback_test.go +++ b/internal/indexer/python_import_fallback_test.go @@ -62,7 +62,6 @@ func TestTrackedPythonNewModuleThroughFallback(t *testing.T) { writeWorkspaceFile(t, provider, "src/PIL/extra.py", "def later(): pass\n") _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/extra.py")}) require.NoError(t, err) - t.Logf("transform calls=%d", calls.Load()) require.GreaterOrEqual(t, calls.Load(), int32(3), "fallback path must run") require.NotNil(t, g.GetNode("provider/src/PIL/extra.py")) diff --git a/internal/resolver/go_module_import_review_test.go b/internal/resolver/go_module_import_retry_state_test.go similarity index 100% rename from internal/resolver/go_module_import_review_test.go rename to internal/resolver/go_module_import_retry_state_test.go diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index a67cdfee8..d31eacc0b 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -288,7 +288,6 @@ func TestGoModuleImportWarmProviderEdit(t *testing.T) { assert.True(t, edges[0].CrossRepo) assert.Equal(t, 1, g.importScans) assert.Equal(t, 4, g.importRows, "one tracked and three unrelated import identities") - t.Logf("lazy seed: global scans=%d, import identities visited=%d", g.importScans, g.importRows) s.EvictFile("b/lib.go") s.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) if route == "mutation" { From 37000fc45bcff2c1972f46d2738f627991e98af6 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:19:41 -0700 Subject: [PATCH 044/111] fix(resolver): count Go import fallback demotions as unresolved applyGoModuleImportReindexes counted every applied row as resolved, including rows demoted to a dep:: or external:: fallback, while the module edge pass counts those as unresolved. Use the same synthetic target test so the reported stats agree. --- internal/resolver/go_module_imports.go | 20 ++++++++++---- internal/resolver/go_module_imports_test.go | 29 +++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 37394692a..209d14cf4 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -205,11 +205,21 @@ func (cr *CrossRepoResolver) applyGoModuleImportReindexes(batch []graph.EdgeRein original := graph.EdgeIdentityFor(row.Edge) original.To = row.OldTo resolved, alreadyCounted := counted[original] - if !alreadyCounted || !resolved { - stats.Resolved++ - } - if alreadyCounted && !resolved { - stats.Unresolved-- + nowResolved := !isSyntheticResolveTarget(row.Edge.To) + if nowResolved { + if !alreadyCounted || !resolved { + stats.Resolved++ + } + if alreadyCounted && !resolved { + stats.Unresolved-- + } + } else { + if !alreadyCounted || resolved { + stats.Unresolved++ + } + if alreadyCounted && resolved { + stats.Resolved-- + } } if row.Edge.CrossRepo { stats.CrossRepoEdges++ diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index d31eacc0b..dd62ac3e7 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -654,3 +654,32 @@ func TestGoModuleImportIncomingAdmissionRefused(t *testing.T) { }) } } + +func TestGoModuleImportFallbackStatsAreUnresolved(t *testing.T) { + for _, target := range []string{"dep::example.org/lib", "external::example.org/lib"} { + for _, priorCount := range []string{"none", "unresolved", "resolved"} { + t.Run(target+"/"+priorCount, func(t *testing.T) { + g := graph.New() + old := &graph.Edge{From: "a/main.go", To: "b/lib.go", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2} + g.AddEdge(old) + fresh := *old + fresh.To = target + stats := &CrossRepoStats{ByRepo: map[string]int{}} + counted := map[graph.EdgeIdentity]bool{} + if priorCount != "none" { + resolved := priorCount == "resolved" + counted[graph.EdgeIdentityFor(old)] = resolved + if resolved { + stats.Resolved = 1 + } else { + stats.Unresolved = 1 + } + } + NewCrossRepo(g).applyGoModuleImportReindexes([]graph.EdgeReindex{{Edge: &fresh, OldTo: old.To}}, stats, counted) + require.Zero(t, stats.Resolved) + require.Equal(t, 1, stats.Unresolved) + require.Zero(t, stats.CrossRepoEdges) + }) + } + } +} From 1d81a612642e482d4f9cded187459f669b5501d1 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:20:05 -0700 Subject: [PATCH 045/111] docs(multi-repo): document tracked Go, Cargo and Python import links Describe how imports of a tracked Go module, Rust crate or Python package link to the provider file, what each requires, when it refuses an ambiguous target, and its limits. Correct the Go limits line now that bridge-linked imports no longer make the provider repo reachable. --- docs/multi-repo.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/multi-repo.md b/docs/multi-repo.md index 85e547ff9..570116a13 100644 --- a/docs/multi-repo.md +++ b/docs/multi-repo.md @@ -126,6 +126,24 @@ the participating repositories' allow-lists. This keeps a repository from silently losing framework edges requested by a sibling while allowing a workspace where every repository opts out to avoid the scans entirely. +### Tracked module imports + +- Go imports can link to a representative source file in a tracked module. + The longest matching root `go.mod` path selects the package. Ambiguous providers + stay unlinked. Local `replace` paths must point to indexed source files. +- Rust `use` imports can link to a tracked library declared in `Cargo.toml`. + Dependency aliases, workspace dependencies, explicit workspace members and + `[lib]` names and paths are supported. Undeclared crates, ambiguous providers + and reserved names such as `std`, `core` and `crate` stay unlinked. +- Python imports can link to a tracked distribution declared in `pyproject.toml` + or `requirements*.txt`. Root and `src/` layouts support modules and regular + packages. A local top-level package blocks a foreign link. Ambiguous providers, + undeclared distributions and Python stdlib imports stay unlinked. + +These import-file links do not make every symbol in the provider reachable. +Calls still need an imported path or type. Versions and build configurations +are not verified. + ### Python package re-exports Calls through a declared, tracked Python distribution can follow one level of @@ -502,9 +520,14 @@ Scoped tool responses carry a `scope_applied` meta field plus a one-line widen h - Entry lookup respects `exports` keys, wildcard subpaths and `null` blocks. It probes the selected `import`/`default` target first, then a `types` condition when that file is not indexed. Without `exports`, it tries `types`, `main` and directory indexes. Unsupported-condition subpaths never guess a directory target. - Bare calls use the caller file's named import binding, including its local alias, not another same-named function anywhere in the provider. A per-binding import links to a symbol marked public by the extractor; otherwise it stays at `external::::` with its `npm_import` marker and creates no call binding. Linked imports retain their specifier, survive provider eviction as external imports, and are rechecked by whole-graph resolution. - **Limits:** Build-output-only entries, such as `main` and `types` pointing into excluded `dist/`, stay external unless an indexed target is declared. Gortex does not infer `src/` from `dist/`. The importing repository's own packages, `npm:` aliases and workspace paths that escape the repository are not linked by this pass. Sibling dependencies declared with `file:`, `link:` or `portal:` are not supported. Default-import bare calls have no exact binding evidence yet and stay unresolved. After an upgrade, JavaScript and TypeScript files are re-extracted once (extractor version 2), so existing imports acquire the specifier and alias metadata; their links appear after that pass. Namespace member calls, CommonJS `require` calls and package re-exports are not linked by this npm import pass. Export targets without `./` are accepted leniently, as in workspace resolution, even though Node rejects them. Call binding is file-scoped: a parameter or local callable with the imported name can bind to the import. + - Go source imports can link to a representative indexed package file. The `go.mod` dependency contract remains separate. Linked imports retain their package identity across provider saves. - Go package-selector calls such as `library.Hello()` use tracked roots' `go.mod` module paths, not checkout names. An import of `example.org/library/sub` can bind to `/sub/file.go::Hello` even when the provider prefix is `b`. - Precise own-module, vendor and staging paths take priority. Otherwise the longest matching module path wins. A primary/non-worktree checkout wins a tie with dedicated worktree copies; ties between independent primary checkouts are refused. Cross-workspace calls still require the caller's `cross_workspace_deps` declaration. - A cross-repo module binding carries confidence `0.6` and origin `ast_inferred`. It is a source hint, not proof that the tracked checkout matches the required version. The edge retains its symbol-bearing import identity so provider edits and full reindexes can rebind it without editing the caller. Module-map changes re-check existing bindings. - - **Limits:** Only root `go.mod` identities are used; nested modules and `go.work` selection are not resolved. Required versions and module/version replacements are not verified. A local `replace` whose destination is untracked prevents module-path binding instead of selecting another checkout with the same module name. Tracked local replacements remain source hints, not full Go build-list resolution. Calls on values typed from another module, such as `client.Do()` for a `library.Client`, can remain `unresolved::*.Do`; receiver-type linking is not implemented here. + - **Limits:** Only each tracked root's `go.mod` identity is used. Separately tracked nested modules can win by longest path; nested manifests within one tracked root and `go.work` selection are not resolved. Required versions and module/version replacements are not verified. A local `replace` whose destination is untracked prevents module-path binding instead of selecting another checkout with the same module name. Tracked local replacements remain source hints, not full Go build-list resolution. Calls on values typed from another module, such as `client.Do()` for a `library.Client`, stay unresolved unless another pass has precise evidence. Importing the module alone does not enable name-only receiver binding. + - Rust `use` imports link to the declared provider's library root or module file. Dependency, dev-dependency, build-dependency and target tables are read. Calls bind only to a unique imported item or receiver in that file. + - **Cargo limits:** This is not Cargo build resolution. Versions, features and target conditions are not evaluated. Absolute or escaping library paths, conflicting aliases and multiple eligible providers are refused. Re-exports and untyped receivers are not inferred. Existing links are not always revisited after declaration or provider-set changes; reindex the provider to refresh them. + - Python distribution imports link only after a consumer declaration and a unique eligible provider match. Qualified calls use the linked module file. Typed methods also require a matching receiver in that file. Package calls can follow the one-level re-exports described above. + - **Python limits:** Distribution versions and arbitrary source layouts are not verified. Providers must expose indexed root or `src/` modules or regular packages. Namespace-only providers and dynamic imports are not linked. Local top-level names take priority, including namespace directories. - **Impact analysis** — `explain_change_impact`, `verify_change`, and `get_test_targets` follow cross-repo edges automatically, grouping results by repository. - **Shared repos** — the same repo can appear in multiple projects with different reference tags. It's indexed once and shared across projects. From fb865be26c9ec8651d0283fbcdaf422bfd0dfec9 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sat, 24 Oct 2026 22:20:14 -0700 Subject: [PATCH 046/111] perf(indexer): scan imports once and defer Cargo manifest probes Watcher and warm paths scanned every import with full metadata once per receipt repo, probed Cargo.toml for every tracked repo on each new resolver, and ran the Python added-file scan even in single-repo daemons. Probe Cargo root manifests inside the existing once.Do, collect every receipt repo in one light import scan and hydrate only bound imports into the selected providers, and skip the Python added-file scan when no tracked-package lookup exists. A configured multi-repo workspace that is still registering keeps the old frontier so warmup loses no consumers. --- internal/indexer/cargo_crate.go | 11 +++-- internal/indexer/cargo_import_restub.go | 46 +++++++++++++----- .../cargo_import_warm_recovery_test.go | 48 +++++++++++++++++++ internal/indexer/incremental_watcher_batch.go | 2 +- internal/indexer/multi.go | 2 +- internal/indexer/python_import_restub.go | 17 +++++++ .../indexer/python_tracked_import_test.go | 18 +++++++ 7 files changed, 124 insertions(+), 20 deletions(-) diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go index ceb051d21..1aabc67df 100644 --- a/internal/indexer/cargo_crate.go +++ b/internal/indexer/cargo_crate.go @@ -38,13 +38,10 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { roots := map[string]manifestTree{} for prefix, meta := range mi.AllMetadata() { if meta != nil && meta.RootPath != "" { - tree := mi.manifestTreeFor(prefix, meta.RootPath) - if tree.isFile("Cargo.toml") { - roots[prefix] = tree - } + roots[prefix] = mi.manifestTreeFor(prefix, meta.RootPath) } } - if len(roots) == 0 { + if len(roots) < 2 { return nil } owners := map[string]*crate{} @@ -55,6 +52,10 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { return func(file, name string) []resolver.CargoCrate { once.Do(func() { for prefix, tree := range roots { + if !tree.isFile("Cargo.toml") { + delete(roots, prefix) + continue + } manifest := readCargoManifest(tree, ".") src, _ := tree.readFile("Cargo.toml") members, exclude := modules.ParseCargoWorkspace(src) diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go index 4d0267093..eb5b15944 100644 --- a/internal/indexer/cargo_import_restub.go +++ b/internal/indexer/cargo_import_restub.go @@ -87,35 +87,55 @@ func restubTrackedCargoImportsFromView(g graph.Store, evicted map[string]struct{ // Retry parked imports; new Rust files can also retarget marked provider bindings. func (mi *MultiIndexer) cargoConsumerFilesForRepo(repo string, retargetBound bool) []string { + return mi.cargoConsumerFilesForRepos([]string{repo}, retargetBound) +} + +func (mi *MultiIndexer) cargoConsumerFilesForRepos(repos []string, retargetBound bool) []string { lookup := mi.cargoCrateLookup() - if lookup == nil { + if lookup == nil || len(repos) == 0 { return nil } - files := make(map[string]struct{}) - edges := graph.EdgesLightSeq(mi.graph, graph.EdgeImports) - if retargetBound { - edges = mi.graph.EdgesByKind(graph.EdgeImports) + wanted := make(map[string]bool, len(repos)) + for _, repo := range repos { + wanted[repo] = true } - for edge := range edges { + files := make(map[string]struct{}) + var sites []graph.EdgeSite + for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { + if !strings.HasSuffix(edge.FilePath, ".rs") { + continue + } spec, external := strings.CutPrefix(edge.To, "external::") if !external { spec, external = strings.CutPrefix(edge.To, "unresolved::import::") } - if !external || !strings.HasSuffix(edge.FilePath, ".rs") { - stub, _ := edge.Meta[resolver.CargoImportStubMeta].(string) - if retargetBound && stub != "" && edge.CrossRepo && strings.HasSuffix(edge.FilePath, ".rs") && graph.RepoPrefixOfID(edge.To) == repo { - files[edge.FilePath] = struct{}{} + if !external { + if retargetBound && edge.CrossRepo && wanted[graph.RepoPrefixOfID(edge.To)] { + sites = append(sites, graph.EdgeSite{From: edge.From, Line: edge.Line, Kind: graph.EdgeImports}) } continue } name, _, _ := strings.Cut(spec, "/") for _, crate := range lookup(edge.FilePath, name) { - if graph.RepoPrefixOfID(crate.Root) == repo || graph.RepoPrefixOfID(edge.From) == repo { + if wanted[graph.RepoPrefixOfID(crate.Root)] || wanted[graph.RepoPrefixOfID(edge.From)] { files[edge.FilePath] = struct{}{} break } } } + // Close the light cursor before hydrating only marked-provider candidates. + for start := 0; start < len(sites); start += 512 { + page := sites[start:min(start+512, len(sites))] + full := mi.graph.GetEdgeCandidates(nil, page) + for _, site := range page { + for _, edge := range full.Site(site.From, site.Line, site.Kind) { + stub, _ := edge.Meta[resolver.CargoImportStubMeta].(string) + if stub != "" && edge.CrossRepo && wanted[graph.RepoPrefixOfID(edge.To)] && strings.HasSuffix(edge.FilePath, ".rs") { + files[edge.FilePath] = struct{}{} + } + } + } + } return sortedStringKeys(files) } @@ -136,8 +156,8 @@ func (mi *MultiIndexer) runWarmCargoResolve(ctx context.Context, files []string, if full { err = mi.runCrossRepoResolveContext(ctx, false, repos...) } else { - for _, repo := range repos { - files = appendUniqueSorted(files, mi.cargoConsumerFilesForRepo(repo, true)...) + if len(repos) > 0 { + files = appendUniqueSorted(files, mi.cargoConsumerFilesForRepos(repos, true)...) } if err = context.Cause(ctx); err == nil { mi.runCrossRepoResolveFiles(files, repos...) diff --git a/internal/indexer/cargo_import_warm_recovery_test.go b/internal/indexer/cargo_import_warm_recovery_test.go index a7376e411..7807530d1 100644 --- a/internal/indexer/cargo_import_warm_recovery_test.go +++ b/internal/indexer/cargo_import_warm_recovery_test.go @@ -3,6 +3,7 @@ package indexer import ( "context" "fmt" + "iter" "os" "path/filepath" "testing" @@ -159,3 +160,50 @@ func TestCargoAddedFileFrontierOnlyMarkedProviderImports(t *testing.T) { require.Equal(t, []string{"consumer/marked.rs"}, mi.cargoConsumerFilesForRepo("provider", true)) require.Empty(t, mi.cargoConsumerFilesForRepo("provider", false)) } + +type cargoWarmScanStore struct { + graph.Store + fullScans, lightScans int +} + +func (s *cargoWarmScanStore) EdgesByKind(kind graph.EdgeKind) iter.Seq[*graph.Edge] { + if kind == graph.EdgeImports { + s.fullScans++ + } + return s.Store.EdgesByKind(kind) +} +func (s *cargoWarmScanStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + if len(kinds) == 1 && kinds[0] == graph.EdgeImports { + s.lightScans++ + } + return graph.EdgesLightSeq(s.Store, kinds...) +} +func TestCargoWarmRecoveryScansImportsOnce(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + for _, root := range []string{provider, consumer} { + writeWorkspaceFile(t, root, "Cargo.toml", "[package]\nname='library'\nversion='1.0.0'\n") + } + g := &cargoWarmScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "unrelated/lib.rs", FilePath: "unrelated/lib.rs", To: "external::unknown/f", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{ + "provider": {RootPath: provider, cargoRetarget: &IndexResult{CargoRetarget: true}}, + "consumer": {RootPath: consumer, cargoRetarget: &IndexResult{CargoRetarget: true}}, + }} + require.NoError(t, mi.runWarmCargoResolve(t.Context(), nil, false)) + require.Zero(t, g.fullScans, "warm recovery must not read every import's metadata") + require.Equal(t, 1, g.lightScans, "all receipt repos must share one scan") +} + +func TestCargoLookupDefersManifestProbesUntilUse(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + mi := &MultiIndexer{graph: graph.New(), repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + lookup := mi.cargoCrateLookup() + require.NotNil(t, lookup, "construction must not probe Cargo manifests") + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname='library'\nversion='1.0.0'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname='app'\nversion='1.0.0'\n[dependencies]\nlibrary='1'\n") + providers := lookup("consumer/src/lib.rs", "library") + require.Len(t, providers, 1) + require.Equal(t, "provider/src/lib.rs", providers[0].Root) + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname='app'\nversion='1.0.0'\n") + require.Equal(t, providers, lookup("consumer/src/lib.rs", "library"), "one pass must retain one manifest snapshot") +} diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index 009db2ce7..6c1a3b3a2 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -538,7 +538,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } crossRepoFiles := appendUniqueSorted(nil, files...) if result != nil && batch != nil { - result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, pythonConsumerFilesForAddedFiles(mi.graph, batch.pythonAddedFiles)...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.pythonAddedFileConsumers(batch.pythonAddedFiles)...) } cargoRetargetRepo := "" if batch != nil { diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 898159e2f..2288e16f9 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -3743,7 +3743,7 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE // resolve/global phases. A full retrack already ran its own resolve/derived // pipeline and only needs the existing cross-repository contract reconcile. if batchMode.deferResolve && batch != nil { - result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, pythonConsumerFilesForAddedFiles(mi.graph, batch.pythonAddedFiles)...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.pythonAddedFileConsumers(batch.pythonAddedFiles)...) } if !result.FullRetrack && !batchMode.deferResolve { mi.resolveIncrementalRepoMutation(prefix, result, receipt, batch) diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index 52b3dfc6b..ef87b1dea 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -174,3 +174,20 @@ func pythonConsumerFilesForAddedFiles(g graph.Store, files []string) []string { } return sortedStringKeys(consumers) } + +func (mi *MultiIndexer) pythonAddedFileConsumers(files []string) []string { + if len(files) == 0 { + return nil + } + // Warmup can publish a provider receipt before its configured consumers + // are registered. Preserve that frontier until the workspace pass. + warming := false + if mi.configMgr != nil { + configuredRepos := mi.configMgr.Global().Repos + warming = len(configuredRepos) > 1 && len(mi.AllMetadata()) < len(configuredRepos) + } + if !warming && mi.pythonPackageLookup() == nil { + return nil + } + return pythonConsumerFilesForAddedFiles(mi.graph, files) +} diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go index 9b2698ecf..60add38bf 100644 --- a/internal/indexer/python_tracked_import_test.go +++ b/internal/indexer/python_tracked_import_test.go @@ -1,6 +1,7 @@ package indexer import ( + "context" "fmt" "iter" "path/filepath" @@ -289,3 +290,20 @@ func TestPythonConsumerLookupClosesImportCursor(t *testing.T) { }) } } + +func TestSingleRepoPythonAddedFileSkipsCrossRepoScan(t *testing.T) { + root := t.TempDir() + writeWorkspaceFile(t, root, "app.py", "import missing\n") + g := &pythonLookupScanStore{Store: graph.New()} + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + _, err := mi.TrackRepoCtx(t.Context(), config.RepoEntry{Path: root, Name: "solo", Workspace: "shared"}) + require.NoError(t, err) + g.importScans = 0 + writeWorkspaceFile(t, root, "missing.py", "pass\n") + _, err = mi.IncrementalReindexRepo("solo", []string{filepath.Join(root, "missing.py")}) + require.NoError(t, err) + require.Zero(t, g.importScans, "a lone Python repo has no foreign provider to recover") +} From 4647050ebc57027c50ada6658cd10049013ba220 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 00:43:48 -0700 Subject: [PATCH 047/111] style: use a tagged switch for the replacement save route staticcheck QF1003 flagged the if/else chain on route in the replacement-file cold-parity test. No behavior change. --- internal/indexer/go_module_import_recovery_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/indexer/go_module_import_recovery_test.go b/internal/indexer/go_module_import_recovery_test.go index 5f003b85d..dc40ea466 100644 --- a/internal/indexer/go_module_import_recovery_test.go +++ b/internal/indexer/go_module_import_recovery_test.go @@ -353,11 +353,12 @@ func TestGoModuleReplacementFileSaveColdParity(t *testing.T) { require.Len(t, before, 1) require.Contains(t, before[0], "a/third_party/library/lib.go cross=false") writeFile(t, replacement, "package library\nfunc Hello() { println(1) }\n") - if route == "fallback" { + switch route { + case "fallback": require.NoError(t, mi.GetIndexer("a").indexFile(replacement, true, nil)) - } else if route == "file" { + case "file": require.NoError(t, mi.GetIndexer("a").IndexFile(replacement)) - } else { + default: _, err := mi.IncrementalReindexRepo("a", []string{replacement}) require.NoError(t, err) } From 3a209d1ad72d67235b69e32ba66e8781869dff92 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:39:51 -0700 Subject: [PATCH 048/111] fix(resolver): bind Python imported methods only through the imported name resolvePythonImportedMethod linked x.show() to a provider method whenever the receiver type name matched, even when the import did not bind that name in the consumer. A local class with the same name then bound to the provider's method. The Python extractor now records the imported-name bindings (with aliases). The resolver requires the receiver to be one of them and refuses when the caller repo defines a type of that name, as the Cargo copy already does. --- internal/parser/languages/python.go | 4 ++++ internal/parser/languages/python_test.go | 8 ++++++++ .../bridge_import_reachability_test.go | 19 +++++++++++++++---- internal/resolver/python_tracked_import.go | 12 +++++++++++- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/internal/parser/languages/python.go b/internal/parser/languages/python.go index 03c4ba8a9..6402b32a8 100644 --- a/internal/parser/languages/python.go +++ b/internal/parser/languages/python.go @@ -934,9 +934,11 @@ func (e *PythonExtractor) emitImport(m parser.QueryResult, filePath, fileID stri func (e *PythonExtractor) emitImportFrom(m parser.QueryResult, filePath, fileID string, src []byte, result *parser.ExtractionResult, imports map[string]string) { mod := m.Captures["import.module"] pyEmitImportNode(filePath, fileID, mod.Text, "", mod.StartLine+1, result) + bindings := make(map[string]any) result.Edges = append(result.Edges, &graph.Edge{ From: fileID, To: "unresolved::import::" + mod.Text, Kind: graph.EdgeImports, FilePath: filePath, Line: mod.StartLine + 1, + Meta: map[string]any{"python_bindings": bindings}, }) def, ok := m.Captures["importfrom.def"] @@ -963,6 +965,7 @@ func (e *PythonExtractor) emitImportFrom(m parser.QueryResult, filePath, fileID continue } imports[name] = mod.Text + "." + name + bindings[name] = name case "aliased_import": var importedName, alias string for j, _nc := 0, int(child.NamedChildCount()); j < _nc; j++ { @@ -978,6 +981,7 @@ func (e *PythonExtractor) emitImportFrom(m parser.QueryResult, filePath, fileID continue } imports[alias] = mod.Text + "." + importedName + bindings[alias] = importedName } } } diff --git a/internal/parser/languages/python_test.go b/internal/parser/languages/python_test.go index 21a0d0c24..72a91d71e 100644 --- a/internal/parser/languages/python_test.go +++ b/internal/parser/languages/python_test.go @@ -515,3 +515,11 @@ func pyNodeIDs(nodes []*graph.Node) []string { } return out } + +func TestPyExtractor_ImportedReceiverBindings(t *testing.T) { + result, err := NewPythonExtractor().Extract("app.py", []byte("from pkg.image import Image as Picture, Other\n")) + require.NoError(t, err) + imports := edgesOfKind(result.Edges, graph.EdgeImports) + require.Len(t, imports, 1) + require.Equal(t, map[string]any{"Picture": "Image", "Other": "Other"}, imports[0].Meta["python_bindings"]) +} diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go index e33c3fc4e..6b097275e 100644 --- a/internal/resolver/bridge_import_reachability_test.go +++ b/internal/resolver/bridge_import_reachability_test.go @@ -107,7 +107,7 @@ func TestCargoCallsStayWithinImportedItem(t *testing.T) { func TestPythonMethodsStayWithinImportedType(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { - for _, scenario := range []string{"method", "foreign_type", "ambiguous"} { + for _, scenario := range []string{"method", "foreign_type", "ambiguous", "local_type", "alias", "unbound"} { t.Run(backend+"/"+scenario, func(t *testing.T) { var g graph.Store = graph.New() if backend == "sqlite" { @@ -122,11 +122,22 @@ func TestPythonMethodsStayWithinImportedType(t *testing.T) { g.AddNode(&graph.Node{ID: provider, FilePath: provider, Kind: graph.KindFile, RepoPrefix: "provider", Language: "python"}) g.AddNode(&graph.Node{ID: provider + "::Image.show", FilePath: provider, Name: "show", Kind: graph.KindMethod, RepoPrefix: "provider", Language: "python", Meta: map[string]any{"receiver": "Image"}}) } - g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/pkg/image.py", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image"}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/pkg/image.py", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image", "python_bindings": map[string]any{"Image": "Image"}}}) if scenario == "ambiguous" { - g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/pkg/aaa.py", Kind: graph.EdgeImports, Line: 2, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.aaa", "python_import": "pkg.aaa"}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/pkg/aaa.py", Kind: graph.EdgeImports, Line: 2, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.aaa", "python_import": "pkg.aaa", "python_bindings": map[string]any{"Image": "Image"}}}) + } + if scenario == "local_type" { + g.AddNode(&graph.Node{ID: file + "::Image", Name: "Image", FilePath: file, Kind: graph.KindType, RepoPrefix: "consumer", Language: "python"}) + } + if scenario == "alias" || scenario == "unbound" { + imports := outEdgesOfKind(g, file, graph.EdgeImports) + imports[0].Meta["python_bindings"] = map[string]any{"Picture": "Image"} + g.ReindexEdges([]graph.EdgeReindex{{Edge: imports[0], OldTo: imports[0].To}}) } receiver := "Image" + if scenario == "alias" { + receiver = "Picture" + } if scenario == "foreign_type" { receiver = "Foreign" } @@ -135,7 +146,7 @@ func TestPythonMethodsStayWithinImportedType(t *testing.T) { require.NoError(t, err) calls := outEdgesOfKind(g, file, graph.EdgeCalls) require.Len(t, calls, 1) - if scenario == "method" { + if scenario == "method" || scenario == "alias" { require.Equal(t, "provider/pkg/image.py::Image.show", calls[0].To) require.True(t, calls[0].CrossRepo) } else { diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go index a33b1b815..6ecc09e94 100644 --- a/internal/resolver/python_tracked_import.go +++ b/internal/resolver/python_tracked_import.go @@ -297,14 +297,24 @@ func (cr *CrossRepoResolver) resolvePythonImportedMethod(e *graph.Edge, target s if receiver == "" { return false } + for _, node := range cr.cachedFindNodesByName(receiver) { + if node.Kind == graph.KindType && node.RepoPrefix == graph.RepoPrefixOfID(e.From) { + return false + } + } var match *graph.Node for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { spec, _ := imp.Meta["python_import"].(string) if imp.Meta[PythonImportStubMeta] == nil || !pythonModuleFileMatches(imp.To, spec) { continue } + bindings, _ := imp.Meta["python_bindings"].(map[string]any) + imported, _ := bindings[receiver].(string) + if imported == "" { + continue + } for _, candidate := range cr.cachedFindNodesByName(strings.TrimPrefix(target, "*.")) { - if candidate.Kind != graph.KindMethod || candidate.FilePath != imp.To || nodeReceiverType(candidate) != receiver || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { + if candidate.Kind != graph.KindMethod || candidate.FilePath != imp.To || nodeReceiverType(candidate) != imported || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { continue } if match != nil && match.ID != candidate.ID { From 700d36770304822dca0c1bb7cdb0ef3533f42d16 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:40:22 -0700 Subject: [PATCH 049/111] fix(resolver): reopen Cargo calls when their import is refused When a revisited Cargo import was refused and restored to its stub, the calls bound through it kept pointing at the provider with CrossRepo set. They were not unresolved, so no later pass revisited them, while a cold index leaves them unresolved. Restoring an import now restubs the calls in that file that carry CargoCallStubMeta and point into the old target file, and returns them so the scoped pass resolves them again. --- internal/resolver/cargo_crate.go | 37 +++++++++++-- internal/resolver/cargo_crate_test.go | 3 +- .../resolver/cargo_import_retarget_test.go | 53 +++++++++++++++++-- internal/resolver/cross_repo.go | 5 +- 4 files changed, 87 insertions(+), 11 deletions(-) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index 57b3c5558..91868429e 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -99,15 +99,15 @@ func (cr *CrossRepoResolver) cargoImportTarget(e *graph.Edge, spec string) *grap // Per-repo resolution converts missing imports to external stubs. Retry just // those Rust imports before the cross-repo unresolved pass builds reachability. -func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats) error { +func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats) (reopened []*graph.Edge, err error) { if cr.cargoCrates == nil { - return ctx.Err() + return reopened, ctx.Err() } // Keep metadata out of the scan. Only declared, tracked crates need hydration. var eligible []*graph.Edge for e := range edges { if err := ctx.Err(); err != nil { - return err + return reopened, err } if e == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { continue @@ -128,7 +128,7 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter // only one bounded page, as in the tracked npm bridge. for start := 0; start < len(eligible); start += resolvePendingPageRows { if err := ctx.Err(); err != nil { - return err + return reopened, err } page := eligible[start:min(start+resolvePendingPageRows, len(eligible))] sites := make([]graph.EdgeSite, 0, len(page)) @@ -151,6 +151,7 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) projected.To, projected.CrossRepo = e.To, false stats.Unresolved++ + reopened = append(reopened, cr.restubCargoImportCalls(e.FilePath, oldTo)...) } } else if node != nil && node.ID != e.To { oldTo := e.To @@ -164,7 +165,7 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter cr.graph.ReindexEdges(batch) } } - return ctx.Err() + return reopened, ctx.Err() } func (cr *CrossRepoResolver) cargoImportSpec(e *graph.Edge) string { @@ -256,3 +257,29 @@ func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target stri } return true } + +// A refused import no longer authorizes calls through its former target file. +func (cr *CrossRepoResolver) restubCargoImportCalls(file, target string) []*graph.Edge { + var batch []graph.EdgeReindex + var reopened []*graph.Edge + for row := range graph.EdgesInScopeSeq(cr.graph, nil, []string{file}, graph.EdgeCalls) { + call := row.Edge + if call == nil || !call.CrossRepo { + continue + } + original, _ := call.Meta[CargoCallStubMeta].(string) + if original == "" || !strings.HasPrefix(call.To, target+"::") { + continue + } + old := call.To + graph.StashRestubProvenance(call) + call.To, call.CrossRepo, call.ConfidenceLabel = original, false, "" + clearEdgeTerminal(call) + reopened = append(reopened, call) + batch = append(batch, graph.EdgeReindex{Edge: call, OldTo: old}) + } + if len(batch) > 0 { + cr.graph.ReindexEdges(batch) + } + return reopened +} diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go index 5ce523804..dd025550e 100644 --- a/internal/resolver/cargo_crate_test.go +++ b/internal/resolver/cargo_crate_test.go @@ -83,7 +83,8 @@ func TestCargoImportLightPaging(t *testing.T) { return []CargoCrate{{Root: "lib/src/lib.rs", Name: "other_crate"}} }) stats := &CrossRepoStats{ByRepo: map[string]int{}} - require.NoError(t, cr.resolveCargoImports(context.Background(), graph.EdgesLightSeq(g, graph.EdgeImports), stats)) + _, err := cr.resolveCargoImports(context.Background(), graph.EdgesLightSeq(g, graph.EdgeImports), stats) + require.NoError(t, err) assert.Zero(t, g.fullScans) assert.Equal(t, 1, g.lightScans) assert.Equal(t, 2, g.pages) diff --git a/internal/resolver/cargo_import_retarget_test.go b/internal/resolver/cargo_import_retarget_test.go index 9e60e9d2b..8140dc4c9 100644 --- a/internal/resolver/cargo_import_retarget_test.go +++ b/internal/resolver/cargo_import_retarget_test.go @@ -2,6 +2,7 @@ package resolver import ( "context" + "fmt" "path/filepath" "slices" "testing" @@ -73,7 +74,8 @@ func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { projected.Kind = graph.EdgeCalls edges = slices.Values([]*graph.Edge{&projected}) } - require.NoError(t, cr.resolveCargoImports(context.Background(), edges, stats)) + _, err := cr.resolveCargoImports(context.Background(), edges, stats) + require.NoError(t, err) imports := outEdgesOfKind(g, file, graph.EdgeImports) require.Len(t, imports, 1) changed := scenario == "unresolved_marker" || scenario == "external_marker" || scenario == "light_projection" || scenario == "multiple_enabled" @@ -92,7 +94,8 @@ func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { require.Zero(t, stats.Resolved) return } - require.NoError(t, cr.resolveCargoImports(context.Background(), slices.Values(imports), stats)) + _, err = cr.resolveCargoImports(context.Background(), slices.Values(imports), stats) + require.NoError(t, err) require.Equal(t, priorWrites, g.writes, "an unchanged native match must not reindex") }) } @@ -109,6 +112,7 @@ func TestCargoRevisitedImportRestoresStubOnRefusal(t *testing.T) { g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "rust"}) } g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "provider/src/util.rs", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: stub}}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "provider/src/util.rs::action", Kind: graph.EdgeCalls, Line: 2, CrossRepo: true, Meta: map[string]any{CargoCallStubMeta: "unresolved::action"}}) cr := NewCrossRepo(g) cr.SetCargoImportRetargetRepo("provider") cr.SetCargoCrateLookup(func(string, string) []CargoCrate { @@ -117,13 +121,56 @@ func TestCargoRevisitedImportRestoresStubOnRefusal(t *testing.T) { } return []CargoCrate{{Root: "provider/src/lib.rs", Name: "other_crate"}, {Root: "fork/src/lib.rs", Name: "other_crate"}} }) - require.NoError(t, cr.resolveCargoImports(t.Context(), graph.EdgesLightSeq(g, graph.EdgeImports), &CrossRepoStats{ByRepo: map[string]int{}})) + _, err := cr.resolveCargoImports(t.Context(), graph.EdgesLightSeq(g, graph.EdgeImports), &CrossRepoStats{ByRepo: map[string]int{}}) + require.NoError(t, err) imports := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeImports) require.Len(t, imports, 1) require.Equal(t, stub, imports[0].To) require.False(t, imports[0].CrossRepo) + calls := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeCalls) + require.Len(t, calls, 1) + require.Equal(t, "unresolved::action", calls[0].To, "warm call must equal a cold unbound call") + require.False(t, calls[0].CrossRepo) }) } } } } + +func TestCargoRefusedImportWarmMatchesCold(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scoped := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/scoped=%v", backend, scoped), func(t *testing.T) { + targets := []string{} + for _, warm := range []bool{false, true} { + g := goImportReviewStore(t, backend) + file, provider := "consumer/src/lib.rs", "provider/src/util.rs" + for _, path := range []string{file, "provider/src/lib.rs", provider} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + g.AddNode(&graph.Node{ID: provider + "::action", FilePath: provider, Name: "action", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + imp, call := "external::other_crate/util/action", "unresolved::action" + if warm { + imp, call = provider, provider+"::action" + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: imp, Kind: graph.EdgeImports, Line: 1, CrossRepo: warm, Meta: map[string]any{CargoImportStubMeta: "external::other_crate/util/action"}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: call, Kind: graph.EdgeCalls, Line: 2, CrossRepo: warm, Meta: map[string]any{CargoCallStubMeta: "unresolved::action"}}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return nil }) + cr.SetCargoImportRetargetRepo("provider") + if scoped { + cr.ResolveForFile("consumer", "src/lib.rs") + } else { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + } + edge := outEdgesOfKind(g, file, graph.EdgeCalls)[0] + require.False(t, edge.CrossRepo) + targets = append(targets, edge.To) + } + require.Equal(t, targets[0], targets[1]) + require.Equal(t, "unresolved::action", targets[0]) + }) + } + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index c1faedc3d..7155d6a35 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -433,7 +433,7 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS if err := cr.resolveTrackedNpmImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { return stats, err } - if err := cr.resolveCargoImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { + if _, err := cr.resolveCargoImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { return stats, err } if _, err := cr.resolveTrackedPythonImports(ctx, cr.trackedPythonImportEdges(), stats, false); err != nil { @@ -724,8 +724,9 @@ func (cr *CrossRepoResolver) ResolveForFile(repoPrefix, relPath string) *CrossRe func (cr *CrossRepoResolver) resolveScopedLocked(edges []*graph.Edge) *CrossRepoStats { stats := &CrossRepoStats{ByRepo: make(map[string]int)} _ = cr.resolveTrackedNpmImports(context.Background(), slices.Values(edges), stats) - _ = cr.resolveCargoImports(context.Background(), slices.Values(edges), stats) + cargoReopened, _ := cr.resolveCargoImports(context.Background(), slices.Values(edges), stats) reopened, _ := cr.resolveTrackedPythonImports(context.Background(), slices.Values(edges), stats, true) + reopened = append(reopened, cargoReopened...) if len(reopened) > 0 { seen := make(map[*graph.Edge]bool, len(edges)) for _, edge := range edges { From 5f042cf0c5a598e7656ea177ce489cc24abaa9e7 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:40:31 -0700 Subject: [PATCH 050/111] fix(resolver): bind calls through grouped and aliased Rust uses The Rust extractor stores the whole use argument, so `use a::b::{X, Y}` gave the item "{X, Y}" and `use a::b::X as Y` gave "X as Y", and calls through either form never bound. The Cargo call bridge now expands the use with parseRustUseBindings and matches each binding's local name, including aliases, against the call. Group members bind only when they live in the linked file. --- internal/resolver/cargo_crate.go | 45 ++++++++++++++++----------- internal/resolver/cargo_crate_test.go | 30 ++++++++++++++++++ 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index 91868429e..dd7d987e0 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -221,31 +221,40 @@ func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target stri continue } spec := strings.TrimPrefix(strings.TrimPrefix(stub, "external::"), "unresolved::import::") - item := spec[strings.LastIndex(spec, "/")+1:] - local := item - if imp.Alias != "" { - local = imp.Alias + bindings := parseRustUseBindings(spec) + if strings.HasSuffix(spec, "/*") { + bindings = []rustUseBinding{{source: spec, local: "*"}} } - for _, candidate := range cr.cachedFindNodesByName(name) { - if candidate.FilePath != imp.To || candidate.Language != "rust" || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { - continue + for _, binding := range bindings { + item, local := rustUseBindingName(binding.source), binding.local + if imp.Alias != "" { + local = imp.Alias + } + lookupName := name + if receiver == "" && local == name { + lookupName = item } - switch candidate.Kind { - case graph.KindFunction: - if strings.HasPrefix(target, "*.") || receiver != "" || (item != "*" && local != name) { + for _, candidate := range cr.cachedFindNodesByName(lookupName) { + if candidate.FilePath != imp.To || candidate.Language != "rust" || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { continue } - case graph.KindMethod: - if receiver == "" || receiver != local || nodeReceiverType(candidate) != item { + switch candidate.Kind { + case graph.KindFunction: + if strings.HasPrefix(target, "*.") || receiver != "" || (item != "*" && local != name) { + continue + } + case graph.KindMethod: + if receiver == "" || receiver != local || nodeReceiverType(candidate) != item { + continue + } + default: continue } - default: - continue - } - if match != nil && match.ID != candidate.ID { - return false + if match != nil && match.ID != candidate.ID { + return false + } + match = candidate } - match = candidate } } if match == nil { diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go index dd025550e..88e38ffd8 100644 --- a/internal/resolver/cargo_crate_test.go +++ b/internal/resolver/cargo_crate_test.go @@ -4,6 +4,7 @@ import ( "context" "iter" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -105,3 +106,32 @@ func TestCargoImportLightPaging(t *testing.T) { }) } } + +func TestCargoGroupedAliasedUses(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, use := range []string{"other_crate/util/{action, Other}", "other_crate/util/action as act"} { + t.Run(backend+"/"+use, func(t *testing.T) { + g := goImportReviewStore(t, backend) + file, provider := "consumer/src/lib.rs", "provider/src/util.rs" + for _, path := range []string{file, "provider/src/lib.rs", provider} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + g.AddNode(&graph.Node{ID: provider + "::action", Name: "action", FilePath: provider, Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + local := "action" + if strings.Contains(use, " as ") { + local = "act" + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "external::" + use, Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::" + local, Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { + return []CargoCrate{{Root: "provider/src/lib.rs", Name: "other_crate"}} + }) + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + require.Equal(t, provider, outEdgesOfKind(g, file, graph.EdgeImports)[0].To) + require.Equal(t, provider+"::action", outEdgesOfKind(g, file, graph.EdgeCalls)[0].To) + }) + } + } +} From 8946c3bf7cb5224f16ad6f870044569ab19da924 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:40:47 -0700 Subject: [PATCH 051/111] docs(multi-repo): state the Cargo module-call and grouped-use limits Calls through an imported module, such as `use lib::util; util::action()`, are not linked by the Cargo bridge. Grouped uses bind only items in the linked module file. --- docs/multi-repo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/multi-repo.md b/docs/multi-repo.md index 570116a13..216935d9e 100644 --- a/docs/multi-repo.md +++ b/docs/multi-repo.md @@ -526,7 +526,7 @@ Scoped tool responses carry a `scope_applied` meta field plus a one-line widen h - A cross-repo module binding carries confidence `0.6` and origin `ast_inferred`. It is a source hint, not proof that the tracked checkout matches the required version. The edge retains its symbol-bearing import identity so provider edits and full reindexes can rebind it without editing the caller. Module-map changes re-check existing bindings. - **Limits:** Only each tracked root's `go.mod` identity is used. Separately tracked nested modules can win by longest path; nested manifests within one tracked root and `go.work` selection are not resolved. Required versions and module/version replacements are not verified. A local `replace` whose destination is untracked prevents module-path binding instead of selecting another checkout with the same module name. Tracked local replacements remain source hints, not full Go build-list resolution. Calls on values typed from another module, such as `client.Do()` for a `library.Client`, stay unresolved unless another pass has precise evidence. Importing the module alone does not enable name-only receiver binding. - Rust `use` imports link to the declared provider's library root or module file. Dependency, dev-dependency, build-dependency and target tables are read. Calls bind only to a unique imported item or receiver in that file. - - **Cargo limits:** This is not Cargo build resolution. Versions, features and target conditions are not evaluated. Absolute or escaping library paths, conflicting aliases and multiple eligible providers are refused. Re-exports and untyped receivers are not inferred. Existing links are not always revisited after declaration or provider-set changes; reindex the provider to refresh them. + - **Cargo limits:** This is not Cargo build resolution. Versions, features and target conditions are not evaluated. Absolute or escaping library paths, conflicting aliases and multiple eligible providers are refused. Re-exports and untyped receivers are not inferred. Module-qualified calls such as `use lib::util; util::action()` are not linked by this bridge. Grouped uses bind calls only when their imported items share the linked module file. Existing links are not always revisited after declaration or provider-set changes; reindex the provider to refresh them. - Python distribution imports link only after a consumer declaration and a unique eligible provider match. Qualified calls use the linked module file. Typed methods also require a matching receiver in that file. Package calls can follow the one-level re-exports described above. - **Python limits:** Distribution versions and arbitrary source layouts are not verified. Providers must expose indexed root or `src/` modules or regular packages. Namespace-only providers and dynamic imports are not linked. Local top-level names take priority, including namespace directories. - **Impact analysis** — `explain_change_impact`, `verify_change`, and `get_test_targets` follow cross-repo edges automatically, grouping results by repository. From 1da873d59c1b16d7c8e9a777e2e771a6b3c920f3 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:40:55 -0700 Subject: [PATCH 052/111] perf(resolver): hydrate only pending callers' imports on scoped passes buildReachableReposIndex loaded full metadata for every cross-repo Go, Rust and Python import in the corpus, on every scoped resolve with pending edges, although a scoped pass reads the index only for the pending callers' files. Scoped passes now pass the pending caller files and hydrate only their imports. Full passes keep the corpus-wide load. --- internal/resolver/cargo_crate_test.go | 19 +++++++++++++++++++ internal/resolver/cross_repo.go | 17 ++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go index 88e38ffd8..bbc61edc0 100644 --- a/internal/resolver/cargo_crate_test.go +++ b/internal/resolver/cargo_crate_test.go @@ -2,6 +2,7 @@ package resolver import ( "context" + "fmt" "iter" "path/filepath" "strings" @@ -135,3 +136,21 @@ func TestCargoGroupedAliasedUses(t *testing.T) { } } } + +func TestScopedBridgeImportHydration(t *testing.T) { + g := &cargoPagingStore{Store: graph.New()} + for i := 0; i < 12; i++ { + file := fmt.Sprintf("app/%d.rs", i) + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "app", Language: "rust"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "lib/src/lib.rs", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::other/action"}}) + } + g.AddNode(&graph.Node{ID: "lib/src/lib.rs", FilePath: "lib/src/lib.rs", Kind: graph.KindFile, RepoPrefix: "lib", Language: "rust"}) + g.AddEdge(&graph.Edge{From: "app/0.rs", FilePath: "app/0.rs", To: "unresolved::action", Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.ResolveForFile("app", "0.rs") + require.Equal(t, 1, g.maxSites, "only the pending caller's import metadata may be hydrated") + g.maxSites = 0 + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + require.Equal(t, 12, g.maxSites, "full passes retain corpus-wide import evidence") +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 7155d6a35..a2743fc11 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -469,7 +469,7 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS defer cr.clearDirIndexes() cr.buildDepModuleIndex() defer cr.clearDepModuleIndex() - cr.buildReachableReposIndex() + cr.buildReachableReposIndex(nil) defer cr.clearReachableReposIndex() if resolveHotCacheEnabled() { @@ -753,7 +753,11 @@ func (cr *CrossRepoResolver) resolveScopedLocked(edges []*graph.Edge) *CrossRepo defer cr.clearDirIndexes() cr.buildDepModuleIndex() defer cr.clearDepModuleIndex() - cr.buildReachableReposIndex() + callerFiles := make(map[string]bool) + for _, edge := range pending { + callerFiles[cr.callerFileID(edge)] = true + } + cr.buildReachableReposIndex(callerFiles) defer cr.clearReachableReposIndex() cr.warmLookupCache(pending) defer cr.clearLookupCache() @@ -859,7 +863,7 @@ func (cr *CrossRepoResolver) clearDirIndexes() { // resolves imports — including cross-repo imports, with a precise // import-path match — so by the time this index is built the import // graph is settled enough to be trustworthy evidence. -func (cr *CrossRepoResolver) buildReachableReposIndex() { +func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]bool) { idx := make(map[string]map[string]struct{}) dirs := make(map[string]map[string]struct{}) cr.jsTSBindingsByFile = make(map[string]map[string]string) @@ -871,6 +875,9 @@ func (cr *CrossRepoResolver) buildReachableReposIndex() { var imports []*graph.Edge ids := make(map[string]struct{}) for e := range graph.EdgesLightSeq(cr.graph, graph.EdgeImports) { + if callerFiles != nil && !callerFiles[e.From] { + continue + } imports = append(imports, e) if e.To != "" { ids[e.To] = struct{}{} @@ -894,8 +901,8 @@ func (cr *CrossRepoResolver) buildReachableReposIndex() { sites = append(sites, graph.EdgeSite{From: e.From, Line: e.Line, Kind: graph.EdgeImports}) } } - // Markers belong to the persisted import, including imports outside a - // scoped resolution frontier. Hydrate only cross-repo imports. + // Full passes retain every import; scoped passes load metadata only + // for the caller files represented by pending edges. full := cr.graph.GetEdgeCandidates(nil, sites) for _, e := range page { for _, candidate := range full.Site(e.From, e.Line, graph.EdgeImports) { From f741317c8354346673298f35cda840f46952b387 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:41:08 -0700 Subject: [PATCH 053/111] perf(resolver): look up bridge call candidates in provider repos only The Cargo and Python call bridges ran a graph-wide name lookup inside the loop over the caller file's bridge imports and kept only nodes in the import's target file. The Python qualified lookup had also moved off the repo-indexed candidates. Per edge, the cost grew with imports times same-named nodes across all repos. Each edge now collects its provider repos once, looks up the name in those repos only, and buckets the candidates by file before matching the imports. Results are unchanged. --- .../bridge_import_reachability_test.go | 59 +++++++++++++++++++ internal/resolver/cargo_crate.go | 19 +++++- internal/resolver/cross_repo.go | 32 +++++++++- internal/resolver/python_tracked_import.go | 9 ++- 4 files changed, 115 insertions(+), 4 deletions(-) diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go index 6b097275e..175aefd38 100644 --- a/internal/resolver/bridge_import_reachability_test.go +++ b/internal/resolver/bridge_import_reachability_test.go @@ -1,6 +1,7 @@ package resolver import ( + "fmt" "path/filepath" "testing" @@ -157,3 +158,61 @@ func TestPythonMethodsStayWithinImportedType(t *testing.T) { } } } + +type bridgeNameCountStore struct { + graph.Store + broad, scoped int +} + +func (s *bridgeNameCountStore) FindNodesByName(name string) []*graph.Node { + s.broad++ + return s.Store.FindNodesByName(name) +} +func (s *bridgeNameCountStore) FindNodesByNameInRepo(name, repo string) []*graph.Node { + s.scoped++ + return s.Store.FindNodesByNameInRepo(name, repo) +} +func TestBridgeCandidatesLookupOncePerProvider(t *testing.T) { + for _, language := range []string{"rust", "python"} { + t.Run(language, func(t *testing.T) { + g := &bridgeNameCountStore{Store: graph.New()} + cr := NewCrossRepo(g) + file := "consumer/app.py" + name := "show" + target := "*.show" + if language == "rust" { + file = "consumer/src/lib.rs" + name = "action" + target = "action" + } + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: language}) + var imports []*graph.Edge + for i := 0; i < 20; i++ { + provider := fmt.Sprintf("provider/pkg/mod%d.py", i) + marker := map[string]any{PythonImportStubMeta: "external::pkg.mod", "python_import": fmt.Sprintf("pkg.mod%d", i), "python_bindings": map[string]any{"Image": "Image"}} + if language == "rust" { + provider = fmt.Sprintf("provider/src/mod%d.rs", i) + marker = map[string]any{CargoImportStubMeta: "external::library/action"} + } + imports = append(imports, &graph.Edge{From: file, FilePath: file, To: provider, Meta: marker}) + if i == 19 { + kind := graph.KindFunction + if language == "python" { + kind = graph.KindMethod + } + g.AddNode(&graph.Node{ID: provider + "::" + name, FilePath: provider, Name: name, Kind: kind, RepoPrefix: "provider", Language: language, Meta: map[string]any{"receiver": "Image"}}) + } + } + cr.bridgeImportsByFile = map[string][]*graph.Edge{file: imports} + e := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{"receiver_type": "Image"}} + if language == "rust" { + delete(e.Meta, "receiver_type") + require.True(t, cr.resolveCargoImportedCall(e, target)) + } else { + require.True(t, cr.resolvePythonImportedMethod(e, target)) + } + require.Zero(t, g.broad, "bridge lookups must not scan all repositories") + require.LessOrEqual(t, g.scoped, 2, "one provider lookup plus Python local-type check") + }) + } +} diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index dd7d987e0..c13101677 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -214,6 +214,23 @@ func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target stri if owner, _, ok := strings.Cut(scoped, "::"); ok { receiver = owner } + names := map[string]bool{name: true} + for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { + stub, _ := imp.Meta[CargoImportStubMeta].(string) + spec := strings.TrimPrefix(strings.TrimPrefix(stub, "external::"), "unresolved::import::") + for _, binding := range parseRustUseBindings(spec) { + if receiver == "" && (binding.local == name || imp.Alias == name) { + names[rustUseBindingName(binding.source)] = true + } + } + } + candidates := make(map[string]map[string][]*graph.Node) + for name, nodes := range cr.bridgeCandidates(e, names, false) { + candidates[name] = make(map[string][]*graph.Node) + for _, node := range nodes { + candidates[name][node.FilePath] = append(candidates[name][node.FilePath], node) + } + } var match *graph.Node for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { stub, _ := imp.Meta[CargoImportStubMeta].(string) @@ -234,7 +251,7 @@ func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target stri if receiver == "" && local == name { lookupName = item } - for _, candidate := range cr.cachedFindNodesByName(lookupName) { + for _, candidate := range candidates[lookupName][imp.To] { if candidate.FilePath != imp.To || candidate.Language != "rust" || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { continue } diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index a2743fc11..274f7ae5b 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -1227,6 +1227,35 @@ func (cr *CrossRepoResolver) scopedCandidates(e *graph.Edge, name string) []*gra return out } +// bridgeCandidates loads each name once per relevant provider, not per import. +func (cr *CrossRepoResolver) bridgeCandidates(e *graph.Edge, names map[string]bool, includeReachable bool) map[string][]*graph.Node { + repos := make(map[string]bool) + file := cr.callerFileID(e) + for _, imp := range cr.bridgeImportsByFile[file] { + repos[graph.RepoPrefixOfID(imp.To)] = true + } + if includeReachable { + repos[cr.callerRepoPrefix(e)] = true + for repo := range cr.reachableReposByFile[file] { + repos[repo] = true + } + } + result := make(map[string][]*graph.Node) + for name := range names { + for repo := range repos { + if repo == "" { + continue + } + if byRepo, ok := cr.nodesByNameRepo[name]; ok { + result[name] = append(result[name], byRepo[repo]...) + } else { + result[name] = append(result[name], cr.graph.FindNodesByNameInRepo(name, repo)...) + } + } + } + return result +} + // cachedGetNode consults the per-pass id cache first, falling through to // the store on a miss (positive-only: absence means "not pre-warmed"). func (cr *CrossRepoResolver) cachedGetNode(id string) *graph.Node { @@ -1307,8 +1336,9 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b reachable := cr.reachabilityChecker(e) importedDirs := cr.importedDirsByFile[cr.callerFileID(e)] find := func(name string) ([]*graph.Node, error) { + candidates := cr.bridgeCandidates(e, map[string]bool{name: true}, true)[name] var eligible []*graph.Node - for _, node := range cr.cachedFindNodesByName(name) { + for _, node := range candidates { if cr.linkedPythonCallCandidate(e, node, module) { eligible = append(eligible, node) continue diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go index 6ecc09e94..9c997d87d 100644 --- a/internal/resolver/python_tracked_import.go +++ b/internal/resolver/python_tracked_import.go @@ -297,11 +297,16 @@ func (cr *CrossRepoResolver) resolvePythonImportedMethod(e *graph.Edge, target s if receiver == "" { return false } - for _, node := range cr.cachedFindNodesByName(receiver) { + for _, node := range cr.graph.FindNodesByNameInRepo(receiver, cr.callerRepoPrefix(e)) { if node.Kind == graph.KindType && node.RepoPrefix == graph.RepoPrefixOfID(e.From) { return false } } + candidates := make(map[string][]*graph.Node) + name := strings.TrimPrefix(target, "*.") + for _, node := range cr.bridgeCandidates(e, map[string]bool{name: true}, false)[name] { + candidates[node.FilePath] = append(candidates[node.FilePath], node) + } var match *graph.Node for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { spec, _ := imp.Meta["python_import"].(string) @@ -313,7 +318,7 @@ func (cr *CrossRepoResolver) resolvePythonImportedMethod(e *graph.Edge, target s if imported == "" { continue } - for _, candidate := range cr.cachedFindNodesByName(strings.TrimPrefix(target, "*.")) { + for _, candidate := range candidates[imp.To] { if candidate.Kind != graph.KindMethod || candidate.FilePath != imp.To || nodeReceiverType(candidate) != imported || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { continue } From e5ec71d97587268512ce7e0264fb047af910d1a0 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:41:18 -0700 Subject: [PATCH 054/111] perf(indexer): skip Cargo import scans when no repo has a Cargo root Moving the Cargo.toml check into the deferred probe made the Cargo lookup non-nil in every workspace with two or more repos, so callers that skip work on a nil lookup ran whole-graph import scans with no Rust present. The lookup now returns nil when no tracked root has a Cargo.toml (one stat per root). Manifest reads stay deferred. The no-manifest test now covers two repos and checks that no import scan runs. --- internal/indexer/cargo_crate.go | 11 +++++------ internal/indexer/cargo_crate_test.go | 7 +++++-- internal/indexer/cargo_import_warm_recovery_test.go | 7 +++++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go index 1aabc67df..ceb051d21 100644 --- a/internal/indexer/cargo_crate.go +++ b/internal/indexer/cargo_crate.go @@ -38,10 +38,13 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { roots := map[string]manifestTree{} for prefix, meta := range mi.AllMetadata() { if meta != nil && meta.RootPath != "" { - roots[prefix] = mi.manifestTreeFor(prefix, meta.RootPath) + tree := mi.manifestTreeFor(prefix, meta.RootPath) + if tree.isFile("Cargo.toml") { + roots[prefix] = tree + } } } - if len(roots) < 2 { + if len(roots) == 0 { return nil } owners := map[string]*crate{} @@ -52,10 +55,6 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { return func(file, name string) []resolver.CargoCrate { once.Do(func() { for prefix, tree := range roots { - if !tree.isFile("Cargo.toml") { - delete(roots, prefix) - continue - } manifest := readCargoManifest(tree, ".") src, _ := tree.readFile("Cargo.toml") members, exclude := modules.ParseCargoWorkspace(src) diff --git a/internal/indexer/cargo_crate_test.go b/internal/indexer/cargo_crate_test.go index df4781b4c..aed5a8fea 100644 --- a/internal/indexer/cargo_crate_test.go +++ b/internal/indexer/cargo_crate_test.go @@ -165,6 +165,9 @@ func TestMultiIndexer_CargoCrateImport(t *testing.T) { } func TestCargoCrateLookupNoManifest(t *testing.T) { - mi := &MultiIndexer{repos: map[string]*RepoMetadata{"plain": {RootPath: t.TempDir()}}} - assert.Nil(t, mi.cargoCrateLookup(), "no Cargo manifest must leave the retry disabled") + g := &cargoWarmScanStore{Store: graph.New()} + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"first": {RootPath: t.TempDir()}, "second": {RootPath: t.TempDir()}}} + assert.Nil(t, mi.cargoCrateLookup(), "a multi-repo workspace without Cargo roots must disable the retry") + require.Empty(t, mi.cargoConsumerFilesForRepo("first", false)) + assert.Zero(t, g.lightScans, "no import scan is needed without Cargo roots") } diff --git a/internal/indexer/cargo_import_warm_recovery_test.go b/internal/indexer/cargo_import_warm_recovery_test.go index 7807530d1..9c88f5c83 100644 --- a/internal/indexer/cargo_import_warm_recovery_test.go +++ b/internal/indexer/cargo_import_warm_recovery_test.go @@ -194,11 +194,14 @@ func TestCargoWarmRecoveryScansImportsOnce(t *testing.T) { require.Equal(t, 1, g.lightScans, "all receipt repos must share one scan") } -func TestCargoLookupDefersManifestProbesUntilUse(t *testing.T) { +func TestCargoLookupDefersManifestReadsUntilUse(t *testing.T) { provider, consumer := t.TempDir(), t.TempDir() mi := &MultiIndexer{graph: graph.New(), repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + for _, root := range []string{provider, consumer} { + writeWorkspaceFile(t, root, "Cargo.toml", "") + } lookup := mi.cargoCrateLookup() - require.NotNil(t, lookup, "construction must not probe Cargo manifests") + require.NotNil(t, lookup, "construction may stat roots but must defer reading Cargo manifests") writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname='library'\nversion='1.0.0'\n") writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname='app'\nversion='1.0.0'\n[dependencies]\nlibrary='1'\n") providers := lookup("consumer/src/lib.rs", "library") From 1ebb921991d80433c548a259cacea6778e7215e3 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:41:38 -0700 Subject: [PATCH 055/111] docs(multi-repo): state that working Python links are not rechecked A linked Python import is not reconsidered when a second provider appears, the dependency declaration is dropped, or a new local top-level package shadows it. The Cargo limits line already states the same gap. --- docs/multi-repo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/multi-repo.md b/docs/multi-repo.md index 216935d9e..5ebe51d6d 100644 --- a/docs/multi-repo.md +++ b/docs/multi-repo.md @@ -528,6 +528,6 @@ Scoped tool responses carry a `scope_applied` meta field plus a one-line widen h - Rust `use` imports link to the declared provider's library root or module file. Dependency, dev-dependency, build-dependency and target tables are read. Calls bind only to a unique imported item or receiver in that file. - **Cargo limits:** This is not Cargo build resolution. Versions, features and target conditions are not evaluated. Absolute or escaping library paths, conflicting aliases and multiple eligible providers are refused. Re-exports and untyped receivers are not inferred. Module-qualified calls such as `use lib::util; util::action()` are not linked by this bridge. Grouped uses bind calls only when their imported items share the linked module file. Existing links are not always revisited after declaration or provider-set changes; reindex the provider to refresh them. - Python distribution imports link only after a consumer declaration and a unique eligible provider match. Qualified calls use the linked module file. Typed methods also require a matching receiver in that file. Package calls can follow the one-level re-exports described above. - - **Python limits:** Distribution versions and arbitrary source layouts are not verified. Providers must expose indexed root or `src/` modules or regular packages. Namespace-only providers and dynamic imports are not linked. Local top-level names take priority, including namespace directories. + - **Python limits:** Distribution versions and arbitrary source layouts are not verified. Providers must expose indexed root or `src/` modules or regular packages. Namespace-only providers and dynamic imports are not linked. Local top-level names take priority, including namespace directories. A working link is not reconsidered when a second provider appears, its dependency declaration is dropped, or a new local top-level package shadows it. Reindex the consumer to refresh it. - **Impact analysis** — `explain_change_impact`, `verify_change`, and `get_test_targets` follow cross-repo edges automatically, grouping results by repository. - **Shared repos** — the same repo can appear in multiple projects with different reference tags. It's indexed once and shared across projects. From 113b67a5a569a2addb015508632c4a031c827669 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:41:49 -0700 Subject: [PATCH 056/111] perf(indexer): gate Python added-file consumers on the repo count pythonAddedFileConsumers built the full Python package lookup (pyproject and requirements reads for every repo) only to test it for nil, on every watcher or reconcile batch that added a .py file. It now checks that two or more repos are loaded. Resolution still validates declarations and providers. --- internal/indexer/python_import_restub.go | 2 +- .../indexer/python_tracked_import_test.go | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index ef87b1dea..bf7f4b7ca 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -186,7 +186,7 @@ func (mi *MultiIndexer) pythonAddedFileConsumers(files []string) []string { configuredRepos := mi.configMgr.Global().Repos warming = len(configuredRepos) > 1 && len(mi.AllMetadata()) < len(configuredRepos) } - if !warming && mi.pythonPackageLookup() == nil { + if !warming && len(mi.AllMetadata()) < 2 { return nil } return pythonConsumerFilesForAddedFiles(mi.graph, files) diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go index 60add38bf..a2e14fe11 100644 --- a/internal/indexer/python_tracked_import_test.go +++ b/internal/indexer/python_tracked_import_test.go @@ -143,6 +143,12 @@ type pythonLookupScanStore struct { graph.Store globalScans int importScans int + nodeReads int +} + +func (s *pythonLookupScanStore) GetNode(id string) *graph.Node { + s.nodeReads++ + return s.Store.GetNode(id) } func (s *pythonLookupScanStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { @@ -307,3 +313,16 @@ func TestSingleRepoPythonAddedFileSkipsCrossRepoScan(t *testing.T) { require.NoError(t, err) require.Zero(t, g.importScans, "a lone Python repo has no foreign provider to recover") } + +func TestPythonAddedFileGateDoesNotBuildLookup(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname='library'\n") + writeWorkspaceFile(t, provider, "mylib.py", "") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies=['library']\n") + g := &pythonLookupScanStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "provider/mylib.py", FilePath: "provider/mylib.py", Kind: graph.KindFile, RepoPrefix: "provider"}) + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: "external::mylib", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + require.Equal(t, []string{"consumer/app.py"}, mi.pythonAddedFileConsumers([]string{"provider/mylib.py"})) + require.Zero(t, g.nodeReads, "the added-file candidate gate must not build package lookup") +} From 3a3eb58dd229fdddafe1c6cd427f6d6ea1725e79 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:41:59 -0700 Subject: [PATCH 057/111] fix(indexer): read configured repos through ReposSnapshot The warm-registration check in pythonAddedFileConsumers read Global().Repos directly, which AddRepo and RemoveRepo mutate under globalConfigMu, and dereferenced a nil Global(). It now checks for nil and uses ReposSnapshot, as the other configured-repo readers do. --- internal/indexer/python_import_restub.go | 6 ++++-- internal/indexer/python_tracked_import_test.go | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index bf7f4b7ca..e51fc683b 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -183,8 +183,10 @@ func (mi *MultiIndexer) pythonAddedFileConsumers(files []string) []string { // are registered. Preserve that frontier until the workspace pass. warming := false if mi.configMgr != nil { - configuredRepos := mi.configMgr.Global().Repos - warming = len(configuredRepos) > 1 && len(mi.AllMetadata()) < len(configuredRepos) + if global := mi.configMgr.Global(); global != nil { + configuredRepos := global.ReposSnapshot() + warming = len(configuredRepos) > 1 && len(mi.AllMetadata()) < len(configuredRepos) + } } if !warming && len(mi.AllMetadata()) < 2 { return nil diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go index a2e14fe11..af9b1ce5d 100644 --- a/internal/indexer/python_tracked_import_test.go +++ b/internal/indexer/python_tracked_import_test.go @@ -326,3 +326,8 @@ func TestPythonAddedFileGateDoesNotBuildLookup(t *testing.T) { require.Equal(t, []string{"consumer/app.py"}, mi.pythonAddedFileConsumers([]string{"provider/mylib.py"})) require.Zero(t, g.nodeReads, "the added-file candidate gate must not build package lookup") } + +func TestPythonAddedFileNilGlobal(t *testing.T) { + mi := &MultiIndexer{graph: graph.New(), configMgr: &config.ConfigManager{}} + require.NotPanics(t, func() { require.Empty(t, mi.pythonAddedFileConsumers([]string{"solo/new.py"})) }) +} From 589782cec9c4190804647eca9acc4a1cf9f68e07 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 01:42:08 -0700 Subject: [PATCH 058/111] perf(indexer): filter Python import tops while scanning for consumers The consumer collector kept one tuple for every Python import edge, including resolved imports with no top and tops no tracked distribution provides, and called Providers for each. It now skips those inside the scan, groups files by (caller, top), and calls Providers once per key after the cursor closes. --- internal/indexer/python_import_restub.go | 25 ++++++++++++++----- .../indexer/python_tracked_import_test.go | 21 ++++++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index e51fc683b..c396b038c 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -115,10 +115,14 @@ func (mi *MultiIndexer) pythonConsumerFilesForRepo(repo string) []string { if lookup == nil { return nil } + return pythonConsumerFilesForLookup(mi.graph, lookup, repo) +} + +func pythonConsumerFilesForLookup(g graph.Store, lookup *resolver.PythonPackageLookup, repo string) []string { files := make(map[string]struct{}) - type candidate struct{ caller, top, file string } - var candidates []candidate - for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { + type candidate struct{ caller, top string } + candidates := make(map[candidate]map[string]struct{}) + for edge := range graph.EdgesLightSeq(g, graph.EdgeImports) { if !strings.HasSuffix(edge.FilePath, ".py") { continue } @@ -129,13 +133,22 @@ func (mi *MultiIndexer) pythonConsumerFilesForRepo(repo string) []string { break } } - candidates = append(candidates, candidate{graph.RepoPrefixOfID(edge.From), top, edge.FilePath}) + if top == "" || !lookup.HasTop(top) { + continue + } + key := candidate{graph.RepoPrefixOfID(edge.From), top} + if candidates[key] == nil { + candidates[key] = make(map[string]struct{}) + } + candidates[key][edge.FilePath] = struct{}{} } // Providers may query the store; release the import cursor first. - for _, item := range candidates { + for item, paths := range candidates { for provider := range lookup.Providers(item.caller, item.top) { if provider == repo || item.caller == repo { - files[item.file] = struct{}{} + for file := range paths { + files[file] = struct{}{} + } break } } diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go index af9b1ce5d..3f636096e 100644 --- a/internal/indexer/python_tracked_import_test.go +++ b/internal/indexer/python_tracked_import_test.go @@ -15,6 +15,7 @@ import ( "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/resolver" "github.com/zzet/gortex/internal/search" ) @@ -331,3 +332,23 @@ func TestPythonAddedFileNilGlobal(t *testing.T) { mi := &MultiIndexer{graph: graph.New(), configMgr: &config.ConfigManager{}} require.NotPanics(t, func() { require.Empty(t, mi.pythonAddedFileConsumers([]string{"solo/new.py"})) }) } + +func TestPythonConsumerLookupFiltersAndDeduplicates(t *testing.T) { + g := graph.New() + for i, target := range []string{"external::known", "external::known.sub", "external::known", "external::unknown", "local/file.py", "module::python:os"} { + file := fmt.Sprintf("consumer/f%d.py", i) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports}) + } + calls := 0 + lookup := &resolver.PythonPackageLookup{ + HasTop: func(top string) bool { return top == "known" }, + Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + calls++ + require.Equal(t, "consumer", caller) + require.Equal(t, "known", top) + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/known.py"}}} + }, + } + require.Equal(t, []string{"consumer/f0.py", "consumer/f1.py", "consumer/f2.py"}, pythonConsumerFilesForLookup(g, lookup, "provider")) + require.Equal(t, 1, calls, "provider lookup is per (caller, top), not per import") +} From d98a5fdc1587ce9ccd069e4a45e42c916c96a6ef Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 06:38:35 -0700 Subject: [PATCH 059/111] fix(resolver): reject nested grouped-use leaf decoys A grouped Rust use such as `use other::util::{action, sub::helper}` or `use other::{util::action, net::fetch}` bound a member whose path goes past the group base by its leaf name in whatever file the import linked. Bind a grouped member only when its parent path equals the group base. --- internal/resolver/cargo_crate.go | 8 +++++ internal/resolver/cargo_crate_test.go | 51 +++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index c13101677..d58d8c8d1 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -242,7 +242,15 @@ func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target stri if strings.HasSuffix(spec, "/*") { bindings = []rustUseBinding{{source: spec, local: "*"}} } + groupBase, _, grouped := strings.Cut(normalizeRustUsePath(spec), "{") + groupBase = strings.TrimSuffix(groupBase, "::") for _, binding := range bindings { + if grouped { + split := strings.LastIndex(binding.source, "::") + if split < 0 || binding.source[:split] != groupBase { + continue + } + } item, local := rustUseBindingName(binding.source), binding.local if imp.Alias != "" { local = imp.Alias diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go index bbc61edc0..62a97d8fc 100644 --- a/internal/resolver/cargo_crate_test.go +++ b/internal/resolver/cargo_crate_test.go @@ -154,3 +154,54 @@ func TestScopedBridgeImportHydration(t *testing.T) { require.NoError(t, err) require.Equal(t, 12, g.maxSites, "full passes retain corpus-wide import evidence") } + +func TestCargoGroupedUsesRejectNestedPathDecoys(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, spec := range []string{"other/util/{action, sub/helper}", "other/{util/action, net/fetch}", "other/{self, action}"} { + t.Run(backend+"/"+spec, func(t *testing.T) { + // Given a real nested item and a same-named decoy in the group's linked file. + g := goImportReviewStore(t, backend) + file, root, module := "consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs" + paths := []string{file, root, module, "provider/src/util/sub.rs", "provider/src/net.rs"} + for _, path := range paths { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + decoy, name := module, "helper" + if strings.HasPrefix(spec, "other/{") { + decoy, name = root, "fetch" + } + if strings.Contains(spec, "{self,") { + name = "action" + } + for _, path := range []string{decoy, "provider/src/util/sub.rs", "provider/src/net.rs"} { + g.AddNode(&graph.Node{ID: path + "::" + name, FilePath: path, Name: name, Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + } + g.AddNode(&graph.Node{ID: module + "::action", FilePath: module, Name: "action", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "external::" + spec, Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::" + name, Kind: graph.EdgeCalls, Line: 2}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::action", Kind: graph.EdgeCalls, Line: 3}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return []CargoCrate{{Root: root, Name: "other"}} }) + // When the tracked bridge resolves the grouped import and its calls. + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + // Then the nested leaf never binds to a decoy in that file. + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 2) + for _, call := range calls { + if call.Line == 2 { + if strings.Contains(spec, "{self,") { + require.Equal(t, root+"::action", call.To) + } else { + require.Equal(t, "unresolved::"+name, call.To) + require.False(t, call.CrossRepo) + } + } + if call.Line == 3 && decoy == module { + require.Equal(t, module+"::action", call.To) + } + } + }) + } + } +} From 05bf46edadd66af10ce99a09d200dc352176d7cc Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 06:38:48 -0700 Subject: [PATCH 060/111] fix(resolver): scope Python receiver shadow checks to the caller file The local-type refusal rejected a typed cross-repo method link when any type with the receiver's name existed anywhere in the caller repo, and it ran an uncached store name query for every receiver-typed edge before checking the file's bridge imports. Return early unless a from-import in the caller file binds the receiver, and refuse only when the caller file itself defines a type with that name. --- .../bridge_import_reachability_test.go | 27 ++++++++++++++++--- internal/resolver/python_tracked_import.go | 19 ++++++++++--- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go index 175aefd38..29ec07f36 100644 --- a/internal/resolver/bridge_import_reachability_test.go +++ b/internal/resolver/bridge_import_reachability_test.go @@ -108,7 +108,7 @@ func TestCargoCallsStayWithinImportedItem(t *testing.T) { func TestPythonMethodsStayWithinImportedType(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { - for _, scenario := range []string{"method", "foreign_type", "ambiguous", "local_type", "alias", "unbound"} { + for _, scenario := range []string{"method", "foreign_type", "ambiguous", "local_type", "other_file_type", "alias", "unbound"} { t.Run(backend+"/"+scenario, func(t *testing.T) { var g graph.Store = graph.New() if backend == "sqlite" { @@ -127,8 +127,13 @@ func TestPythonMethodsStayWithinImportedType(t *testing.T) { if scenario == "ambiguous" { g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/pkg/aaa.py", Kind: graph.EdgeImports, Line: 2, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.aaa", "python_import": "pkg.aaa", "python_bindings": map[string]any{"Image": "Image"}}}) } - if scenario == "local_type" { - g.AddNode(&graph.Node{ID: file + "::Image", Name: "Image", FilePath: file, Kind: graph.KindType, RepoPrefix: "consumer", Language: "python"}) + // Given a same-named type, only a definition in this caller file shadows the import. + if scenario == "local_type" || scenario == "other_file_type" { + typeFile := file + if scenario == "other_file_type" { + typeFile = "consumer/other.py" + } + g.AddNode(&graph.Node{ID: typeFile + "::Image", Name: "Image", FilePath: typeFile, Kind: graph.KindType, RepoPrefix: "consumer", Language: "python"}) } if scenario == "alias" || scenario == "unbound" { imports := outEdgesOfKind(g, file, graph.EdgeImports) @@ -147,7 +152,7 @@ func TestPythonMethodsStayWithinImportedType(t *testing.T) { require.NoError(t, err) calls := outEdgesOfKind(g, file, graph.EdgeCalls) require.Len(t, calls, 1) - if scenario == "method" || scenario == "alias" { + if scenario == "method" || scenario == "alias" || scenario == "other_file_type" { require.Equal(t, "provider/pkg/image.py::Image.show", calls[0].To) require.True(t, calls[0].CrossRepo) } else { @@ -216,3 +221,17 @@ func TestBridgeCandidatesLookupOncePerProvider(t *testing.T) { }) } } + +func TestPythonUnimportedReceiverAvoidsNameLookups(t *testing.T) { + // Given a Python file with no bridge import binding the receiver. + g := &bridgeNameCountStore{Store: graph.New()} + file := "consumer/tests.py" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "python"}) + cr := NewCrossRepo(g) + edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{"receiver_type": "TestCase"}} + // When a typed method is considered by the tracked bridge. + require.False(t, cr.resolvePythonImportedMethod(edge, "*.assertEqual")) + // Then it performs no store name lookup. + require.Zero(t, g.broad) + require.Zero(t, g.scoped) +} diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go index 9c997d87d..7a829bda9 100644 --- a/internal/resolver/python_tracked_import.go +++ b/internal/resolver/python_tracked_import.go @@ -297,8 +297,21 @@ func (cr *CrossRepoResolver) resolvePythonImportedMethod(e *graph.Edge, target s if receiver == "" { return false } - for _, node := range cr.graph.FindNodesByNameInRepo(receiver, cr.callerRepoPrefix(e)) { - if node.Kind == graph.KindType && node.RepoPrefix == graph.RepoPrefixOfID(e.From) { + file := cr.callerFileID(e) + var imports []*graph.Edge + for _, imp := range cr.bridgeImportsByFile[file] { + spec, _ := imp.Meta["python_import"].(string) + bindings, _ := imp.Meta["python_bindings"].(map[string]any) + imported, _ := bindings[receiver].(string) + if imported != "" && imp.Meta[PythonImportStubMeta] != nil && pythonModuleFileMatches(imp.To, spec) { + imports = append(imports, imp) + } + } + if len(imports) == 0 { + return false + } + for _, node := range cr.graph.GetFileNodes(file) { + if node.Kind == graph.KindType && node.Name == receiver { return false } } @@ -308,7 +321,7 @@ func (cr *CrossRepoResolver) resolvePythonImportedMethod(e *graph.Edge, target s candidates[node.FilePath] = append(candidates[node.FilePath], node) } var match *graph.Node - for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { + for _, imp := range imports { spec, _ := imp.Meta["python_import"].(string) if imp.Meta[PythonImportStubMeta] == nil || !pythonModuleFileMatches(imp.To, spec) { continue From f6e69a61fe21fef89554c960a7ac982efa06813b Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 06:38:57 -0700 Subject: [PATCH 061/111] perf(resolver): warm caller nodes before scoped import indexing resolveScopedLocked built its caller-file set before warming the lookup cache, so every pending edge cost one uncached GetNode. Warm the cache first. --- internal/resolver/cargo_crate_test.go | 18 ++++++++++++++++++ internal/resolver/cross_repo.go | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go index 62a97d8fc..cf0f60428 100644 --- a/internal/resolver/cargo_crate_test.go +++ b/internal/resolver/cargo_crate_test.go @@ -205,3 +205,21 @@ func TestCargoGroupedUsesRejectNestedPathDecoys(t *testing.T) { } } } + +func TestScopedCallerFileUsesWarmedNodeCache(t *testing.T) { + // Given many pending edges with one caller node. + g := &resolverBatchCountingStore{Store: graph.New()} + file, caller := "app/file.go", "app/file.go::run" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "app", Language: "go"}) + g.AddNode(&graph.Node{ID: caller, FilePath: file, Kind: graph.KindFunction, RepoPrefix: "app", Language: "go"}) + var edges []*graph.Edge + for i := 0; i < 30; i++ { + edges = append(edges, &graph.Edge{From: caller, FilePath: file, To: "unresolved::missing", Kind: graph.EdgeCalls, Line: i + 1}) + } + g.AddBatch(nil, edges) + // When the scoped resolver builds its pending caller-file set. + cr := NewCrossRepo(g) + cr.resolveScopedLocked(edges) + // Then the batched cache, not one GetNode per edge, supplies caller placement. + require.Zero(t, g.getNodeCalls) +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 274f7ae5b..e12984143 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -749,6 +749,8 @@ func (cr *CrossRepoResolver) resolveScopedLocked(edges []*graph.Edge) *CrossRepo return stats } + cr.warmLookupCache(pending) + defer cr.clearLookupCache() cr.buildDirIndexes() defer cr.clearDirIndexes() cr.buildDepModuleIndex() @@ -759,8 +761,6 @@ func (cr *CrossRepoResolver) resolveScopedLocked(edges []*graph.Edge) *CrossRepo } cr.buildReachableReposIndex(callerFiles) defer cr.clearReachableReposIndex() - cr.warmLookupCache(pending) - defer cr.clearLookupCache() var reindexBatch []graph.EdgeReindex for _, edge := range pending { From f7053028e139b4b73dcf08cf4ec516bbd5259281 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 06:39:12 -0700 Subject: [PATCH 062/111] perf(indexer): skip Cargo recovery without two Cargo roots A cross-repo Cargo link needs a provider in another tracked repo, so with one Cargo root no link can form. Return no lookup unless at least two tracked roots have Cargo.toml, so full passes, tracks and frontiers stop scanning for Cargo imports. --- internal/indexer/cargo_crate.go | 2 +- internal/indexer/cargo_crate_test.go | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go index ceb051d21..41119c893 100644 --- a/internal/indexer/cargo_crate.go +++ b/internal/indexer/cargo_crate.go @@ -44,7 +44,7 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { } } } - if len(roots) == 0 { + if len(roots) < 2 { return nil } owners := map[string]*crate{} diff --git a/internal/indexer/cargo_crate_test.go b/internal/indexer/cargo_crate_test.go index aed5a8fea..fbf561423 100644 --- a/internal/indexer/cargo_crate_test.go +++ b/internal/indexer/cargo_crate_test.go @@ -2,6 +2,7 @@ package indexer import ( "context" + "fmt" "path/filepath" "strings" "testing" @@ -165,9 +166,20 @@ func TestMultiIndexer_CargoCrateImport(t *testing.T) { } func TestCargoCrateLookupNoManifest(t *testing.T) { - g := &cargoWarmScanStore{Store: graph.New()} - mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"first": {RootPath: t.TempDir()}, "second": {RootPath: t.TempDir()}}} - assert.Nil(t, mi.cargoCrateLookup(), "a multi-repo workspace without Cargo roots must disable the retry") - require.Empty(t, mi.cargoConsumerFilesForRepo("first", false)) - assert.Zero(t, g.lightScans, "no import scan is needed without Cargo roots") + for _, cargoRoots := range []int{0, 1} { + t.Run(fmt.Sprintf("cargo_roots=%d", cargoRoots), func(t *testing.T) { + // Given two repositories, with fewer than two Cargo roots. + first, second := t.TempDir(), t.TempDir() + if cargoRoots == 1 { + writeWorkspaceFile(t, first, "Cargo.toml", "[package]\nname='library'\n") + } + g := &cargoWarmScanStore{Store: graph.New()} + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"first": {RootPath: first}, "second": {RootPath: second}}} + // When cross-repo Cargo recovery is requested. + assert.Nil(t, mi.cargoCrateLookup(), "fewer than two Cargo roots cannot form a tracked bridge") + require.Empty(t, mi.cargoConsumerFilesForRepo("first", false)) + // Then it performs no import scan. + assert.Zero(t, g.lightScans, "no cross-repo Cargo provider is possible") + }) + } } From de1aff8440e4091f22d02748874e452fce5047d5 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 06:39:19 -0700 Subject: [PATCH 063/111] test(resolver): cover Cargo glob and call-restub rejection paths Add a call bound through a glob use and a second call bound into another provider file that must keep its link, so the glob case and the restub target-prefix and CrossRepo guards each have a test that fails when the guard is removed. --- .../bridge_import_reachability_test.go | 6 ++- .../resolver/cargo_import_retarget_test.go | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go index 29ec07f36..ac8d7bd21 100644 --- a/internal/resolver/bridge_import_reachability_test.go +++ b/internal/resolver/bridge_import_reachability_test.go @@ -56,7 +56,7 @@ func TestTrackedBridgeImportsDoNotAuthorizeBareNames(t *testing.T) { func TestCargoCallsStayWithinImportedItem(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { - for _, scenario := range []string{"function", "constructor", "method", "foreign_type", "ambiguous"} { + for _, scenario := range []string{"function", "glob", "constructor", "method", "foreign_type", "ambiguous"} { t.Run(backend+"/"+scenario, func(t *testing.T) { var g graph.Store = graph.New() if backend == "sqlite" { @@ -69,6 +69,10 @@ func TestCargoCallsStayWithinImportedItem(t *testing.T) { g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "rust"}) kind, name, target, item := graph.KindFunction, "action", "unresolved::action", "action" meta := map[string]any{} + // Given a glob use, the ordinary function call still needs exact file evidence. + if scenario == "glob" { + item = "*" + } if scenario == "constructor" { kind, name, target, item = graph.KindMethod, "new", "unresolved::new", "Foo" meta["rust_path"] = "Foo::new" diff --git a/internal/resolver/cargo_import_retarget_test.go b/internal/resolver/cargo_import_retarget_test.go index 8140dc4c9..513e28fa0 100644 --- a/internal/resolver/cargo_import_retarget_test.go +++ b/internal/resolver/cargo_import_retarget_test.go @@ -174,3 +174,42 @@ func TestCargoRefusedImportWarmMatchesCold(t *testing.T) { } } } + +func TestCargoRestubCallsPreservesUnrelatedLinks(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + // Given a revoked import and three marked calls with different binding evidence. + g := goImportReviewStore(t, backend) + file, target := "consumer/src/lib.rs", "provider/src/util.rs" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "rust"}) + for _, row := range []struct { + to string + cross bool + line int + }{ + {target + "::action", true, 1}, + {"other/src/lib.rs::action", true, 2}, + {target + "::local", false, 3}, + } { + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, To: row.to, Line: row.line, CrossRepo: row.cross, Meta: map[string]any{CargoCallStubMeta: "unresolved::action"}}) + } + // When only calls through the revoked target are restubbed. + reopened := NewCrossRepo(g).restubCargoImportCalls(file, target) + // Then the other provider and non-cross-repo call keep their links. + require.Len(t, reopened, 1) + for _, call := range outEdgesOfKind(g, file, graph.EdgeCalls) { + switch call.Line { + case 1: + require.Equal(t, "unresolved::action", call.To) + require.False(t, call.CrossRepo) + case 2: + require.Equal(t, "other/src/lib.rs::action", call.To) + require.True(t, call.CrossRepo) + case 3: + require.Equal(t, target+"::local", call.To) + require.False(t, call.CrossRepo) + } + } + }) + } +} From 453195e692d65191079449d52b76fee15db4a9f2 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 06:39:36 -0700 Subject: [PATCH 064/111] docs(indexer): name Python import bindings in the extractor version comment --- internal/indexer/extractor_version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/indexer/extractor_version.go b/internal/indexer/extractor_version.go index 340e45c7a..1feb74117 100644 --- a/internal/indexer/extractor_version.go +++ b/internal/indexer/extractor_version.go @@ -49,7 +49,7 @@ var extractorVersions = map[string]int{ "julia": 3, // callee and macro decoding from CST children (chained/parametric callees, Base.@time, Base.:(==)), macro/operator exports, member_of for module consts and nested modules, @doc and public metadata (was: bespoke tree-sitter extractor replaced the regex extractor) "javascript": 2, // renamed per-binding imports persist import_alias Meta; older rows keyed a renamed import by its original name "typescript": 2, // renamed per-binding imports persist import_alias Meta; older rows keyed a renamed import by its original name - "python": 2, // package relative from-imports retain exact re-export bindings + "python": 2, // package relative from-imports retain exact re-export bindings; from-imports retain python_bindings } // extractorSaltExtLang maps a lower-case file extension to the language From 069fa1765bd7ed00e6044ba402ccda3f67ae5f9a Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 06:39:45 -0700 Subject: [PATCH 065/111] docs: state exact Python typed-method bridge conditions Typed methods link only through a receiver name that `from X import Name [as Alias]` binds; plain `import pkg.mod` and wildcard imports never link typed methods, and a same-named type in the caller file refuses the link. --- docs/multi-repo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/multi-repo.md b/docs/multi-repo.md index 5ebe51d6d..a1337b29b 100644 --- a/docs/multi-repo.md +++ b/docs/multi-repo.md @@ -527,7 +527,7 @@ Scoped tool responses carry a `scope_applied` meta field plus a one-line widen h - **Limits:** Only each tracked root's `go.mod` identity is used. Separately tracked nested modules can win by longest path; nested manifests within one tracked root and `go.work` selection are not resolved. Required versions and module/version replacements are not verified. A local `replace` whose destination is untracked prevents module-path binding instead of selecting another checkout with the same module name. Tracked local replacements remain source hints, not full Go build-list resolution. Calls on values typed from another module, such as `client.Do()` for a `library.Client`, stay unresolved unless another pass has precise evidence. Importing the module alone does not enable name-only receiver binding. - Rust `use` imports link to the declared provider's library root or module file. Dependency, dev-dependency, build-dependency and target tables are read. Calls bind only to a unique imported item or receiver in that file. - **Cargo limits:** This is not Cargo build resolution. Versions, features and target conditions are not evaluated. Absolute or escaping library paths, conflicting aliases and multiple eligible providers are refused. Re-exports and untyped receivers are not inferred. Module-qualified calls such as `use lib::util; util::action()` are not linked by this bridge. Grouped uses bind calls only when their imported items share the linked module file. Existing links are not always revisited after declaration or provider-set changes; reindex the provider to refresh them. - - Python distribution imports link only after a consumer declaration and a unique eligible provider match. Qualified calls use the linked module file. Typed methods also require a matching receiver in that file. Package calls can follow the one-level re-exports described above. + - Python distribution imports link only after a consumer declaration and a unique eligible provider match. Qualified calls use the linked module file. Typed methods require a receiver name bound by `from X import Name [as Alias]` and a matching provider method in that file. Plain `import pkg.mod` and wildcard imports never link typed methods. A same-named type defined in the caller file refuses the link. Package calls can follow the one-level re-exports described above. - **Python limits:** Distribution versions and arbitrary source layouts are not verified. Providers must expose indexed root or `src/` modules or regular packages. Namespace-only providers and dynamic imports are not linked. Local top-level names take priority, including namespace directories. A working link is not reconsidered when a second provider appears, its dependency declaration is dropped, or a new local top-level package shadows it. Reindex the consumer to refresh it. - **Impact analysis** — `explain_change_impact`, `verify_change`, and `get_test_targets` follow cross-repo edges automatically, grouping results by repository. - **Shared repos** — the same repo can appear in multiple projects with different reference tags. It's indexed once and shared across projects. From 364a9e3a438740d534efa8576aaf68b8951f1f04 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 07:21:08 -0700 Subject: [PATCH 066/111] fix(resolver): require Cargo members to match the probed module cargoImportTarget falls back to an ancestor module file or the crate root when the imported module has no file of its own, without recording which module matched. `use other::sys::{open}` with no sys.rs then linked open() to a same-named function in lib.rs, and the ungrouped form had the same gap. Return the matched module path, persist it on the import as cargo_import_module, and bind a member call only when its full parent path equals that module. Imports bound before this change carry no matched path and link no member calls until they are revisited. --- .../bridge_import_reachability_test.go | 6 +-- internal/resolver/cargo_crate.go | 53 ++++++++++++------- internal/resolver/cargo_crate_test.go | 44 ++++++++++++++- .../resolver/cargo_import_retarget_test.go | 16 ++++-- internal/resolver/cross_repo.go | 4 +- 5 files changed, 94 insertions(+), 29 deletions(-) diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go index ac8d7bd21..87a931c0c 100644 --- a/internal/resolver/bridge_import_reachability_test.go +++ b/internal/resolver/bridge_import_reachability_test.go @@ -89,9 +89,9 @@ func TestCargoCallsStayWithinImportedItem(t *testing.T) { g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: "provider", Language: "rust"}) g.AddNode(&graph.Node{ID: path + "::" + name, FilePath: path, Name: name, Kind: kind, RepoPrefix: "provider", Language: "rust", Meta: map[string]any{"receiver": "Foo"}}) } - g.AddEdge(&graph.Edge{From: file, FilePath: file, To: imported, Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::library/util/" + item}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: imported, Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::library/util/" + item, CargoImportModuleMeta: "library::util"}}) if scenario == "ambiguous" { - g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/src/aaa.rs", Kind: graph.EdgeImports, Line: 2, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::library/aaa/" + item}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/src/aaa.rs", Kind: graph.EdgeImports, Line: 2, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::library/aaa/" + item, CargoImportModuleMeta: "library::aaa"}}) } g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeCalls, Line: 3, Meta: meta}) _, err := NewCrossRepo(g).ResolveAllContext(t.Context()) @@ -201,7 +201,7 @@ func TestBridgeCandidatesLookupOncePerProvider(t *testing.T) { marker := map[string]any{PythonImportStubMeta: "external::pkg.mod", "python_import": fmt.Sprintf("pkg.mod%d", i), "python_bindings": map[string]any{"Image": "Image"}} if language == "rust" { provider = fmt.Sprintf("provider/src/mod%d.rs", i) - marker = map[string]any{CargoImportStubMeta: "external::library/action"} + marker = map[string]any{CargoImportStubMeta: "external::library/action", CargoImportModuleMeta: "library"} } imports = append(imports, &graph.Edge{From: file, FilePath: file, To: provider, Meta: marker}) if i == 19 { diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index d58d8c8d1..3f284ba09 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -12,6 +12,9 @@ import ( // CargoImportStubMeta preserves the import's exact target before a tracked bind. const CargoImportStubMeta = "cargo_import_stub" +// CargoImportModuleMeta records the module path actually matched by file probing. +const CargoImportModuleMeta = "cargo_import_module" + // CargoCallStubMeta retains the extractor's function or method target before linking. const CargoCallStubMeta = "cargo_call_stub" @@ -38,23 +41,23 @@ func (cr *CrossRepoResolver) SetCargoImportRetargetRepo(repos ...string) { } } -func (cr *CrossRepoResolver) cargoImportTarget(e *graph.Edge, spec string) *graph.Node { +func (cr *CrossRepoResolver) cargoImportTarget(e *graph.Edge, spec string) (*graph.Node, string) { if cr.cargoCrates == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { - return nil + return nil, "" } segs := strings.Split(spec, "/") switch segs[0] { case "", "crate", "self", "super", "std", "core", "alloc": - return nil + return nil, "" } for _, seg := range segs { if seg == ".." || seg == "." || seg == "" { - return nil + return nil, "" } } crates := cr.cargoCrates(e.FilePath, segs[0]) if len(crates) == 0 { - return nil + return nil, "" } callerRepo, callerWS := cr.callerRepoPrefix(e), cr.callerWorkspaceID(e) var root *graph.Node @@ -64,18 +67,18 @@ func (cr *CrossRepoResolver) cargoImportTarget(e *graph.Edge, spec string) *grap continue } if candidate.RepoPrefix == callerRepo { - return nil // same-repo resolution stays unchanged + return nil, "" // same-repo resolution stays unchanged } if !cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(candidate), crate.Name+strings.TrimPrefix(spec, segs[0])) { continue } if root != nil { - return nil // two eligible providers, even if only one has the module + return nil, "" // two eligible providers, even if only one has the module } root = candidate } if root == nil { - return nil + return nil, "" } // Same longest-prefix module-file probes as resolveRustUseFile, anchored // at the manifest's library root rather than a guessed src directory. @@ -85,16 +88,16 @@ func (cr *CrossRepoResolver) cargoImportTarget(e *graph.Edge, spec string) *grap for _, id := range []string{stem + ".rs", stem + "/mod.rs"} { if node := cr.cachedGetNode(id); node != nil && node.Kind == graph.KindFile && node.Language == "rust" { if found != nil { - return nil + return nil, "" } found = node } } if found != nil { - return found + return found, strings.Join(segs[:n], "::") } } - return root // item or glob import from the crate itself + return root, segs[0] // item or glob import from the crate itself } // Per-repo resolution converts missing imports to external stubs. Retry just @@ -142,11 +145,12 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter if e.To != projected.To { continue } - node := cr.cargoImportTarget(e, cr.cargoImportSpec(e)) + node, module := cr.cargoImportTarget(e, cr.cargoImportSpec(e)) if node == nil && e.CrossRepo { if stub, _ := e.Meta[CargoImportStubMeta].(string); stub != "" { oldTo := e.To e.To, e.CrossRepo = stub, false + delete(e.Meta, CargoImportModuleMeta) clearEdgeTerminal(e) batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) projected.To, projected.CrossRepo = e.To, false @@ -155,9 +159,15 @@ func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter } } else if node != nil && node.ID != e.To { oldTo := e.To - cr.bindCargoImport(e, node, stats) + cr.bindCargoImport(e, node, module, stats) batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) projected.To, projected.CrossRepo = e.To, e.CrossRepo + } else if node != nil && e.Meta[CargoImportModuleMeta] != module { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + e.Meta[CargoImportModuleMeta] = module + batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: e.To}) } } } @@ -189,13 +199,14 @@ func (cr *CrossRepoResolver) cargoImportSpec(e *graph.Edge) string { return spec } -func (cr *CrossRepoResolver) bindCargoImport(e *graph.Edge, node *graph.Node, stats *CrossRepoStats) { +func (cr *CrossRepoResolver) bindCargoImport(e *graph.Edge, node *graph.Node, module string, stats *CrossRepoStats) { if e.Meta == nil { e.Meta = make(map[string]any) } if original, _ := e.Meta[CargoImportStubMeta].(string); original == "" { e.Meta[CargoImportStubMeta] = e.To } + e.Meta[CargoImportModuleMeta] = module e.To, e.CrossRepo = node.ID, true clearEdgeTerminal(e) stats.Resolved++ @@ -240,16 +251,18 @@ func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target stri spec := strings.TrimPrefix(strings.TrimPrefix(stub, "external::"), "unresolved::import::") bindings := parseRustUseBindings(spec) if strings.HasSuffix(spec, "/*") { - bindings = []rustUseBinding{{source: spec, local: "*"}} + bindings = []rustUseBinding{{source: normalizeRustUsePath(spec), local: "*"}} } groupBase, _, grouped := strings.Cut(normalizeRustUsePath(spec), "{") groupBase = strings.TrimSuffix(groupBase, "::") + module, _ := imp.Meta[CargoImportModuleMeta].(string) for _, binding := range bindings { - if grouped { - split := strings.LastIndex(binding.source, "::") - if split < 0 || binding.source[:split] != groupBase { - continue - } + split := strings.LastIndex(binding.source, "::") + if split < 0 || module == "" || binding.source[:split] != module { + continue + } + if grouped && binding.source[:split] != groupBase { + continue } item, local := rustUseBindingName(binding.source), binding.local if imp.Alias != "" { diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go index cf0f60428..904ed819c 100644 --- a/internal/resolver/cargo_crate_test.go +++ b/internal/resolver/cargo_crate_test.go @@ -19,7 +19,9 @@ func TestCargoImportUntrackedSkipsNodeReads(t *testing.T) { cr := NewCrossRepo(g) cr.SetCargoCrateLookup(func(_, _ string) []CargoCrate { return nil }) e := &graph.Edge{From: "app/src/lib.rs", To: "external::serde/Item", Kind: graph.EdgeImports, FilePath: "app/src/lib.rs"} - assert.Nil(t, cr.cargoImportTarget(e, "serde/Item")) + node, module := cr.cargoImportTarget(e, "serde/Item") + assert.Nil(t, node) + assert.Empty(t, module) assert.Zero(t, g.getNodeCalls, "unknown crates must be rejected before caller node reads") } @@ -223,3 +225,43 @@ func TestScopedCallerFileUsesWarmedNodeCache(t *testing.T) { // Then the batched cache, not one GetNode per edge, supplies caller placement. require.Zero(t, g.getNodeCalls) } + +func TestCargoCallRequiresExactParentModule(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, spec := range []string{"other/sys/{open}", "other/sys/open", "other/util/sys/{open}", "other/util/sys/open", "other/util/{open}", "other/util/open"} { + t.Run(backend+"/"+spec, func(t *testing.T) { + // Given a missing sys module and decoys in its ancestor files. + g := goImportReviewStore(t, backend) + file, root, module := "consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs" + for _, path := range []string{file, root, module} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + for _, path := range []string{root, module} { + g.AddNode(&graph.Node{ID: path + "::open", FilePath: path, Name: "open", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "external::" + spec, Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::open", Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return []CargoCrate{{Root: root, Name: "other"}} }) + // When the import falls back to the nearest indexed module file. + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + // Then only an exact parent module authorizes its member call. + imp := outEdgesOfKind(g, file, graph.EdgeImports)[0] + matched := "other::util" + if strings.HasPrefix(spec, "other/sys/") { + matched = "other" + } + require.Equal(t, matched, imp.Meta[CargoImportModuleMeta]) + call := outEdgesOfKind(g, file, graph.EdgeCalls)[0] + if strings.Contains(spec, "/sys/") { + require.Equal(t, "unresolved::open", call.To) + require.False(t, call.CrossRepo) + } else { + require.Equal(t, module+"::open", call.To) + require.True(t, call.CrossRepo) + } + }) + } + } +} diff --git a/internal/resolver/cargo_import_retarget_test.go b/internal/resolver/cargo_import_retarget_test.go index 513e28fa0..71348f549 100644 --- a/internal/resolver/cargo_import_retarget_test.go +++ b/internal/resolver/cargo_import_retarget_test.go @@ -24,7 +24,7 @@ func (s *cargoRetargetWriteStore) ReindexEdges(batch []graph.EdgeReindex) { func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { - for _, scenario := range []string{"unresolved_marker", "external_marker", "unmarked", "other_provider", "already_correct", "disabled", "light_projection", "multiple_enabled", "nil_input", "wrong_kind"} { + for _, scenario := range []string{"unresolved_marker", "external_marker", "unmarked", "other_provider", "already_correct", "already_correct_legacy", "disabled", "light_projection", "multiple_enabled", "nil_input", "wrong_kind"} { t.Run(backend+"/"+scenario, func(t *testing.T) { var native graph.Store = graph.New() if backend == "sqlite" { @@ -50,8 +50,11 @@ func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { if scenario == "other_provider" { edge.To = "other/src/lib.rs" } - if scenario == "already_correct" { + if scenario == "already_correct" || scenario == "already_correct_legacy" { edge.To = module + if scenario == "already_correct" { + edge.Meta[CargoImportModuleMeta] = "other_crate::util" + } } priorTarget := edge.To g.AddEdge(edge) @@ -78,7 +81,7 @@ func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { require.NoError(t, err) imports := outEdgesOfKind(g, file, graph.EdgeImports) require.Len(t, imports, 1) - changed := scenario == "unresolved_marker" || scenario == "external_marker" || scenario == "light_projection" || scenario == "multiple_enabled" + changed := scenario == "unresolved_marker" || scenario == "external_marker" || scenario == "light_projection" || scenario == "multiple_enabled" || scenario == "already_correct_legacy" if changed { require.Equal(t, module, imports[0].To) require.Equal(t, 1, g.writes) @@ -89,6 +92,13 @@ func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { if scenario != "unmarked" { require.Equal(t, original, imports[0].Meta[CargoImportStubMeta]) } + if changed || scenario == "already_correct" { + require.Equal(t, "other_crate::util", imports[0].Meta[CargoImportModuleMeta]) + } + if scenario == "already_correct_legacy" { + require.Zero(t, stats.Resolved, "refreshing metadata is not a new binding") + } + priorWrites := g.writes if scenario == "nil_input" || scenario == "wrong_kind" { require.Zero(t, stats.Resolved) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index e12984143..f59224b06 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -1653,8 +1653,8 @@ func (cr *CrossRepoResolver) resolveImport(e *graph.Edge, importPath string, sta // evidence fallbacks below before deciding that the import is ambiguous. } - if node := cr.cargoImportTarget(e, importPath); node != nil { - cr.bindCargoImport(e, node, stats) + if node, module := cr.cargoImportTarget(e, importPath); node != nil { + cr.bindCargoImport(e, node, module, stats) return } From 8e1bb8247a479a84176267ff83296b584cf09b01 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 07:21:26 -0700 Subject: [PATCH 067/111] perf(resolver): cache Python caller-file types for each pass The Python shadow check loaded all of the caller file's nodes again for every call on an imported type. Snapshot the local type names once per Python bridge file with from-import bindings before resolution starts, read it during the pass, and clear it afterwards. --- .../bridge_import_reachability_test.go | 59 +++++++++++++++++++ internal/resolver/cross_repo.go | 18 +++++- internal/resolver/python_tracked_import.go | 6 +- 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go index 87a931c0c..74d1c3592 100644 --- a/internal/resolver/bridge_import_reachability_test.go +++ b/internal/resolver/bridge_import_reachability_test.go @@ -171,6 +171,12 @@ func TestPythonMethodsStayWithinImportedType(t *testing.T) { type bridgeNameCountStore struct { graph.Store broad, scoped int + fileReads int +} + +func (s *bridgeNameCountStore) GetFileNodes(file string) []*graph.Node { + s.fileReads++ + return s.Store.GetFileNodes(file) } func (s *bridgeNameCountStore) FindNodesByName(name string) []*graph.Node { @@ -239,3 +245,56 @@ func TestPythonUnimportedReceiverAvoidsNameLookups(t *testing.T) { require.Zero(t, g.broad) require.Zero(t, g.scoped) } + +func TestPythonReceiverTypesReadOncePerFilePerPass(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scoped := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/scoped=%v", backend, scoped), func(t *testing.T) { + // Given eight calls on one imported receiver in a single file. + g := &bridgeNameCountStore{Store: goImportReviewStore(t, backend)} + file, provider := "consumer/app.py", "provider/pkg/image.py" + for _, path := range []string{file, provider} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "python"}) + } + g.AddNode(&graph.Node{ID: provider + "::Image.show", Name: "show", FilePath: provider, Kind: graph.KindMethod, RepoPrefix: "provider", Language: "python", Meta: map[string]any{"receiver": "Image"}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: provider, Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image", "python_bindings": map[string]any{"Image": "Image"}}}) + for i := 0; i < 8; i++ { + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::*.show", Kind: graph.EdgeCalls, Line: i + 2, Meta: map[string]any{"receiver_type": "Image"}}) + } + cr := NewCrossRepo(g) + resolve := func() { + if scoped { + cr.resolveScopedLocked(g.GetOutEdges(file)) + } else { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + } + } + // When a full or scoped pass resolves the calls. + resolve() + // Then local type names are read once, not once per edge. + require.Equal(t, 1, g.fileReads) + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 8) + for _, call := range calls { + require.Equal(t, provider+"::Image.show", call.To) + } + // A later pass must refresh the snapshot after a same-file type is added. + g.AddNode(&graph.Node{ID: file + "::Image", Name: "Image", FilePath: file, Kind: graph.KindType, RepoPrefix: "consumer", Language: "python"}) + var batch []graph.EdgeReindex + for _, call := range calls { + old := call.To + call.To = "unresolved::*.show" + call.CrossRepo = false + batch = append(batch, graph.EdgeReindex{Edge: call, OldTo: old}) + } + g.ReindexEdges(batch) + resolve() + require.Equal(t, 2, g.fileReads) + for _, call := range outEdgesOfKind(g, file, graph.EdgeCalls) { + require.Equal(t, "unresolved::*.show", call.To) + } + }) + } + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index f59224b06..1365c4121 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -111,9 +111,10 @@ type CrossRepoResolver struct { // importedDirsByFile retains the package placement from those same import // rows. A Go call the per-repo package guard refused must not regain a // same-repo name-only binding just because the repository is reachable. - importedDirsByFile map[string]map[string]struct{} - jsTSBindingsByFile map[string]map[string]string - bridgeImportsByFile map[string][]*graph.Edge + importedDirsByFile map[string]map[string]struct{} + jsTSBindingsByFile map[string]map[string]string + bridgeImportsByFile map[string][]*graph.Edge + pythonLocalTypesByFile map[string]map[string]bool // depModuleIndex bridges Go imports to dep:: contract // nodes from the caller's go.mod. Same shape and rationale as // the field of the same name on Resolver — see resolver.go for @@ -868,6 +869,7 @@ func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]boo dirs := make(map[string]map[string]struct{}) cr.jsTSBindingsByFile = make(map[string]map[string]string) cr.bridgeImportsByFile = make(map[string][]*graph.Edge) + cr.pythonLocalTypesByFile = make(map[string]map[string]bool) // Materialise metadata-free import projections and batch-load only target // placement fields. A per-edge point lookup here is a query round-trip per // import on a disk backend, which under the cross-repo pass's import @@ -954,6 +956,15 @@ func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]boo if e.Meta[CargoImportStubMeta] != nil || e.Meta[PythonImportStubMeta] != nil { cr.bridgeImportsByFile[e.From] = append(cr.bridgeImportsByFile[e.From], e) } + if bindings, _ := e.Meta["python_bindings"].(map[string]any); isPythonSourcePath(e.FilePath) && e.Meta[PythonImportStubMeta] != nil && len(bindings) > 0 && cr.pythonLocalTypesByFile[e.From] == nil { + types := make(map[string]bool) + for _, node := range cr.graph.GetFileNodes(e.FilePath) { + if node.Kind == graph.KindType { + types[node.Name] = true + } + } + cr.pythonLocalTypesByFile[e.From] = types + } // A linked module file is not evidence for every symbol in its repo. if npmLinked || e.Meta["go_import"] != nil || e.Meta[CargoImportStubMeta] != nil || e.Meta[PythonImportStubMeta] != nil { continue @@ -975,6 +986,7 @@ func (cr *CrossRepoResolver) clearReachableReposIndex() { cr.importedDirsByFile = nil cr.jsTSBindingsByFile = nil cr.bridgeImportsByFile = nil + cr.pythonLocalTypesByFile = nil } // A package-guard refusal is stronger than the cross-repo pass's same-repo diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go index 7a829bda9..a58c52e45 100644 --- a/internal/resolver/python_tracked_import.go +++ b/internal/resolver/python_tracked_import.go @@ -310,10 +310,8 @@ func (cr *CrossRepoResolver) resolvePythonImportedMethod(e *graph.Edge, target s if len(imports) == 0 { return false } - for _, node := range cr.graph.GetFileNodes(file) { - if node.Kind == graph.KindType && node.Name == receiver { - return false - } + if cr.pythonLocalTypesByFile[file][receiver] { + return false } candidates := make(map[string][]*graph.Node) name := strings.TrimPrefix(target, "*.") From c36f79f8f8f3ff6ab1df8957584ae301affc896f Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 07:21:35 -0700 Subject: [PATCH 068/111] test(resolver): expose Python unmatched-receiver guard regressions Count file-node reads in the counting store and give the caller file a bridge import that binds a different name, so removing the early return for an unbound receiver fails the test. The expected bound is one provider lookup. --- internal/resolver/bridge_import_reachability_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go index 74d1c3592..f25698a3b 100644 --- a/internal/resolver/bridge_import_reachability_test.go +++ b/internal/resolver/bridge_import_reachability_test.go @@ -227,23 +227,25 @@ func TestBridgeCandidatesLookupOncePerProvider(t *testing.T) { require.True(t, cr.resolvePythonImportedMethod(e, target)) } require.Zero(t, g.broad, "bridge lookups must not scan all repositories") - require.LessOrEqual(t, g.scoped, 2, "one provider lookup plus Python local-type check") + require.Equal(t, 1, g.scoped, "one provider lookup") }) } } func TestPythonUnimportedReceiverAvoidsNameLookups(t *testing.T) { - // Given a Python file with no bridge import binding the receiver. + // Given a bridge import that binds a different name, not the call receiver. g := &bridgeNameCountStore{Store: graph.New()} file := "consumer/tests.py" g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "python"}) cr := NewCrossRepo(g) + cr.bridgeImportsByFile = map[string][]*graph.Edge{file: {{From: file, FilePath: file, To: "provider/pkg/image.py", Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image", "python_bindings": map[string]any{"Different": "Image"}}}}} edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{"receiver_type": "TestCase"}} // When a typed method is considered by the tracked bridge. require.False(t, cr.resolvePythonImportedMethod(edge, "*.assertEqual")) - // Then it performs no store name lookup. + // Then it performs no file-node or store name lookup. require.Zero(t, g.broad) require.Zero(t, g.scoped) + require.Zero(t, g.fileReads) } func TestPythonReceiverTypesReadOncePerFilePerPass(t *testing.T) { From 23e38a9bb189a587af3f876cffe2431f048ef918 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 07:21:59 -0700 Subject: [PATCH 069/111] docs: state Cargo direct-group and exact-parent call limits --- docs/multi-repo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/multi-repo.md b/docs/multi-repo.md index a1337b29b..a6179f78c 100644 --- a/docs/multi-repo.md +++ b/docs/multi-repo.md @@ -526,7 +526,7 @@ Scoped tool responses carry a `scope_applied` meta field plus a one-line widen h - A cross-repo module binding carries confidence `0.6` and origin `ast_inferred`. It is a source hint, not proof that the tracked checkout matches the required version. The edge retains its symbol-bearing import identity so provider edits and full reindexes can rebind it without editing the caller. Module-map changes re-check existing bindings. - **Limits:** Only each tracked root's `go.mod` identity is used. Separately tracked nested modules can win by longest path; nested manifests within one tracked root and `go.work` selection are not resolved. Required versions and module/version replacements are not verified. A local `replace` whose destination is untracked prevents module-path binding instead of selecting another checkout with the same module name. Tracked local replacements remain source hints, not full Go build-list resolution. Calls on values typed from another module, such as `client.Do()` for a `library.Client`, stay unresolved unless another pass has precise evidence. Importing the module alone does not enable name-only receiver binding. - Rust `use` imports link to the declared provider's library root or module file. Dependency, dev-dependency, build-dependency and target tables are read. Calls bind only to a unique imported item or receiver in that file. - - **Cargo limits:** This is not Cargo build resolution. Versions, features and target conditions are not evaluated. Absolute or escaping library paths, conflicting aliases and multiple eligible providers are refused. Re-exports and untyped receivers are not inferred. Module-qualified calls such as `use lib::util; util::action()` are not linked by this bridge. Grouped uses bind calls only when their imported items share the linked module file. Existing links are not always revisited after declaration or provider-set changes; reindex the provider to refresh them. + - **Cargo limits:** This is not Cargo build resolution. Versions, features and target conditions are not evaluated. Absolute or escaping library paths, conflicting aliases and multiple eligible providers are refused. Re-exports and untyped receivers are not inferred. Module-qualified calls such as `use lib::util; util::action()` are not linked by this bridge. Only direct members of a single group bind, such as `use lib::util::{a, b}`. Nested members inside braces are not linked. Grouped and ungrouped member calls require an indexed file for the exact parent module; an import that falls back to an ancestor file does not authorize those calls. Existing links are not always revisited after declaration or provider-set changes; reindex the provider to refresh them. - Python distribution imports link only after a consumer declaration and a unique eligible provider match. Qualified calls use the linked module file. Typed methods require a receiver name bound by `from X import Name [as Alias]` and a matching provider method in that file. Plain `import pkg.mod` and wildcard imports never link typed methods. A same-named type defined in the caller file refuses the link. Package calls can follow the one-level re-exports described above. - **Python limits:** Distribution versions and arbitrary source layouts are not verified. Providers must expose indexed root or `src/` modules or regular packages. Namespace-only providers and dynamic imports are not linked. Local top-level names take priority, including namespace directories. A working link is not reconsidered when a second provider appears, its dependency declaration is dropped, or a new local top-level package shadows it. Reindex the consumer to refresh it. - **Impact analysis** — `explain_change_impact`, `verify_change`, and `get_test_targets` follow cross-repo edges automatically, grouping results by repository. From 5ff9b1794bd4441e974acd4c7d0be4e35f58aa6c Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 07:47:57 -0700 Subject: [PATCH 070/111] perf(resolver): batch Python caller-file type snapshots The per-pass type snapshot read each Python consumer file with a linked from-import through its own GetFileNodes call, including on warmup. Collect the qualifying paths and fill the snapshot with one GetFileNodesByPaths call. The test now uses two bridge imports in one caller file, so the build-once check fails it when removed. --- .../bridge_import_reachability_test.go | 56 +++++++++++++++++-- internal/resolver/cross_repo.go | 19 ++++--- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go index f25698a3b..d5a594d2b 100644 --- a/internal/resolver/bridge_import_reachability_test.go +++ b/internal/resolver/bridge_import_reachability_test.go @@ -170,8 +170,10 @@ func TestPythonMethodsStayWithinImportedType(t *testing.T) { type bridgeNameCountStore struct { graph.Store - broad, scoped int - fileReads int + broad, scoped int + fileReads int + fileBatchReads int + fileBatchPaths []string } func (s *bridgeNameCountStore) GetFileNodes(file string) []*graph.Node { @@ -179,6 +181,12 @@ func (s *bridgeNameCountStore) GetFileNodes(file string) []*graph.Node { return s.Store.GetFileNodes(file) } +func (s *bridgeNameCountStore) GetFileNodesByPaths(paths []string) map[string][]*graph.Node { + s.fileBatchReads++ + s.fileBatchPaths = append(s.fileBatchPaths, paths...) + return s.Store.GetFileNodesByPaths(paths) +} + func (s *bridgeNameCountStore) FindNodesByName(name string) []*graph.Node { s.broad++ return s.Store.FindNodesByName(name) @@ -260,6 +268,7 @@ func TestPythonReceiverTypesReadOncePerFilePerPass(t *testing.T) { } g.AddNode(&graph.Node{ID: provider + "::Image.show", Name: "show", FilePath: provider, Kind: graph.KindMethod, RepoPrefix: "provider", Language: "python", Meta: map[string]any{"receiver": "Image"}}) g.AddEdge(&graph.Edge{From: file, FilePath: file, To: provider, Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image", "python_bindings": map[string]any{"Image": "Image"}}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: provider, Kind: graph.EdgeImports, Line: 20, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image", "python_bindings": map[string]any{"Different": "Other"}}}) for i := 0; i < 8; i++ { g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::*.show", Kind: graph.EdgeCalls, Line: i + 2, Meta: map[string]any{"receiver_type": "Image"}}) } @@ -275,7 +284,9 @@ func TestPythonReceiverTypesReadOncePerFilePerPass(t *testing.T) { // When a full or scoped pass resolves the calls. resolve() // Then local type names are read once, not once per edge. - require.Equal(t, 1, g.fileReads) + require.Zero(t, g.fileReads) + require.Equal(t, 1, g.fileBatchReads) + require.Len(t, g.fileBatchPaths, 1) calls := outEdgesOfKind(g, file, graph.EdgeCalls) require.Len(t, calls, 8) for _, call := range calls { @@ -292,7 +303,9 @@ func TestPythonReceiverTypesReadOncePerFilePerPass(t *testing.T) { } g.ReindexEdges(batch) resolve() - require.Equal(t, 2, g.fileReads) + require.Zero(t, g.fileReads) + require.Equal(t, 2, g.fileBatchReads) + require.Len(t, g.fileBatchPaths, 2) for _, call := range outEdgesOfKind(g, file, graph.EdgeCalls) { require.Equal(t, "unresolved::*.show", call.To) } @@ -300,3 +313,38 @@ func TestPythonReceiverTypesReadOncePerFilePerPass(t *testing.T) { } } } + +func TestPythonReceiverTypeSnapshotBatchesFiles(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, count := range []int{0, 6} { + t.Run(fmt.Sprintf("%s/files=%d", backend, count), func(t *testing.T) { + // Given several from-import callers and an unrelated wildcard importer. + g := &bridgeNameCountStore{Store: goImportReviewStore(t, backend)} + provider := "provider/pkg/image.py" + g.AddNode(&graph.Node{ID: provider, FilePath: provider, Kind: graph.KindFile, RepoPrefix: "provider", Language: "python"}) + g.AddEdge(&graph.Edge{From: "consumer/wild.py", FilePath: "consumer/wild.py", To: provider, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_bindings": map[string]any{}}}) + for i := 0; i < count; i++ { + file := fmt.Sprintf("consumer/caller%d.py", i) + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "python"}) + g.AddNode(&graph.Node{ID: file + "::Local", FilePath: file, Kind: graph.KindType, Name: "Local", RepoPrefix: "consumer", Language: "python"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: provider, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_bindings": map[string]any{"Image": "Image"}}}) + } + // When a full pass builds the import evidence index, with no typed-call filter. + cr := NewCrossRepo(g) + cr.buildReachableReposIndex(nil) + // Then qualifying files are fetched in one batch and no point reads. + require.Zero(t, g.fileReads) + expectedBatches := 0 + if count > 0 { + expectedBatches = 1 + } + require.Equal(t, expectedBatches, g.fileBatchReads) + require.Len(t, g.fileBatchPaths, count) + require.Len(t, cr.pythonLocalTypesByFile, count) + for _, types := range cr.pythonLocalTypesByFile { + require.True(t, types["Local"]) + } + }) + } + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 1365c4121..4c1875ff6 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -870,6 +870,7 @@ func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]boo cr.jsTSBindingsByFile = make(map[string]map[string]string) cr.bridgeImportsByFile = make(map[string][]*graph.Edge) cr.pythonLocalTypesByFile = make(map[string]map[string]bool) + var pythonFiles []string // Materialise metadata-free import projections and batch-load only target // placement fields. A per-edge point lookup here is a query round-trip per // import on a disk backend, which under the cross-repo pass's import @@ -957,13 +958,8 @@ func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]boo cr.bridgeImportsByFile[e.From] = append(cr.bridgeImportsByFile[e.From], e) } if bindings, _ := e.Meta["python_bindings"].(map[string]any); isPythonSourcePath(e.FilePath) && e.Meta[PythonImportStubMeta] != nil && len(bindings) > 0 && cr.pythonLocalTypesByFile[e.From] == nil { - types := make(map[string]bool) - for _, node := range cr.graph.GetFileNodes(e.FilePath) { - if node.Kind == graph.KindType { - types[node.Name] = true - } - } - cr.pythonLocalTypesByFile[e.From] = types + cr.pythonLocalTypesByFile[e.From] = make(map[string]bool) + pythonFiles = append(pythonFiles, e.FilePath) } // A linked module file is not evidence for every symbol in its repo. if npmLinked || e.Meta["go_import"] != nil || e.Meta[CargoImportStubMeta] != nil || e.Meta[PythonImportStubMeta] != nil { @@ -977,6 +973,15 @@ func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]boo set[to.RepoPrefix] = struct{}{} } } + if len(pythonFiles) > 0 { + for file, nodes := range cr.graph.GetFileNodesByPaths(pythonFiles) { + for _, node := range nodes { + if node.Kind == graph.KindType { + cr.pythonLocalTypesByFile[file][node.Name] = true + } + } + } + } cr.reachableReposByFile = idx cr.importedDirsByFile = dirs } From 9dde603810985f345e2e51c66517dbd047af07fe Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 07:48:05 -0700 Subject: [PATCH 071/111] fix(indexer): clear Cargo module evidence when restubbing imports restubTrackedCargoImportsFromView (untrack and provider-file eviction) reset the import to its stub but kept cargo_import_module, so the stub carried a module path a cold index would not have. Delete it there, as the resolver restub already does, and assert the field is gone on both paths. --- internal/indexer/cargo_import_restub.go | 3 ++ internal/indexer/cargo_import_restub_test.go | 31 +++++++++++++++++++ .../resolver/cargo_import_retarget_test.go | 3 +- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go index eb5b15944..aa0ee2401 100644 --- a/internal/indexer/cargo_import_restub.go +++ b/internal/indexer/cargo_import_restub.go @@ -75,6 +75,9 @@ func restubTrackedCargoImportsFromView(g graph.Store, evicted map[string]struct{ graph.StashRestubProvenance(edge) edge.ConfidenceLabel = "" } + if edge.Kind == graph.EdgeImports { + delete(edge.Meta, resolver.CargoImportModuleMeta) + } edge.To, edge.CrossRepo = stub, false batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: old}) } diff --git a/internal/indexer/cargo_import_restub_test.go b/internal/indexer/cargo_import_restub_test.go index 1e5332da1..5403e4fc9 100644 --- a/internal/indexer/cargo_import_restub_test.go +++ b/internal/indexer/cargo_import_restub_test.go @@ -66,3 +66,34 @@ func TestCargoConsumerFrontierIncludesUnresolvedImportStubs(t *testing.T) { mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} require.Equal(t, []string{"consumer/a.rs", "consumer/b.rs"}, mi.cargoConsumerFilesForRepo("provider", false)) } + +func TestCargoRestubClearsMatchedModuleEvidence(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"untrack", "file_eviction"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + // Given a tracked import with its previously matched module path. + g := cargoWarmStore(t, backend) + file, target, stub := "consumer/src/lib.rs", "provider/src/util.rs", "external::other_crate/util" + for _, path := range []string{file, target} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{resolver.CargoImportStubMeta: stub, resolver.CargoImportModuleMeta: "other_crate::util"}}) + // When untracking or file eviction restubs the import. + var consumers []string + if mode == "untrack" { + consumers = restubTrackedCargoRepoIncoming(g, "provider") + } else { + consumers = restubTrackedCargoImportsFromView(g, map[string]struct{}{target: {}}, g.GetInEdgesByNodeIDs([]string{target}), false) + } + // Then its persisted state has the stub, not matched-module evidence. + require.Equal(t, []string{file}, consumers) + imports := g.GetOutEdges(file) + require.Len(t, imports, 1) + require.Equal(t, stub, imports[0].To) + require.False(t, imports[0].CrossRepo) + require.NotContains(t, imports[0].Meta, resolver.CargoImportModuleMeta) + require.Equal(t, stub, imports[0].Meta[resolver.CargoImportStubMeta]) + }) + } + } +} diff --git a/internal/resolver/cargo_import_retarget_test.go b/internal/resolver/cargo_import_retarget_test.go index 71348f549..b7e68bb3a 100644 --- a/internal/resolver/cargo_import_retarget_test.go +++ b/internal/resolver/cargo_import_retarget_test.go @@ -121,7 +121,7 @@ func TestCargoRevisitedImportRestoresStubOnRefusal(t *testing.T) { for _, file := range []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs", "fork/src/lib.rs"} { g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "rust"}) } - g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "provider/src/util.rs", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: stub}}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "provider/src/util.rs", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: stub, CargoImportModuleMeta: "other_crate::util"}}) g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "provider/src/util.rs::action", Kind: graph.EdgeCalls, Line: 2, CrossRepo: true, Meta: map[string]any{CargoCallStubMeta: "unresolved::action"}}) cr := NewCrossRepo(g) cr.SetCargoImportRetargetRepo("provider") @@ -137,6 +137,7 @@ func TestCargoRevisitedImportRestoresStubOnRefusal(t *testing.T) { require.Len(t, imports, 1) require.Equal(t, stub, imports[0].To) require.False(t, imports[0].CrossRepo) + require.NotContains(t, imports[0].Meta, CargoImportModuleMeta) calls := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeCalls) require.Len(t, calls, 1) require.Equal(t, "unresolved::action", calls[0].To, "warm call must equal a cold unbound call") From e84949b8e571c0b524af667a7c46ee36aa099041 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 12:42:45 -0700 Subject: [PATCH 072/111] fix(indexer): invalidate removed Go package projections before settlement Provider removal kept cached package-file identities. Deferred imports could reuse a deleted file or a refused tie instead of matching a cold index. Drop only the removed provider's projection under the store resolve mutex before settling parked identities. Invalidate even when a tie produced no parked rows. Preserve the shared retry object and other providers' projections. --- internal/indexer/go_module_revision_test.go | 53 +++++++++++++++++++-- internal/indexer/repository_untrack.go | 9 ++-- internal/resolver/go_module_imports.go | 13 +++++ internal/resolver/go_module_imports_test.go | 17 +++++++ 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/internal/indexer/go_module_revision_test.go b/internal/indexer/go_module_revision_test.go index 2bbf7a18b..91543183c 100644 --- a/internal/indexer/go_module_revision_test.go +++ b/internal/indexer/go_module_revision_test.go @@ -2,6 +2,7 @@ package indexer import ( "context" + "maps" "os" "path/filepath" "strings" @@ -138,6 +139,14 @@ func TestGoModuleUntrackSettlesColdEquivalent(t *testing.T) { cold := revisionColdGraph(t, backend, coldRepos) require.Equal(t, want, goModuleLifecycleCall(t, cold).To) assert.Equal(t, want, goModuleLifecycleCall(t, g).To) + // Import settlement must match cold immediately, without a global resolve. + wantImports := revisionCallTargets(cold, "a/main.go", graph.EdgeImports) + if mode != "tie" { + require.Equal(t, []string{"external::example.org/library"}, wantImports) + } else { + require.Equal(t, []string{"fork/lib.go"}, wantImports) + } + assert.ElementsMatch(t, wantImports, revisionCallTargets(g, "a/main.go", graph.EdgeImports)) }) } } @@ -250,7 +259,7 @@ func TestGoModuleUntrackRetainsOnlyParkedIdentities(t *testing.T) { }, parked, "both caller-owned references must survive and settle") assert.IsType(t, []graph.EdgeIdentity{}, parked, "teardown must not retain full metadata-bearing edge objects") g.EvictFile("b/lib.go") - require.NoError(t, mi.finalizeUntrackedGoModuleCalls(parked)) + require.NoError(t, mi.finalizeUntrackedGoModuleCalls("b", parked)) cold := revisionColdGraph(t, backend, repos[:1]) assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) }) @@ -328,7 +337,7 @@ func TestGoModuleParkedLookupRejectsMismatchedRows(t *testing.T) { require.NotEmpty(t, g.(graph.EdgeIdentityBatchFinder).FindEdgesByIdentities(parked), "parked row must survive purge") wrapped := &goModuleMismatchedFinderStore{Store: g} mi.graph = wrapped - require.NoError(t, mi.finalizeUntrackedGoModuleCalls(parked)) + require.NoError(t, mi.finalizeUntrackedGoModuleCalls("b", parked)) assert.Zero(t, wrapped.writes, "identity mismatch must not schedule an edge mutation") assert.Zero(t, wrapped.lookups, "identity mismatch must be rejected before symbol lookup") }) @@ -372,7 +381,7 @@ func TestGoModuleParkedLookupWithoutFinder(t *testing.T) { parked := restubGoModuleRepoIncoming(g, "b") g.EvictFile("b/lib.go") mi.graph = &goModuleNoFinderStore{Store: g} - require.NoError(t, mi.finalizeUntrackedGoModuleCalls(parked)) + require.NoError(t, mi.finalizeUntrackedGoModuleCalls("b", parked)) cold := revisionColdGraph(t, backend, repos[:1]) assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) }) @@ -454,3 +463,41 @@ func TestGoModuleSameStampCallerReindexColdEquivalent(t *testing.T) { }) } } + +// Given A imports B and an unrelated C is tracked, when C is untracked, +// then A's resolved import and B's package projection remain unchanged. +func TestGoModuleUntrackUnrelatedRepoKeepsImport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + third := t.TempDir() + writeFile(t, filepath.Join(third, "go.mod"), "module example.org/unrelated\n") + writeFile(t, filepath.Join(third, "lib.go"), "package unrelated\nfunc Other() {}\n") + for _, entry := range append(repos[:2], config.RepoEntry{Path: third, Name: "c", Workspace: "shared"}) { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + imports := g.GetOutEdges("a/main.go") + var before *graph.Edge + for _, edge := range imports { + if edge.Kind == graph.EdgeImports { + copy := *edge + copy.Meta = maps.Clone(edge.Meta) + before = © + } + } + require.NotNil(t, before) + require.Equal(t, "b/lib.go", before.To) + require.True(t, before.CrossRepo) + _, _, err := mi.UntrackRepoChecked(t.Context(), "c") + require.NoError(t, err) + for _, edge := range g.GetOutEdges("a/main.go") { + if edge.Kind == graph.EdgeImports { + require.Equal(t, *before, *edge) + } + } + require.Equal(t, []string{"b/lib.go"}, revisionCallTargets(g, "a/main.go", graph.EdgeImports)) + require.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) + }) + } +} diff --git a/internal/indexer/repository_untrack.go b/internal/indexer/repository_untrack.go index 12835a506..e87feb7a2 100644 --- a/internal/indexer/repository_untrack.go +++ b/internal/indexer/repository_untrack.go @@ -297,7 +297,7 @@ func (mi *MultiIndexer) purgeRepositoryPayload( if err := purger.PurgeRepo(repoPrefix); err != nil { return 0, 0, fmt.Errorf("purge repository payload: %w", err) } - if err := mi.finalizeUntrackedGoModuleCalls(parked); err != nil { + if err := mi.finalizeUntrackedGoModuleCalls(repoPrefix, parked); err != nil { return 0, 0, err } if meta != nil { @@ -307,16 +307,17 @@ func (mi *MultiIndexer) purgeRepositoryPayload( } nodesRemoved, edgesRemoved, err = evictRepoAllGenerations(mi.graph, repoPrefix) if err == nil { - err = mi.finalizeUntrackedGoModuleCalls(parked) + err = mi.finalizeUntrackedGoModuleCalls(repoPrefix, parked) } return nodesRemoved, edgesRemoved, err } -func (mi *MultiIndexer) finalizeUntrackedGoModuleCalls(parked []graph.EdgeIdentity) error { +func (mi *MultiIndexer) finalizeUntrackedGoModuleCalls(removedRepo string, parked []graph.EdgeIdentity) error { + cr := mi.newCrossRepoResolver() + cr.InvalidateGoModulePackageRepo(removedRepo) if len(parked) == 0 { return nil } - cr := mi.newCrossRepoResolver() if err := cr.ResolveDeferredGoModuleEdgeIdentities(parked); err != nil { return err } diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 209d14cf4..904c0b783 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -377,6 +377,19 @@ func (cr *CrossRepoResolver) rememberGoModuleImports(nodes []*graph.Node) { } } +// InvalidateGoModulePackageRepo drops only a removed provider's file projection. +// Package projections share the store resolve mutex, not the retry queue mutex. +func (cr *CrossRepoResolver) InvalidateGoModulePackageRepo(repo string) { + if repo == "" { + return + } + cr.mu.Lock() + defer cr.mu.Unlock() + if cr.goModuleRetry != nil { + delete(cr.goModuleRetry.packages, repo) + } +} + // Only retained statements registered for this provider are probed on edits. // No global node scan is needed to reconstruct imports deleted by eviction. func (cr *CrossRepoResolver) goModuleImportIDs(repo string) []string { diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index dd62ac3e7..bd51efa7f 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -683,3 +683,20 @@ func TestGoModuleImportFallbackStatsAreUnresolved(t *testing.T) { } } } + +func TestInvalidateGoModulePackageRepoKeepsOtherProviders(t *testing.T) { + // Given one shared retry object with independent provider projections. + state := &GoModuleDepRetry{complete: true, packages: map[string]map[string]graph.FileNodeIdentity{ + "b": {"b": {ID: "b/lib.go", FilePath: "b/lib.go", RepoPrefix: "b"}}, + "c": {"c": {ID: "c/lib.go", FilePath: "c/lib.go", RepoPrefix: "c"}}, + }} + cr := NewCrossRepo(graph.New()) + cr.SetGoModuleDepRetry(state, 1) + // When only C's projection is invalidated. + cr.InvalidateGoModulePackageRepo("c") + // Then B's cached identity and the shared retry state are preserved. + require.Same(t, state, cr.goModuleRetry) + require.NotContains(t, state.packages, "c") + require.Equal(t, graph.FileNodeIdentity{ID: "b/lib.go", FilePath: "b/lib.go", RepoPrefix: "b"}, state.packages["b"]["b"]) + require.True(t, state.complete, "package invalidation must not replace the terminal-sweep policy") +} From 5b6ba83afcf09d59b878b145ebf65cac020c1d6d Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 13:16:28 -0700 Subject: [PATCH 073/111] perf(indexer): build the untrack resolver only for parked Go references Every untrack built a full cross-repo resolver, reading module manifests across all roots, only to drop the removed repository's package projection, even when nothing was parked and the repository had no Go code. Drop the projection directly on the shared retry state under the store resolve mutex, and build the resolver only when parked references need settling. The JS-only untrack test now also requires that no module snapshot is read. --- internal/indexer/module_work_count_test.go | 6 ++++++ internal/indexer/repository_untrack.go | 8 ++++++-- internal/resolver/go_module_imports.go | 15 ++++++--------- internal/resolver/go_module_imports_test.go | 7 ++----- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/internal/indexer/module_work_count_test.go b/internal/indexer/module_work_count_test.go index 7a2ef74e6..0176b157a 100644 --- a/internal/indexer/module_work_count_test.go +++ b/internal/indexer/module_work_count_test.go @@ -128,9 +128,15 @@ func TestNonGoUntrackSkipsModuleSweep(t *testing.T) { } require.NoError(t, mi.SeedGoModuleDeps(t.Context())) g.scans = 0 + mi.goModuleMu.Lock() + mi.goModuleChecked = time.Time{} // any module snapshot read would refresh this stamp + mi.goModuleMu.Unlock() _, _, err = mi.UntrackRepoChecked(t.Context(), "a") require.NoError(t, err) assert.Zero(t, g.scans, "JS-only untrack must not rescan Go terminals") + mi.goModuleMu.Lock() + defer mi.goModuleMu.Unlock() + assert.True(t, mi.goModuleChecked.IsZero(), "untrack with nothing parked must not build a cross-repo resolver") } type goModuleUntrackScanStore struct{ *goModuleRetryScanStore } diff --git a/internal/indexer/repository_untrack.go b/internal/indexer/repository_untrack.go index e87feb7a2..a7db767c7 100644 --- a/internal/indexer/repository_untrack.go +++ b/internal/indexer/repository_untrack.go @@ -313,11 +313,15 @@ func (mi *MultiIndexer) purgeRepositoryPayload( } func (mi *MultiIndexer) finalizeUntrackedGoModuleCalls(removedRepo string, parked []graph.EdgeIdentity) error { - cr := mi.newCrossRepoResolver() - cr.InvalidateGoModulePackageRepo(removedRepo) + // A refused tie parks nothing, so drop the projection before the early return. + mu := mi.graph.ResolveMutex() + mu.Lock() + mi.goModuleRetry.InvalidatePackageRepo(removedRepo) + mu.Unlock() if len(parked) == 0 { return nil } + cr := mi.newCrossRepoResolver() if err := cr.ResolveDeferredGoModuleEdgeIdentities(parked); err != nil { return err } diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 904c0b783..5b76010cf 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -377,17 +377,14 @@ func (cr *CrossRepoResolver) rememberGoModuleImports(nodes []*graph.Node) { } } -// InvalidateGoModulePackageRepo drops only a removed provider's file projection. -// Package projections share the store resolve mutex, not the retry queue mutex. -func (cr *CrossRepoResolver) InvalidateGoModulePackageRepo(repo string) { - if repo == "" { +// InvalidatePackageRepo drops only a removed provider's file projection. +// The caller holds the store resolve mutex, which guards package projections; +// the retry queue mutex does not. +func (r *GoModuleDepRetry) InvalidatePackageRepo(repo string) { + if r == nil || repo == "" { return } - cr.mu.Lock() - defer cr.mu.Unlock() - if cr.goModuleRetry != nil { - delete(cr.goModuleRetry.packages, repo) - } + delete(r.packages, repo) } // Only retained statements registered for this provider are probed on edits. diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index bd51efa7f..fcef8f936 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -684,18 +684,15 @@ func TestGoModuleImportFallbackStatsAreUnresolved(t *testing.T) { } } -func TestInvalidateGoModulePackageRepoKeepsOtherProviders(t *testing.T) { +func TestInvalidatePackageRepoKeepsOtherProviders(t *testing.T) { // Given one shared retry object with independent provider projections. state := &GoModuleDepRetry{complete: true, packages: map[string]map[string]graph.FileNodeIdentity{ "b": {"b": {ID: "b/lib.go", FilePath: "b/lib.go", RepoPrefix: "b"}}, "c": {"c": {ID: "c/lib.go", FilePath: "c/lib.go", RepoPrefix: "c"}}, }} - cr := NewCrossRepo(graph.New()) - cr.SetGoModuleDepRetry(state, 1) // When only C's projection is invalidated. - cr.InvalidateGoModulePackageRepo("c") + state.InvalidatePackageRepo("c") // Then B's cached identity and the shared retry state are preserved. - require.Same(t, state, cr.goModuleRetry) require.NotContains(t, state.packages, "c") require.Equal(t, graph.FileNodeIdentity{ID: "b/lib.go", FilePath: "b/lib.go", RepoPrefix: "b"}, state.packages["b"]["b"]) require.True(t, state.complete, "package invalidation must not replace the terminal-sweep policy") From 759b16ab7c3fd326992c58caf1ae61d3c32491d7 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 16:24:55 -0700 Subject: [PATCH 074/111] test(indexer): list why DeltaWriter lacks the Python import projection scanner trackedPythonImportEdges uses ScanTestEdgeProjections only to page the import scan. Without it, it reads the composed EdgesLightSeq import scan with the same filter, so the delta view keeps its masks and gives the same eligible imports. --- internal/indexer/edit_delta_capability_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/indexer/edit_delta_capability_test.go b/internal/indexer/edit_delta_capability_test.go index 390e19b6d..8b271b886 100644 --- a/internal/indexer/edit_delta_capability_test.go +++ b/internal/indexer/edit_delta_capability_test.go @@ -38,6 +38,7 @@ var editDeltaAbsentCapabilities = map[string]string{ "anonymous{ReindexEdgesApplied}": "batch-drop receipt for persistent stores; DeltaWriter applies its reindexes locally and unconditionally through ReindexEdges", "anonymous{AddBatchChecked}": "checked durable publication: DeltaWriter stages non-failing writes in its local work graph through AddBatch; its owner checks final layer publication, and forwarding this capability would bypass the delta", "anonymous{WriteMark}": "process/store write telemetry, not a graph capability: the per-save indexer and resolver receive the underlying store handle's sampler explicitly; a MultiIndexer without this optional sampler falls back to process-wide counters", + "anonymous{ScanTestEdgeProjections}": "paging accelerator: trackedPythonImportEdges falls back to the composed EdgesLightSeq/EdgesByKind import scan with the same eligibility filter, preserving delta masks", "graph.ContractExtractionEdgeReader": "physical checked extraction adjacency; outgoingMetadataContext dispatches the composed DeltaWriter through its selected view rather than forwarding base-only rows", "graph.OverlayLayerContractExtractionEdgeReader": "physical ancestry-layer projection, not the composed delta; outgoingMetadataContext applies each selected layer and its masks explicitly", From d47c49b8dab6219e2b96694dd5c0c9cb36ffc4c2 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 16:25:14 -0700 Subject: [PATCH 075/111] fix(resolver): restate Python import metadata after file saves Only the whole module attribution pass wrote python_import, so a save that changed a Python import's dotted spelling left the edge without it, and the file no longer matched a cold index. Single-file and batched file attribution now run the same attribution core over the saved file's own edges. Non-Python saves return at once, and no graph-wide scan is added. The metadata checklist names python_import's restaters, and a test edits an import on the in-memory and SQLite stores and compares the edge with a cold index. --- .../indexer/edit_delta_meta_key_class_test.go | 64 ++++++++++++++++++- .../resolver/incremental_attribution_cache.go | 4 ++ internal/resolver/module_attribution.go | 34 ++++++++-- internal/resolver/resolver.go | 1 + 4 files changed, 97 insertions(+), 6 deletions(-) diff --git a/internal/indexer/edit_delta_meta_key_class_test.go b/internal/indexer/edit_delta_meta_key_class_test.go index da703ae7b..27b555575 100644 --- a/internal/indexer/edit_delta_meta_key_class_test.go +++ b/internal/indexer/edit_delta_meta_key_class_test.go @@ -6,9 +6,13 @@ import ( "sort" "testing" + "github.com/stretchr/testify/require" "go.uber.org/zap" "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" ) // metaKeysRestatedOutsideTheFingerprint names every node or edge metadata key @@ -43,8 +47,9 @@ var metaKeysRestatedOutsideTheFingerprint = map[string]string{ "role": "contracts pass (contract nodes)", "symbol_id": "contracts pass (contract nodes)", // External module stubs: the module scanner / external-call attribution. - "ecosystem": "modules scanner / resolver external_call_attribution (module nodes)", - "import_path": "modules scanner / resolver external_call_attribution (module nodes)", + "ecosystem": "modules scanner / resolver external_call_attribution (module nodes)", + "import_path": "modules scanner / resolver external_call_attribution (module nodes)", + "python_import": "resolver attributeNonGoModuleImports / resolveTrackedPythonImports (exact dotted import spelling)", // Test classification after extraction (test_edges.go). "is_test": "test pass (test_edges.go)", "is_test_file": "test pass (test_edges.go)", @@ -141,3 +146,58 @@ func TestEveryMetadataKeyIsFingerprintedOrRestated(t *testing.T) { }) } } + +func TestPythonImportSpellingEditColdEquivalent(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + openStore := func() graph.Store { + if backend == "memory" { + return graph.New() + } + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db + } + newIndexer := func(g graph.Store) *Indexer { + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + idx := New(g, reg, config.Default().Index, zap.NewNop()) + idx.SetRepoPrefix("consumer") + t.Cleanup(idx.Close) + return idx + } + imports := func(g graph.Store) []*graph.Edge { + var out []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports { + out = append(out, edge) + } + } + require.Len(t, out, 1) + return out + } + // Given an attributed import with a stable local alias and module target. + root := t.TempDir() + file := filepath.Join(root, "app.py") + writeFile(t, file, "import pkg.mod as current\ndef run(): return current.open()\n") + warm := openStore() + idx := newIndexer(warm) + _, err := idx.Index(root) + require.NoError(t, err) + before := imports(warm)[0] + require.Equal(t, "pkg.mod", before.Meta["python_import"]) + require.Equal(t, "module::pypi:pkg", before.To) + // When only the dotted module spelling changes, keeping the alias and top. + writeFile(t, file, "import pkg.other as current\ndef run(): return current.open()\n") + require.NoError(t, idx.IndexFile(file)) + cold := openStore() + _, err = newIndexer(cold).Index(root) + require.NoError(t, err) + // Then the incremental edge, including python_import, equals a cold index. + require.Equal(t, "pkg.other", imports(warm)[0].Meta["python_import"]) + require.Equal(t, "module::pypi:pkg", imports(warm)[0].To) + require.Equal(t, imports(cold), imports(warm)) + }) + } +} diff --git a/internal/resolver/incremental_attribution_cache.go b/internal/resolver/incremental_attribution_cache.go index 7ea55d990..d52bda2f1 100644 --- a/internal/resolver/incremental_attribution_cache.go +++ b/internal/resolver/incremental_attribution_cache.go @@ -107,6 +107,10 @@ func (r *Resolver) runFileAttributionPassesForFilesLocked(frontier incrementalFi // before the builtin and external materialisation passes inspect them. r.flushIncrementalAttributionReindexes() legs.lap("flush") + for _, path := range frontier.paths { + r.attributePythonModuleImportsForFile(path) + } + legs.lap("python_modules") if !r.graphHasLanguage("go") { return diff --git a/internal/resolver/module_attribution.go b/internal/resolver/module_attribution.go index d4123c7c6..bd69fdb25 100644 --- a/internal/resolver/module_attribution.go +++ b/internal/resolver/module_attribution.go @@ -1,6 +1,8 @@ package resolver import ( + "iter" + "slices" "strings" "github.com/zzet/gortex/internal/graph" @@ -40,7 +42,28 @@ func (r *Resolver) attributeNonGoModuleImports() { if !r.graphHasLanguage("python") && !r.graphHasLanguage("dart") { return } - fileLang := r.collectFileLanguages() + r.attributeNonGoModuleImportEdges(r.graph.EdgesByKind(graph.EdgeImports), graph.EdgesLightSeq(r.graph, graph.EdgeDependsOnModule), r.collectFileLanguages()) +} + +// Python saves reuse their preloaded file frontier, including dependency edges. +func (r *Resolver) attributePythonModuleImportsForFile(file string) { + if !isPythonSourcePath(file) { + return + } + languages := make(map[string]string) + for _, node := range r.incrementalFileNodes(file) { + if node.Kind == graph.KindFile && node.Language == "python" { + languages[node.ID] = node.Language + } + } + if len(languages) == 0 { + return + } + edges := r.fileOutEdges(file) + r.attributeNonGoModuleImportEdges(slices.Values(edges), slices.Values(edges), languages) +} + +func (r *Resolver) attributeNonGoModuleImportEdges(imports, dependencies iter.Seq[*graph.Edge], fileLang map[string]string) { type pendingEdge struct { edge *graph.Edge oldTo string @@ -50,8 +73,8 @@ func (r *Resolver) attributeNonGoModuleImports() { moduleSeeds := map[string]moduleSeed{} dependsSeen := map[string]map[string]struct{}{} // fileID → set of moduleIDs - for e := range r.graph.EdgesByKind(graph.EdgeImports) { - if !strings.HasPrefix(e.To, "external::") { + for e := range imports { + if e.Kind != graph.EdgeImports || !strings.HasPrefix(e.To, "external::") { continue } // Scoped warm pass: an unchanged repo's external imports were already @@ -120,7 +143,10 @@ func (r *Resolver) attributeNonGoModuleImports() { // read on every backend, plus a Go-side map build that turns // the per-rewrite check into a constant-time lookup. existingDepends := make(map[string]map[string]struct{}) - for e := range graph.EdgesLightSeq(r.graph, graph.EdgeDependsOnModule) { + for e := range dependencies { + if e.Kind != graph.EdgeDependsOnModule { + continue + } // Only the rewrites' own callers (in scope) are queried below, so a // scoped pass needs the dedupe index for just those repos. if !r.edgeFromInScope(e.From) { diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 719735700..71194af11 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -3585,6 +3585,7 @@ func (r *Resolver) runFileAttributionPassesLocked() { // matches runFileAttributionPassesLocked: bare-name binding runs before // builtin attribution so a local named `len` shadows the builtin. func (r *Resolver) runFileAttributionPassesForFileLocked(filePath string) { + r.attributePythonModuleImportsForFile(filePath) r.rebindGoMethodReceiversForFile(filePath) r.bindBareNameScopeRefsForFile(filePath) r.bindDataflowCalleeRefsForFile(filePath) From 4f02233be557b2b040638578e8650e1cf8b031ef Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 17:03:57 -0700 Subject: [PATCH 076/111] fix(resolver): batch Python module attribution at save boundaries The batched save path ran Python module attribution once per file, so a batch of N Python files cost up to 3N small store writes against the flush-by-pass contract. The shared per-file function also ran it inside scoped ResolveAll, which already has its own module pass later. attributePythonModuleImportsForFiles now collects the frontier's Python files and cached out-edges and makes one attribution call per pass. It runs from the two single-file save boundaries and once in the batched pass, not from the shared per-file function. Work-count tests cover 3- and 7-file batches and a scoped ResolveAll. --- .../resolver/incremental_attribution_cache.go | 4 +- internal/resolver/module_attribution.go | 22 +++--- .../python_module_attribution_batch_test.go | 73 +++++++++++++++++++ internal/resolver/resolver.go | 3 +- 4 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 internal/resolver/python_module_attribution_batch_test.go diff --git a/internal/resolver/incremental_attribution_cache.go b/internal/resolver/incremental_attribution_cache.go index d52bda2f1..aa83cc4fe 100644 --- a/internal/resolver/incremental_attribution_cache.go +++ b/internal/resolver/incremental_attribution_cache.go @@ -107,9 +107,7 @@ func (r *Resolver) runFileAttributionPassesForFilesLocked(frontier incrementalFi // before the builtin and external materialisation passes inspect them. r.flushIncrementalAttributionReindexes() legs.lap("flush") - for _, path := range frontier.paths { - r.attributePythonModuleImportsForFile(path) - } + r.attributePythonModuleImportsForFiles(frontier.paths) legs.lap("python_modules") if !r.graphHasLanguage("go") { diff --git a/internal/resolver/module_attribution.go b/internal/resolver/module_attribution.go index bd69fdb25..820a8564d 100644 --- a/internal/resolver/module_attribution.go +++ b/internal/resolver/module_attribution.go @@ -45,21 +45,25 @@ func (r *Resolver) attributeNonGoModuleImports() { r.attributeNonGoModuleImportEdges(r.graph.EdgesByKind(graph.EdgeImports), graph.EdgesLightSeq(r.graph, graph.EdgeDependsOnModule), r.collectFileLanguages()) } -// Python saves reuse their preloaded file frontier, including dependency edges. -func (r *Resolver) attributePythonModuleImportsForFile(file string) { - if !isPythonSourcePath(file) { - return - } +// Python saves collect their cached frontier before flushing attribution writes. +func (r *Resolver) attributePythonModuleImportsForFiles(files []string) { languages := make(map[string]string) - for _, node := range r.incrementalFileNodes(file) { - if node.Kind == graph.KindFile && node.Language == "python" { - languages[node.ID] = node.Language + var edges []*graph.Edge + for _, file := range files { + if !isPythonSourcePath(file) { + continue + } + for _, node := range r.incrementalFileNodes(file) { + if node.Kind == graph.KindFile && node.Language == "python" { + languages[node.ID] = node.Language + edges = append(edges, r.fileOutEdges(file)...) + break + } } } if len(languages) == 0 { return } - edges := r.fileOutEdges(file) r.attributeNonGoModuleImportEdges(slices.Values(edges), slices.Values(edges), languages) } diff --git a/internal/resolver/python_module_attribution_batch_test.go b/internal/resolver/python_module_attribution_batch_test.go new file mode 100644 index 000000000..6ba4df813 --- /dev/null +++ b/internal/resolver/python_module_attribution_batch_test.go @@ -0,0 +1,73 @@ +package resolver + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" +) + +func TestPythonModuleAttributionBatchWritesArePerPass(t *testing.T) { + for _, count := range []int{3, 7} { + t.Run(strconv.Itoa(count), func(t *testing.T) { + base := graph.New() + frontier := incrementalFileFrontier{nodesByFile: map[string][]*graph.Node{}, outByNode: map[string][]*graph.Edge{}} + var imports []*graph.Edge + for i := 0; i < count; i++ { + path := "app/f" + strconv.Itoa(i) + ".py" + seedFile(base, path, "python") + edge := seedExternalImport(base, path, "package_"+strconv.Itoa(i)+".mod") + imports = append(imports, edge) + frontier.paths = append(frontier.paths, path) + frontier.nodesByFile[path] = base.GetFileNodes(path) + frontier.outByNode[path] = []*graph.Edge{edge} + } + seedFile(base, "outside/main.py", "python") + outside := seedExternalImport(base, "outside/main.py", "outside_package") + store := &moduleAttributionBatchRecordingStore{Store: base} + r := New(store) + r.prepareIncrementalAttributionCache(frontier) + defer r.clearIncrementalAttributionCache() + // Given cached Python imports, when the save batch attributes them, + // then writes cover the entire pass rather than each file. + r.runFileAttributionPassesForFilesLocked(frontier) + assert.Equal(t, []moduleAttributionBatchCall{{nodes: count}, {edges: count}}, store.batchCalls) + assert.Equal(t, []int{count}, store.reindexSizes) + for i, edge := range imports { + spec := "package_" + strconv.Itoa(i) + assert.Equal(t, "module::pypi:"+spec, edge.To) + assert.Equal(t, spec+".mod", edge.Meta["python_import"]) + require.NotNil(t, base.GetNode(edge.To)) + assert.Len(t, outEdgesOfKind(base, edge.From, graph.EdgeDependsOnModule), 1) + } + assert.Equal(t, "external::outside_package", outside.To, "a save must not sweep files outside its frontier") + }) + } +} + +func TestPythonModuleAttributionScopedResolveAllWritesOnce(t *testing.T) { + base := graph.New() + const count = 3 + var imports []*graph.Edge + for i := 0; i < count; i++ { + path := "consumer/f" + strconv.Itoa(i) + ".py" + base.AddNode(&graph.Node{ID: path, Kind: graph.KindFile, Name: path, FilePath: path, Language: "python", RepoPrefix: "consumer"}) + imports = append(imports, seedExternalImport(base, path, "package_"+strconv.Itoa(i)+".mod")) + } + // A pending reference makes ResolveAll reach its post-resolution tail. + base.AddEdge(&graph.Edge{From: "consumer/f0.py", To: "unresolved::Missing", Kind: graph.EdgeCalls, FilePath: "consumer/f0.py", Line: 2}) + store := &moduleAttributionBatchRecordingStore{Store: base} + r := New(store) + r.SetScope(map[string]struct{}{"consumer": {}}) + // Given a small scoped warm restart, when ResolveAll runs its file tail, + // then module attribution remains one later whole-scope pass. + r.ResolveAll() + assert.Equal(t, []moduleAttributionBatchCall{{nodes: count}, {edges: count}}, store.batchCalls) + assert.Equal(t, []int{count}, store.reindexSizes) + for i, edge := range imports { + assert.Equal(t, "module::pypi:package_"+strconv.Itoa(i), edge.To) + assert.Equal(t, "package_"+strconv.Itoa(i)+".mod", edge.Meta["python_import"]) + } +} diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 71194af11..01dd058d4 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -2749,6 +2749,7 @@ func (r *Resolver) ResolveFileAndIncoming(filePath string) *ResolveStats { r.resolveFileEdgesLocked(filePath, stats) forwardDuration := time.Since(forwardStarted) attributionStarted := time.Now() + r.attributePythonModuleImportsForFiles([]string{filePath}) r.runFileAttributionPassesForFileLocked(filePath) attributionDuration := time.Since(attributionStarted) incomingStarted := time.Now() @@ -3329,6 +3330,7 @@ func (r *Resolver) startIncrementalPhase(logger *zap.Logger, phase string) func( // has built the per-pass indexes. func (r *Resolver) resolveFileLocked(filePath string, stats *ResolveStats) { r.resolveFileEdgesLocked(filePath, stats) + r.attributePythonModuleImportsForFiles([]string{filePath}) r.runFileAttributionPassesForFileLocked(filePath) } @@ -3585,7 +3587,6 @@ func (r *Resolver) runFileAttributionPassesLocked() { // matches runFileAttributionPassesLocked: bare-name binding runs before // builtin attribution so a local named `len` shadows the builtin. func (r *Resolver) runFileAttributionPassesForFileLocked(filePath string) { - r.attributePythonModuleImportsForFile(filePath) r.rebindGoMethodReceiversForFile(filePath) r.bindBareNameScopeRefsForFile(filePath) r.bindDataflowCalleeRefsForFile(filePath) From 8a449f3f8de2e82bf6dfd41fb0d0d9d26d928edd Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Sun, 25 Oct 2026 17:04:11 -0700 Subject: [PATCH 077/111] test(indexer): name Python save-time metadata restaters The python_import checklist entry now names the save-time restater with the whole-resolve and tracked-binding ones. It sits in its own group, so the existing module-stub lines keep their original alignment. --- internal/indexer/edit_delta_meta_key_class_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/indexer/edit_delta_meta_key_class_test.go b/internal/indexer/edit_delta_meta_key_class_test.go index 27b555575..15c55c454 100644 --- a/internal/indexer/edit_delta_meta_key_class_test.go +++ b/internal/indexer/edit_delta_meta_key_class_test.go @@ -47,9 +47,11 @@ var metaKeysRestatedOutsideTheFingerprint = map[string]string{ "role": "contracts pass (contract nodes)", "symbol_id": "contracts pass (contract nodes)", // External module stubs: the module scanner / external-call attribution. - "ecosystem": "modules scanner / resolver external_call_attribution (module nodes)", - "import_path": "modules scanner / resolver external_call_attribution (module nodes)", - "python_import": "resolver attributeNonGoModuleImports / resolveTrackedPythonImports (exact dotted import spelling)", + "ecosystem": "modules scanner / resolver external_call_attribution (module nodes)", + "import_path": "modules scanner / resolver external_call_attribution (module nodes)", + + // Python imports are restated by whole resolves, file saves, and tracked binding. + "python_import": "resolver attributeNonGoModuleImports / attributePythonModuleImportsForFiles / resolveTrackedPythonImports (exact dotted import spelling)", // Test classification after extraction (test_edges.go). "is_test": "test pass (test_edges.go)", "is_test_file": "test pass (test_edges.go)", From 71f8b063bd55193930032b1e441ad7e121f0e2e1 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:58:07 -0700 Subject: [PATCH 078/111] fix(resolver): preserve same-line Python import identities Keep colliding PyPI imports on distinct external targets while recording module and exact import evidence. Verify later provider binding, unchanged saves and cold parity on both stores. This cold-output change is covered by this branch's move of resolver semantics to 6. --- .../indexer/python_bridge_identity_test.go | 69 +++++++++++++++++++ internal/resolver/module_attribution.go | 25 +++++-- internal/resolver/module_attribution_test.go | 11 ++- internal/resolver/version.go | 1 + 4 files changed, 96 insertions(+), 10 deletions(-) create mode 100644 internal/indexer/python_bridge_identity_test.go diff --git a/internal/indexer/python_bridge_identity_test.go b/internal/indexer/python_bridge_identity_test.go new file mode 100644 index 000000000..a2a0199ff --- /dev/null +++ b/internal/indexer/python_bridge_identity_test.go @@ -0,0 +1,69 @@ +package indexer + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func newPythonBridgeIdentityIndexer(t *testing.T, backend string, entries []config.RepoEntry) (*MultiIndexer, graph.Store) { + t.Helper() + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + cm := newTestConfigManager(t) + cm.Global().Repos = entries + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + return mi, g +} + +func TestTrackedPythonSameLineImportIdentities(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'dist'\n") + writeWorkspaceFile(t, provider, "pkg/__init__.py", "") + writeWorkspaceFile(t, provider, "pkg/a.py", "def f(): pass\n") + writeWorkspaceFile(t, provider, "pkg/b.py", "def g(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['dist']\n") + source := "import pkg.a, pkg.b\ndef run():\n pkg.b.g()\n" + writeWorkspaceFile(t, consumer, "app.py", source) + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + _, err := mi.IndexAll() + require.NoError(t, err) + check := func() { + require.ElementsMatch(t, []string{"provider/pkg/a.py", "provider/pkg/b.py"}, revisionCallTargets(g, "consumer/app.py", graph.EdgeImports)) + require.ElementsMatch(t, []string{"provider/pkg/b.py::g"}, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + } + // Given two dotted imports on one line, cold indexing keeps both. + check() + // No-change and edited-file saves must link those same two files. + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "app.py"))) + check() + writeWorkspaceFile(t, consumer, "app.py", source+"# comment\n") + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "app.py"))) + check() + cold, coldGraph := newPythonBridgeIdentityIndexer(t, backend, entries) + _, err = cold.IndexAll() + require.NoError(t, err) + require.ElementsMatch(t, revisionCallTargets(coldGraph, "consumer/app.py", graph.EdgeImports), revisionCallTargets(g, "consumer/app.py", graph.EdgeImports)) + }) + } +} diff --git a/internal/resolver/module_attribution.go b/internal/resolver/module_attribution.go index 820a8564d..741a9afcc 100644 --- a/internal/resolver/module_attribution.go +++ b/internal/resolver/module_attribution.go @@ -69,11 +69,13 @@ func (r *Resolver) attributePythonModuleImportsForFiles(files []string) { func (r *Resolver) attributeNonGoModuleImportEdges(imports, dependencies iter.Seq[*graph.Edge], fileLang map[string]string) { type pendingEdge struct { - edge *graph.Edge - oldTo string - moduleID string + edge *graph.Edge + oldTo string + moduleID string + targetIdentity graph.EdgeIdentity } var rewrites []pendingEdge + targetSites := make(map[graph.EdgeIdentity]int) moduleSeeds := map[string]moduleSeed{} dependsSeen := map[string]map[string]struct{}{} // fileID → set of moduleIDs @@ -95,10 +97,15 @@ func (r *Resolver) attributeNonGoModuleImportEdges(imports, dependencies iter.Se if !ok { continue } + projected := *e + projected.To = moduleID + identity := graph.EdgeIdentityFor(&projected) + targetSites[identity]++ rewrites = append(rewrites, pendingEdge{ - edge: e, - oldTo: e.To, - moduleID: moduleID, + edge: e, + oldTo: e.To, + moduleID: moduleID, + targetIdentity: identity, }) moduleSeeds[moduleID] = moduleSeed{ id: moduleID, @@ -180,7 +187,11 @@ func (r *Resolver) attributeNonGoModuleImportEdges(imports, dependencies iter.Se } p.edge.Meta["python_import"] = strings.TrimPrefix(p.oldTo, "external::") } - p.edge.To = p.moduleID + // Distinct Python spellings on one line must survive until tracked retry. + // Keep their external identities, while still recording module evidence. + if !strings.HasPrefix(p.moduleID, "module::pypi:") || targetSites[p.targetIdentity] == 1 { + p.edge.To = p.moduleID + } p.edge.Origin = graph.OriginASTResolved reindexBatch = append(reindexBatch, graph.EdgeReindex{Edge: p.edge, OldTo: p.oldTo}) diff --git a/internal/resolver/module_attribution_test.go b/internal/resolver/module_attribution_test.go index e214f9713..3f9290e67 100644 --- a/internal/resolver/module_attribution_test.go +++ b/internal/resolver/module_attribution_test.go @@ -267,6 +267,12 @@ func TestAttributeNonGo_BatchedWritesPreserveDedupeAndOrdering(t *testing.T) { require.NotNil(t, base.GetNode("module::pypi:requests")) for i, edge := range imports { want := "module::pypi:numpy" + if i == 0 { + want = "external::numpy" + } + if i == 1 { + want = "external::numpy.linalg" + } if i == len(imports)-1 { want = "module::pypi:requests" } @@ -279,9 +285,8 @@ func TestAttributeNonGo_BatchedWritesPreserveDedupeAndOrdering(t *testing.T) { assert.Len(t, outEdgesOfKind(base, "app/c.py", graph.EdgeDependsOnModule), 1, "existingDepends must suppress an already persisted dependency") assert.Len(t, outEdgesOfKind(base, "app/d.py", graph.EdgeDependsOnModule), 1) - // Both app/a.py imports share line 1, so reindexing them to the same - // module converges them onto one edge identity. - assert.Equal(t, 8, base.EdgeCount(), "four import identities plus four dependency identities") + // Same-line spellings keep their distinct identities before tracked retry. + assert.Equal(t, 9, base.EdgeCount(), "five import identities plus four dependency identities") } func TestAttributeNonGo_BoundsModuleAndDependencyWriteBatches(t *testing.T) { diff --git a/internal/resolver/version.go b/internal/resolver/version.go index 7e569ef9d..1a1c81b31 100644 --- a/internal/resolver/version.go +++ b/internal/resolver/version.go @@ -41,6 +41,7 @@ import ( // for an unchanged corpus. // Go module binding changes candidate order, cross-repo provenance and import identity. // Python import and call stubs add stored metadata required by tracked-package linking. +// Same-line Python submodule imports retain distinct stored identities. const ResolutionSemanticsVersion = 6 // resolverVersionDomain versions the ENCODING below, not the contract. Two From 6a7d9fdfd482dee62327ddb4b0429645cdb0ed7b Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:58:32 -0700 Subject: [PATCH 079/111] fix(resolver): preserve Python call identities before provider tracking Keep colliding tracked-Python call stubs on per-symbol terminals so late provider tracking reopens every call. Cover default synthesis, unchanged saves and cold agreement on both stores. --- .../indexer/python_bridge_identity_test.go | 29 +++++++++++++++++++ internal/resolver/external_calls.go | 8 +++++ 2 files changed, 37 insertions(+) diff --git a/internal/indexer/python_bridge_identity_test.go b/internal/indexer/python_bridge_identity_test.go index a2a0199ff..df0d7396b 100644 --- a/internal/indexer/python_bridge_identity_test.go +++ b/internal/indexer/python_bridge_identity_test.go @@ -67,3 +67,32 @@ func TestTrackedPythonSameLineImportIdentities(t *testing.T) { }) } } + +func TestTrackedPythonSameLineCallsAfterConsumerFirst(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'dist'\n") + writeWorkspaceFile(t, provider, "pkg/__init__.py", "") + writeWorkspaceFile(t, provider, "pkg/image.py", "def f(): pass\ndef g(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['dist']\n") + writeWorkspaceFile(t, consumer, "app.py", "import pkg.image as image\ndef run():\n image.f(); image.g()\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, nil) + _, err := mi.TrackRepoCtx(context.Background(), entries[0]) + require.NoError(t, err) + require.True(t, mi.GetIndexer("consumer").externalCallSynthesisEnabled(), "exercise default synthesis policy") + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + require.NoError(t, err) + want := []string{"provider/pkg/image.py::f", "provider/pkg/image.py::g"} + require.ElementsMatch(t, want, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "pkg/image.py")}) + require.NoError(t, err) + require.ElementsMatch(t, want, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + cold, cg := newPythonBridgeIdentityIndexer(t, backend, entries) + _, err = cold.IndexAll() + require.NoError(t, err) + require.ElementsMatch(t, revisionCallTargets(cg, "consumer/app.py::run", graph.EdgeCalls), revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + }) + } +} diff --git a/internal/resolver/external_calls.go b/internal/resolver/external_calls.go index 00a86478e..c3c433712 100644 --- a/internal/resolver/external_calls.go +++ b/internal/resolver/external_calls.go @@ -258,6 +258,14 @@ func synthesizeExternalCalls(g graph.Store, collect externalCallCandidateSeq) in // edge on its bookkeeping-string terminal. continue } + // A later tracked Python provider needs every original symbol. + // Leave colliding calls on their distinct terminals rather than + // collapsing them onto the package-only synthetic node. + if stub, _ := e.Meta[PythonCallStubMeta].(string); stub != "" && + ((i > 0 && sameExternalCallDestination(pending[i-1], c)) || + (i+1 < len(pending) && sameExternalCallDestination(c, pending[i+1]))) { + continue + } if canDrop && last != nil && sameExternalCallDestination(*last, c) { converged = append(converged, e) continue From f4746774cca5a477ee77fa67e3cc81b72a1a6e5c Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:58:40 -0700 Subject: [PATCH 080/111] fix(resolver): normalize Cargo aliases for call workspace policy Apply the import's provider crate name to call authorization while keeping the exact-file and workspace checks. --- internal/resolver/cargo_crate.go | 12 ++++++- internal/resolver/cargo_crate_test.go | 47 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index 3f284ba09..643f2c19e 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -249,6 +249,16 @@ func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target stri continue } spec := strings.TrimPrefix(strings.TrimPrefix(stub, "external::"), "unresolved::import::") + workspaceSpec := spec + if cr.cargoCrates != nil { + alias, _, _ := strings.Cut(spec, "/") + for _, crate := range cr.cargoCrates(imp.FilePath, alias) { + if graph.RepoPrefixOfID(crate.Root) == graph.RepoPrefixOfID(imp.To) { + workspaceSpec = crate.Name + strings.TrimPrefix(spec, alias) + break + } + } + } bindings := parseRustUseBindings(spec) if strings.HasSuffix(spec, "/*") { bindings = []rustUseBinding{{source: normalizeRustUsePath(spec), local: "*"}} @@ -273,7 +283,7 @@ func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target stri lookupName = item } for _, candidate := range candidates[lookupName][imp.To] { - if candidate.FilePath != imp.To || candidate.Language != "rust" || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { + if candidate.FilePath != imp.To || candidate.Language != "rust" || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), workspaceSpec) { continue } switch candidate.Kind { diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go index 904ed819c..fa59e87a3 100644 --- a/internal/resolver/cargo_crate_test.go +++ b/internal/resolver/cargo_crate_test.go @@ -265,3 +265,50 @@ func TestCargoCallRequiresExactParentModule(t *testing.T) { } } } + +func TestCargoAliasedCallsWorkspacePolicy(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, allowed := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/allowed=%v", backend, allowed), func(t *testing.T) { + g := goImportReviewStore(t, backend) + for _, file := range []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs"} { + ws := "private" + if strings.HasPrefix(file, "consumer/") { + ws = "shared" + } + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), WorkspaceID: ws, Language: "rust"}) + } + g.AddNode(&graph.Node{ID: "provider/src/util.rs::helper", Name: "helper", FilePath: "provider/src/util.rs", Kind: graph.KindFunction, RepoPrefix: "provider", WorkspaceID: "private", Language: "rust"}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "external::oc/util/helper", Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "unresolved::helper", Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(_, alias string) []CargoCrate { + if alias != "oc" { + return nil + } + return []CargoCrate{{Root: "provider/src/lib.rs", Name: "other_crate"}} + }) + module := "oc" + if allowed { + module = "other_crate" + } + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { + return []CrossWorkspaceDepRule{{Workspace: "private", Modules: []string{module}}} + }) + for pass := 0; pass < 2; pass++ { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + imports := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeImports) + require.Len(t, imports, 1) + require.Equal(t, allowed, imports[0].CrossRepo) + calls := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeCalls) + require.Len(t, calls, 1) + require.Equal(t, allowed, calls[0].CrossRepo) + if allowed { + require.Equal(t, "provider/src/util.rs::helper", calls[0].To) + } + } + }) + } + } +} From 36c247403b7aa1e957bd2afbe372bac5144596a3 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:58:48 -0700 Subject: [PATCH 081/111] fix(resolver): report deferred Go import recovery Propagate both recovery-budget refusals and keep the retry state incomplete so callers can request the full pass. Verify bounded refusal and unbounded repair on both stores. --- internal/resolver/cross_repo.go | 26 +++++---- internal/resolver/cross_repo_incremental.go | 20 ++++++- .../go_module_import_recovery_test.go | 4 +- .../resolver/go_module_import_seed_test.go | 55 ++++++++++++++++++- internal/resolver/go_modules_cross_repo.go | 13 ++++- 5 files changed, 99 insertions(+), 19 deletions(-) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 4c1875ff6..95759c164 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -133,18 +133,19 @@ type CrossRepoResolver struct { // dangling edge). Off (the default, and the whole-pass-locked path) it is // a no-op: nothing can mutate the graph mid-pass, so every edge and // candidate is live by construction. - validateLiveness bool - crossWorkspaceLookup CrossWorkspaceDepLookup - goModules map[string]string - goModulePaths goModulePathIndex - goModuleRepos map[string][]string - goModuleReplacements map[string][]GoModuleReplacement - goModuleRetry *GoModuleDepRetry - goModuleRevision uint64 - goModuleDemotedMu sync.Mutex - goModuleDemotedFiles map[string]struct{} - goModuleSourceFiles map[string]struct{} - goModuleSourceSites map[graph.EdgeSite]map[string]struct{} + validateLiveness bool + crossWorkspaceLookup CrossWorkspaceDepLookup + goModules map[string]string + goModulePaths goModulePathIndex + goModuleRepos map[string][]string + goModuleReplacements map[string][]GoModuleReplacement + goModuleRetry *GoModuleDepRetry + goModuleImportsDeferred int + goModuleRevision uint64 + goModuleDemotedMu sync.Mutex + goModuleDemotedFiles map[string]struct{} + goModuleSourceFiles map[string]struct{} + goModuleSourceSites map[graph.EdgeSite]map[string]struct{} // npmAlias rewrites a JS/TS import specifier that matches an // npm-alias dependency key in the importing file's nearest- // ancestor package.json. Same contract as the field of the @@ -417,6 +418,7 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS if err := ctx.Err(); err != nil { return stats, err } + cr.goModuleImportsDeferred = 0 for repo := range cr.goModules { if err := cr.restoreGoModuleImports(ctx, repo, stats); err != nil { return stats, err diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 0f812cfe9..64ad57b5a 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -27,7 +27,7 @@ func (cr *CrossRepoResolver) ResolveMutationFiles(resolutionFiles, materializati } // CrossRepoIncomingAdmission is one cross-repository mutation pass's bounded -// incoming-stub admission: the completeness fact plus the typed refusal that +// incoming-stub and Go-import admission: the completeness fact plus the typed refusal that // produced it (*graph.BoundedLocalizationLimitError, nil when nothing was // refused). // @@ -65,7 +65,7 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed return stats, admission } - importBudget := &graph.IncomingSourceBudget{} + importBudget := newGoModuleImportRecoveryBudget() // Provider recovery needs the lazy seed before the pass lock is taken. // Ordinary consumer frontiers keep local resolution ahead of the bridge. if len(definitionFiles) > 0 { @@ -75,6 +75,7 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed } cr.mu.Lock() defer cr.mu.Unlock() + cr.goModuleImportsDeferred = 0 budget := &graph.IncomingSourceBudget{} if len(resolutionFiles) > 0 { @@ -187,10 +188,12 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed repos[repo] = true ids := cr.goModuleImportIDs(repo) if len(ids) > importBudget.Remaining() { + cr.goModuleImportsDeferred += len(ids) cr.logger.Warn("cross-repo resolve: Go module imports deferred to full pass", zap.Int("imports", len(ids)), zap.Int("remaining", importBudget.Remaining())) continue } if err := importBudget.Charge(len(ids)); err != nil { + cr.goModuleImportsDeferred += len(ids) cr.logger.Warn("cross-repo resolve: Go module imports deferred to full pass", zap.Error(err)) continue } @@ -201,6 +204,19 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed } } + if deferred := cr.goModuleImportsDeferred; deferred > 0 { + admission.Refused = true + admission.Dropped += deferred + admission.Limit = graph.MaxIncomingSourceCandidateRows + if admission.Refusal == nil { + admission.Refusal = &graph.BoundedLocalizationLimitError{Resource: "Go module import recovery", Limit: graph.MaxIncomingSourceCandidateRows} + } + if state := cr.goModuleRetry; state != nil { + state.mu.Lock() + state.complete = false + state.mu.Unlock() + } + } DetectCrossRepoEdgesForMutationFiles(cr.graph, edgeSourceFiles, definitionFiles) return stats, admission } diff --git a/internal/resolver/go_module_import_recovery_test.go b/internal/resolver/go_module_import_recovery_test.go index 71d663ab2..26be57830 100644 --- a/internal/resolver/go_module_import_recovery_test.go +++ b/internal/resolver/go_module_import_recovery_test.go @@ -84,7 +84,9 @@ func TestGoModuleImportBudgetDoesNotStarveCalls(t *testing.T) { cr = resolver(g, state) // When the import recovery exceeds its own ceiling, the call leg must still bind. _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/new/new.go"}) - assert.False(t, admission.Refused, "import refusal is not a call admission refusal") + assert.True(t, admission.Refused, "deferred imports must be reported even while independent calls bind") + assert.Error(t, admission.Refusal) + assert.False(t, state.complete, "deferred import recovery remains retryable") var calls []*graph.Edge for _, e := range g.GetOutEdges("a/main.go::Caller") { if e.Kind == graph.EdgeCalls { diff --git a/internal/resolver/go_module_import_seed_test.go b/internal/resolver/go_module_import_seed_test.go index 443cb3cfb..855886e4c 100644 --- a/internal/resolver/go_module_import_seed_test.go +++ b/internal/resolver/go_module_import_seed_test.go @@ -110,7 +110,8 @@ func TestGoModuleImportOversizedProviderDoesNotStarveNext(t *testing.T) { state.imports["b-small"] = map[string]struct{}{id: {}} if route == "frontier" { _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"a-big/lib.go", "b-small/lib.go"}) - require.False(t, admission.Refused) + require.True(t, admission.Refused) + require.Error(t, admission.Refusal) } else { require.NoError(t, cr.recheckGoModuleBindings(t.Context(), nil, &CrossRepoStats{ByRepo: map[string]int{}})) } @@ -155,3 +156,55 @@ func TestGoModuleNestedProviderFullAndWarmParity(t *testing.T) { } } } + +func TestGoModuleImportRecoveryBudgetRefusal(t *testing.T) { + old := newGoModuleImportRecoveryBudget + newGoModuleImportRecoveryBudget = func() *graph.IncomingSourceBudget { + b := &graph.IncomingSourceBudget{} + require.NoError(t, b.Charge(graph.MaxIncomingSourceCandidateRows-1)) + return b + } + t.Cleanup(func() { newGoModuleImportRecoveryBudget = old }) + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"frontier", "identity"} { + t.Run(backend+"/"+route, func(t *testing.T) { + g := goImportReviewStore(t, backend) + g.AddNode(&graph.Node{ID: "provider/lib.go", Name: "lib.go", FilePath: "provider/lib.go", Kind: graph.KindFile, RepoPrefix: "provider", Language: "go"}) + ids := map[string]struct{}{} + for i := 0; i < 2; i++ { + file := fmt.Sprintf("consumer/f%d.go", i) + id := file + "::import::example.org/lib" + ids[id] = struct{}{} + g.AddBatch([]*graph.Node{{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "go"}, {ID: id, FilePath: file, StartLine: 1, Kind: graph.KindImport, RepoPrefix: "consumer", Language: "go", Meta: map[string]any{"path": "example.org/lib"}}}, []*graph.Edge{{From: file, FilePath: file, To: "dep::example.org/lib", Kind: graph.EdgeImports, Line: 1}}) + } + cr := NewCrossRepo(g) + modules := map[string]string{"provider": "example.org/lib", "consumer": "example.org/app"} + cr.SetGoModules(modules) + state := cr.goModuleImportState() + state.importsSeeded = true + state.imports["provider"] = ids + state.complete = route == "frontier" + state.modules = modules + changed := "provider/lib.go" + if route == "identity" { + changed = "consumer/f0.go" + } + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{changed}) + require.True(t, admission.Refused) + require.Error(t, admission.Refusal) + require.GreaterOrEqual(t, admission.Dropped, 2) + require.False(t, state.complete) + // The explicit full pass is unbounded and repairs the deferred imports. + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + for i := 0; i < 2; i++ { + rows := outEdgesOfKind(g, fmt.Sprintf("consumer/f%d.go", i), graph.EdgeImports) + require.Len(t, rows, 1) + require.Equal(t, "provider/lib.go", rows[0].To) + } + _, err = cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + }) + } + } +} diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index fd59d2211..057b8b8fb 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -49,6 +49,10 @@ func (cr *CrossRepoResolver) resolveGoExtern(e *graph.Edge, spec string, stats * } } +var newGoModuleImportRecoveryBudget = func() *graph.IncomingSourceBudget { + return &graph.IncomingSourceBudget{} +} + // GoModuleDepRetry remembers the last completed legacy-terminal sweep for a // store's module/config revision. MultiIndexer shares it across pass objects. type GoModuleDepRetry struct { @@ -120,7 +124,7 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross cr.resolveGoModuleEdgesLocked(deferred, stats) state.mu.Lock() if len(cr.goModules) == 0 { - state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, true + state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, cr.goModuleImportsDeferred == 0 state.replacements = maps.Clone(cr.goModuleReplacements) state.mu.Unlock() return nil @@ -225,7 +229,7 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross } state.mu.Lock() defer state.mu.Unlock() - state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, true + state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, cr.goModuleImportsDeferred == 0 state.replacements = maps.Clone(cr.goModuleReplacements) return nil } @@ -517,6 +521,7 @@ func (cr *CrossRepoResolver) RetryGoModuleDeps(ctx context.Context) error { if err := func() error { cr.mu.Lock() defer cr.mu.Unlock() + cr.goModuleImportsDeferred = 0 return cr.retryGoModuleDeps(ctx, stats) }(); err != nil { return err @@ -659,17 +664,19 @@ func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previo } // A changed module can make another provider's fallback imports bindable, // even when that provider is outside the saved-file frontier. - importBudget := &graph.IncomingSourceBudget{} + importBudget := newGoModuleImportRecoveryBudget() for _, repo := range repos { if cr.goModules[repo] == "" { continue } importIDs := cr.goModuleImportIDs(repo) if len(importIDs) > importBudget.Remaining() { + cr.goModuleImportsDeferred += len(importIDs) cr.logger.Warn("Go module import identity recheck deferred to full pass", zap.Int("imports", len(importIDs)), zap.Int("remaining", importBudget.Remaining())) continue } if err := importBudget.Charge(len(importIDs)); err != nil { + cr.goModuleImportsDeferred += len(importIDs) cr.logger.Warn("Go module import identity recheck deferred to full pass", zap.Error(err)) continue } From e5700b6c465c7675a84b6a5487156ef434977a79 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:58:56 -0700 Subject: [PATCH 082/111] fix(resolver): preserve concrete Go imports during restoration Do not rebuild a second import over a legitimate binding, such as a vendored import. Keep separate same-line statement recovery and cover both stores. --- internal/resolver/go_module_imports.go | 25 ++++++++------------- internal/resolver/go_module_imports_test.go | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index 5b76010cf..e573e4a11 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -452,6 +452,14 @@ func (cr *CrossRepoResolver) restoreGoModuleImportBatch(ctx context.Context, rep if e.Kind != graph.EdgeImports || e.Line != line { continue } + // Any existing concrete Go-file binding owns this statement, + // including per-repo vendored bindings outside the module map. + boundPath, _ := e.Meta["go_import"].(string) + if (boundPath == "" || boundPath == path) && strings.HasSuffix(e.To, ".go") && !graph.IsUnresolvedTarget(e.To) && + !isSyntheticResolveTarget(e.To) && !strings.HasPrefix(e.To, externalCallPrefix) { + present = true + break + } target := graph.UnresolvedName(e.To) if target == "" { target = e.To @@ -463,22 +471,7 @@ func (cr *CrossRepoResolver) restoreGoModuleImportBatch(ctx context.Context, rep present = true break } - replacement, _ := goModuleReplacementFor(cr.goModuleReplacements, n.RepoPrefix, path) - if replacement.Directory != "" && graph.RepoPrefixOfID(e.To) == n.RepoPrefix && - strings.HasSuffix(e.To, ".go") && filePathDir(e.To) == replacement.Directory { - present = true - break - } - // A longer module may own the bound package, even when this - // statement is also registered under the edited provider. - targetRepo := graph.RepoPrefixOfID(e.To) - targetModule := cr.goModules[targetRepo] - if strings.HasSuffix(e.To, ".go") && targetModule != "" && - (path == targetModule || strings.HasPrefix(path, targetModule+"/")) && - filePathDir(e.To) == targetRepo+strings.TrimPrefix(path, targetModule) { - present = true - break - } + } if !present { cr.graph.AddEdge(&graph.Edge{From: n.FilePath, To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: n.FilePath, Line: line}) diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index fcef8f936..c696cac1e 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -697,3 +697,26 @@ func TestInvalidatePackageRepoKeepsOtherProviders(t *testing.T) { require.Equal(t, graph.FileNodeIdentity{ID: "b/lib.go", FilePath: "b/lib.go", RepoPrefix: "b"}, state.packages["b"]["b"]) require.True(t, state.complete, "package invalidation must not replace the terminal-sweep policy") } + +func TestGoModuleImportRestorePreservesVendoredBinding(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + for _, file := range []string{"consumer/main.go", "consumer/vendor/example.org/lib/lib.go", "provider/lib.go"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "go"}) + } + g.AddNode(&graph.Node{ID: "consumer/main.go::import::example.org/lib", FilePath: "consumer/main.go", StartLine: 2, Kind: graph.KindImport, RepoPrefix: "consumer", Language: "go", Meta: map[string]any{"path": "example.org/lib"}}) + g.AddEdge(&graph.Edge{From: "consumer/main.go", FilePath: "consumer/main.go", To: "consumer/vendor/example.org/lib/lib.go", Kind: graph.EdgeImports, Line: 2, Origin: graph.OriginASTResolved}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"provider": "example.org/lib"}) + for pass := 0; pass < 2; pass++ { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + rows := outEdgesOfKind(g, "consumer/main.go", graph.EdgeImports) + require.Len(t, rows, 1) + require.Equal(t, "consumer/vendor/example.org/lib/lib.go", rows[0].To) + require.False(t, rows[0].CrossRepo) + } + }) + } +} From de0b61785ec0a833fa3b6e28a40a8d534415fe88 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:59:05 -0700 Subject: [PATCH 083/111] fix(indexer): park local Go imports on admission skips Preserve caller-owned imports when size or content admission replaces their target with a skip node. Use the native restub contract and cover both returns plus normal saves. --- .../go_module_import_lifecycle_test.go | 57 +++++++++++++++++++ internal/indexer/indexer.go | 13 ++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/internal/indexer/go_module_import_lifecycle_test.go b/internal/indexer/go_module_import_lifecycle_test.go index a9a082654..c36ee98ec 100644 --- a/internal/indexer/go_module_import_lifecycle_test.go +++ b/internal/indexer/go_module_import_lifecycle_test.go @@ -13,6 +13,7 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/resolver" "github.com/zzet/gortex/internal/search" ) @@ -182,3 +183,59 @@ func TestGoModuleImportParkBeforeEviction(t *testing.T) { } } } + +// The optional asset capability makes the otherwise code-only Go language +// exercise the content-admission return as well as the ordinary size cap. +type goImportAdmissionExtractor struct{ parser.Extractor } + +func (goImportAdmissionExtractor) AssetClass() parser.AssetClass { return parser.AssetData } + +func TestGoModuleLocalImportSurvivesAdmissionSkip(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"size", "content", "normal"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + root := t.TempDir() + writeWorkspaceFile(t, root, "main.go", "package p\n") + writeWorkspaceFile(t, root, "lib.go", "package p\nfunc F() {}\n") + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + idx := newTestIndexer(g) + _, err := idx.Index(root) + require.NoError(t, err) + g.AddEdge(&graph.Edge{From: "main.go", FilePath: "main.go", To: "lib.go", Kind: graph.EdgeImports, Line: 1, Origin: graph.OriginASTResolved, Confidence: 1, Meta: map[string]any{"go_import": "example.org/lib", "go_import_fallback": "external::example.org/lib"}}) + if mode == "size" { + idx.config.MaxFileSize = 1 + } + if mode == "content" { + ext, ok := idx.registry.GetByLanguage("go") + require.True(t, ok) + idx.registry.Register(goImportAdmissionExtractor{ext}) + idx.config.Content.IndexData = false + } + for pass := 0; pass < 2; pass++ { + require.NoError(t, idx.IndexFileNoResolve(filepath.Join(root, "lib.go"))) + var imports []*graph.Edge + for _, e := range g.GetOutEdges("main.go") { + if e.Kind == graph.EdgeImports { + imports = append(imports, e) + } + } + require.Len(t, imports, 1) + if mode == "normal" { + require.Equal(t, "lib.go", imports[0].To) + } else { + require.Equal(t, "unresolved::import::example.org/lib", imports[0].To) + require.False(t, imports[0].CrossRepo) + require.Empty(t, imports[0].Origin) + require.Zero(t, imports[0].Confidence) + } + } + }) + } + } +} diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index c45f3dace..2613506a3 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -5215,6 +5215,15 @@ func (idx *Indexer) indexFile( } } + parkCarriedGoImports := func() []*graph.Edge { + for _, e := range carriedGoImports { + graph.StashRestubProvenance(e) + e.To = goModuleRestubTarget(e, "") + e.CrossRepo, e.ConfidenceLabel = false, "" + } + return carriedGoImports + } + // A delta probe already owns a shared admission lease together with its // transformed source and parse tree. Validate and consume that snapshot // before acquiring another lease; acquiring first can self-deadlock when @@ -5279,7 +5288,7 @@ func (idx *Indexer) indexFile( } } evictExisting(result) - idx.graph.AddBatch([]*graph.Node{n}, nil) + idx.graph.AddBatch([]*graph.Node{n}, parkCarriedGoImports()) if !idx.recordFileReadVersion(mtimeKey, absPath, readVersion) { return idx.fileIndexFailureError(absPath) } @@ -5303,7 +5312,7 @@ func (idx *Indexer) indexFile( } } evictExisting(result) - idx.graph.AddBatch([]*graph.Node{n}, nil) + idx.graph.AddBatch([]*graph.Node{n}, parkCarriedGoImports()) if !idx.recordFileReadVersion(mtimeKey, absPath, readVersion) { return idx.fileIndexFailureError(absPath) } From ae7e7b860566a9fe6ca7e64eb7db8791a58889e0 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:59:12 -0700 Subject: [PATCH 084/111] fix(resolver): admit restubbed Cargo imports in the prepass Build import bridge evidence before resolving calls from the same provider save. --- internal/resolver/cargo_crate.go | 3 +++ internal/resolver/cargo_crate_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index 643f2c19e..60e4baec0 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -188,6 +188,9 @@ func (cr *CrossRepoResolver) cargoImportSpec(e *graph.Edge) string { target = strings.Replace(target, "unresolved::import::", "external::", 1) } spec, external := strings.CutPrefix(target, "external::") + if !external { + spec, external = strings.CutPrefix(target, "unresolved::import::") + } if !external { return "" } diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go index fa59e87a3..46b50db43 100644 --- a/internal/resolver/cargo_crate_test.go +++ b/internal/resolver/cargo_crate_test.go @@ -312,3 +312,29 @@ func TestCargoAliasedCallsWorkspacePolicy(t *testing.T) { } } } + +func TestCargoUnresolvedImportAndCallSinglePass(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + for _, file := range []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "rust"}) + } + g.AddNode(&graph.Node{ID: "provider/src/util.rs::helper", Name: "helper", FilePath: "provider/src/util.rs", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "unresolved::import::other_crate/util/helper", Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "unresolved::helper", Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { + return []CargoCrate{{Root: "provider/src/lib.rs", Name: "other_crate"}} + }) + for pass := 0; pass < 2; pass++ { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + rows := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeCalls) + require.Len(t, rows, 1) + require.Equal(t, "provider/src/util.rs::helper", rows[0].To) + require.True(t, rows[0].CrossRepo) + } + }) + } +} From f8856e9754bbe651e3781a7cfd46aae2e8814641 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:59:20 -0700 Subject: [PATCH 085/111] fix(indexer): clear Python re-export evidence when restubbing calls Remove obsolete package provenance in every call-parking route and finish the direct Python rebind handoff. Verify complete rebound edge payloads on both stores. --- internal/indexer/incremental_batch.go | 1 + internal/indexer/indexer.go | 1 + internal/indexer/python_import_restub.go | 2 + internal/indexer/python_import_restub_test.go | 58 +++++++++++++++++++ internal/resolver/cross_repo.go | 1 + 5 files changed, 63 insertions(+) diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index f3be650c0..474739bf7 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -1568,6 +1568,7 @@ func restubIncomingRefsFromView( } if original, _ := edge.Meta[resolver.PythonCallStubMeta].(string); edge.Kind == graph.EdgeCalls && edge.CrossRepo && original != "" { edge.To, edge.CrossRepo, edge.ConfidenceLabel = original, false, "" + delete(edge.Meta, resolver.PythonReExportFileMeta) } reindexes = append(reindexes, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 2613506a3..5f422a8b8 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -6045,6 +6045,7 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string, [] } if original, _ := e.Meta[resolver.PythonCallStubMeta].(string); e.Kind == graph.EdgeCalls && e.CrossRepo && original != "" { e.To, e.CrossRepo, e.ConfidenceLabel = original, false, "" + delete(e.Meta, resolver.PythonReExportFileMeta) } batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) } diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index c396b038c..655bc83d5 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -91,6 +91,7 @@ func restubTrackedPythonImportsFromView(g graph.Store, evicted map[string]struct for _, prior := range incoming[previous] { if prior != nil && prior.From == call.From && prior.Kind == call.Kind && prior.Line == call.Line && prior.To == previous { prior.To, prior.CrossRepo, prior.ConfidenceLabel = call.To, false, "" + delete(prior.Meta, resolver.PythonReExportFileMeta) } } } @@ -98,6 +99,7 @@ func restubTrackedPythonImportsFromView(g graph.Store, evicted map[string]struct if edge.Kind == graph.EdgeCalls { graph.StashRestubProvenance(edge) edge.ConfidenceLabel = "" + delete(edge.Meta, resolver.PythonReExportFileMeta) } edge.To, edge.CrossRepo = stub, false batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: old}) diff --git a/internal/indexer/python_import_restub_test.go b/internal/indexer/python_import_restub_test.go index 404c7c674..1dddca9b5 100644 --- a/internal/indexer/python_import_restub_test.go +++ b/internal/indexer/python_import_restub_test.go @@ -58,3 +58,61 @@ func TestPythonRestubIgnoresMetadataOnlyTargets(t *testing.T) { require.Equal(t, target, edge.To) require.True(t, edge.CrossRepo) } + +func TestPythonRestubClearsReExportEvidence(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"point", "batch", "park"} { + t.Run(backend+"/"+route, func(t *testing.T) { + _, g := newPythonBridgeIdentityIndexer(t, backend, nil) + file, target := "consumer/app.py", "provider/pkg/mod.py" + caller := file + "::run" + g.AddNode(&graph.Node{ID: caller, Name: "run", FilePath: file, Kind: graph.KindFunction, RepoPrefix: "consumer", Language: "python"}) + g.AddBatch([]*graph.Node{{ID: file, Name: "app.py", FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "python"}, {ID: target, Name: "mod.py", FilePath: target, Kind: graph.KindFile, RepoPrefix: "provider", Language: "python"}, {ID: target + "::f", Name: "f", FilePath: target, Kind: graph.KindFunction, RepoPrefix: "provider", Language: "python"}}, nil) + imp := &graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports, CrossRepo: true, Line: 1, Meta: map[string]any{resolver.PythonImportStubMeta: "external::pkg.mod", "python_import": "pkg.mod"}} + linkedImport := *imp + g.AddEdge(imp) + stub := "unresolved::extern::pkg.mod::f" + g.AddEdge(&graph.Edge{From: caller, FilePath: file, To: stub, Kind: graph.EdgeCalls, Line: 2, Meta: map[string]any{resolver.PythonCallStubMeta: stub}}) + resolve := func() { + cr := resolver.NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *resolver.PythonPackageLookup { return nil }) + cr.ResolveForFile("consumer", "app.py") + } + resolve() + resolve() + calls := func() []*graph.Edge { + var rows []*graph.Edge + for _, e := range g.GetOutEdges(caller) { + if e.Kind == graph.EdgeCalls { + rows = append(rows, e) + } + } + require.Len(t, rows, 1) + return rows + } + require.Equal(t, target+"::f", calls()[0].To) + cold := builderRenderEdges(calls()) + call := calls()[0] + call.Meta[resolver.PythonReExportFileMeta] = "provider/pkg/__init__.py" + g.ReindexEdges([]graph.EdgeReindex{{Edge: call, OldTo: call.To}}) + switch route { + case "point": + newTestIndexer(g).restubIncomingRefs(target) + case "batch": + stages := []*incrementalBatchStage{{graphPath: target, priorNodes: g.GetFileNodes(target)}} + restubIncomingRefsFromView(g, stages, loadIncrementalPriorView(g, stages)) + case "park": + restubTrackedPythonRepoIncoming(g, "provider") + } + require.Equal(t, stub, calls()[0].To) + require.NotContains(t, calls()[0].Meta, resolver.PythonReExportFileMeta) + g.RemoveEdge(file, "external::pkg.mod", graph.EdgeImports) + g.AddEdge(&linkedImport) + resolve() + require.Equal(t, cold, builderRenderEdges(calls())) + resolve() + require.Equal(t, cold, builderRenderEdges(calls())) + }) + } + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 95759c164..67d49e654 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -1370,6 +1370,7 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b return eligible, nil } if resolvePythonModuleExtern(e, module, symbol, callerRepo, &ResolveStats{}, find) || cr.resolvePythonPackageReExport(e, module, symbol) { + graph.RestoreRestubProvenance(e) stats.Resolved++ if e.CrossRepo { stats.CrossRepoEdges++ From 66f696ddf11fd18d3b1a891270edc51a529fa6ea Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:59:27 -0700 Subject: [PATCH 086/111] fix(resolver): refuse foreign Python test-package bindings Exclude root and src packages named test or tests at provider discovery and at Python call resolution. Keep consumer-local bindings and normal modules. pytest test paths and other non-exported packages are not covered yet. --- internal/indexer/python_tracked_import.go | 3 ++ .../indexer/python_tracked_import_test.go | 54 +++++++++++++++++++ internal/resolver/python_absolute_imports.go | 12 +++++ 3 files changed, 69 insertions(+) diff --git a/internal/indexer/python_tracked_import.go b/internal/indexer/python_tracked_import.go index 2e9295931..a5873b239 100644 --- a/internal/indexer/python_tracked_import.go +++ b/internal/indexer/python_tracked_import.go @@ -97,6 +97,9 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { continue } top := strings.TrimSuffix(name, ".py") + if resolver.IsPythonTestPackageFile(file, "") { + continue // Root/src test packages are not provider exports. + } node := mi.graph.GetNode(repo + "/" + file) if node == nil || node.Kind != graph.KindFile || node.RepoPrefix != repo { continue diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go index 3f636096e..ead3a3ab1 100644 --- a/internal/indexer/python_tracked_import_test.go +++ b/internal/indexer/python_tracked_import_test.go @@ -352,3 +352,57 @@ func TestPythonConsumerLookupFiltersAndDeduplicates(t *testing.T) { require.Equal(t, []string{"consumer/f0.py", "consumer/f1.py", "consumer/f2.py"}, pythonConsumerFilesForLookup(g, lookup, "provider")) require.Equal(t, 1, calls, "provider lookup is per (caller, top), not per import") } + +func TestTrackedPythonProviderTestPackagesRefused(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, dir := range []string{"", "src/"} { + for _, top := range []string{"tests", "test"} { + for _, local := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s%s/local=%v", backend, dir, top, local), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + providerAlias := "provider" + if dir == "src/" && top == "test" { + providerAlias = "tests" + } + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'image-distribution'\n") + writeWorkspaceFile(t, provider, dir+top+"/__init__.py", "") + writeWorkspaceFile(t, provider, dir+top+"/helpers.py", "def f(): pass\n") + writeWorkspaceFile(t, provider, dir+"image.py", "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['image-distribution']\n") + writeWorkspaceFile(t, consumer, "app.py", "import "+top+".helpers\nimport image\ndef run():\n "+top+".helpers.f()\n") + if local { + writeWorkspaceFile(t, consumer, top+"/__init__.py", "") + writeWorkspaceFile(t, consumer, top+"/helpers.py", "def f(): pass\n") + } + mi, g := newPythonBridgeIdentityIndexer(t, backend, []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: providerAlias, Workspace: "shared"}}) + _, err := mi.IndexAll() + require.NoError(t, err) + check := func() { + var targets []string + for _, e := range g.GetOutEdges("consumer/app.py") { + if e.Kind != graph.EdgeImports { + continue + } + targets = append(targets, e.To) + require.NotEqual(t, providerAlias+"/"+dir+top+"/helpers.py", e.To) + if e.To == providerAlias+"/"+dir+"image.py" { + require.True(t, e.CrossRepo) + } + } + require.Contains(t, targets, providerAlias+"/"+dir+"image.py") + if local { + require.Contains(t, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls), "consumer/"+top+"/helpers.py::f") + } + for _, e := range g.GetOutEdges("consumer/app.py::run") { + require.NotEqual(t, providerAlias+"/"+dir+top+"/helpers.py::f", e.To) + } + } + check() + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "app.py"))) + check() + }) + } + } + } + } +} diff --git a/internal/resolver/python_absolute_imports.go b/internal/resolver/python_absolute_imports.go index 3793299ec..09a6c94b1 100644 --- a/internal/resolver/python_absolute_imports.go +++ b/internal/resolver/python_absolute_imports.go @@ -58,6 +58,15 @@ func pythonModuleFileMatches(filePath, module string) bool { return false } +// IsPythonTestPackageFile identifies the supported root/src test packages, +// relative to the provider rather than its repository alias. +func IsPythonTestPackageFile(filePath, repoPrefix string) bool { + relative := strings.TrimPrefix(filePath, repoPrefix+"/") + relative = strings.TrimPrefix(relative, "src/") + top, _, nested := strings.Cut(relative, "/") + return nested && (top == "tests" || top == "test") +} + // resolvePythonModuleExtern resolves a Python caller's extern edge onto the // module-level function or class it imports. It tries the two readings of a // dotted import path in turn — the symbol defined in the module importPath @@ -88,6 +97,9 @@ func resolvePythonModuleExtern(e *graph.Edge, importPath, symbol, callerRepo str var match *graph.Node ambiguous := false for _, c := range candidates { + if c.RepoPrefix != callerRepo && IsPythonTestPackageFile(c.FilePath, c.RepoPrefix) { + continue + } if c.Kind != graph.KindFunction && c.Kind != graph.KindType { continue } From ed61cf1c8e2f5818336934f8704341e3a9275ae1 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:59:34 -0700 Subject: [PATCH 087/111] fix(resolver): keep AST evidence for all Cargo imported calls Give import-authorized functions and path-qualified constructors the same evidence as wildcard methods. Keep the exact-module rejection unchanged. --- internal/resolver/cargo_crate.go | 4 +-- internal/resolver/cargo_crate_test.go | 41 +++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go index 60e4baec0..955f29098 100644 --- a/internal/resolver/cargo_crate.go +++ b/internal/resolver/cargo_crate.go @@ -312,9 +312,7 @@ func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target stri return false } e.To, e.CrossRepo = match.ID, true - if strings.HasPrefix(target, "*.") { - e.Confidence, e.Origin = 0.85, graph.OriginASTResolved - } + e.Confidence, e.Origin = 0.85, graph.OriginASTResolved return true } diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go index 46b50db43..90c8cc3cc 100644 --- a/internal/resolver/cargo_crate_test.go +++ b/internal/resolver/cargo_crate_test.go @@ -338,3 +338,44 @@ func TestCargoUnresolvedImportAndCallSinglePass(t *testing.T) { }) } } + +func TestCargoImportAuthorizedCallEvidence(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"function", "path_constructor", "wildcard_method"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + g := goImportReviewStore(t, backend) + file, provider := "consumer/src/lib.rs", "provider/src/lib.rs" + for _, id := range []string{file, provider} { + g.AddNode(&graph.Node{ID: id, FilePath: id, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(id), Language: "rust"}) + } + name, item, target, kind := "helper", "helper", "unresolved::helper", graph.KindFunction + meta := map[string]any{} + if mode != "function" { + item, kind = "Foo", graph.KindMethod + name = "new" + target = "unresolved::new" + meta["rust_path"] = "Foo::new" + } + if mode == "wildcard_method" { + name, target = "go", "unresolved::*.go" + delete(meta, "rust_path") + meta["receiver_type"] = "Foo" + } + g.AddNode(&graph.Node{ID: provider + "::" + name, Name: name, FilePath: provider, Kind: kind, RepoPrefix: "provider", Language: "rust", Meta: map[string]any{"receiver": "Foo"}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "external::other/" + item, Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeCalls, Line: 2, Meta: meta}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return []CargoCrate{{Root: provider, Name: "other"}} }) + for pass := 0; pass < 2; pass++ { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + rows := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, rows, 1) + require.Equal(t, provider+"::"+name, rows[0].To) + require.Equal(t, 0.85, rows[0].Confidence) + require.Equal(t, graph.OriginASTResolved, rows[0].Origin) + } + }) + } + } +} From 7406b88090f62de05e82644632b9510688f99d95 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:59:42 -0700 Subject: [PATCH 088/111] perf(resolver): batch Go import ownership and statistics lookups Reuse source file payloads and batch target ownership for cross-repo import counts. Assert native work counts and unchanged-pass behavior on both stores. --- internal/resolver/go_module_imports.go | 30 +++++++++++++++---- internal/resolver/go_module_imports_test.go | 33 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go index e573e4a11..c2404d7c7 100644 --- a/internal/resolver/go_module_imports.go +++ b/internal/resolver/go_module_imports.go @@ -24,6 +24,17 @@ func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files [ var batch []graph.EdgeReindex for _, path := range paths { cr.rememberGoModuleImports(nodes[path]) + var source *graph.Node + for _, node := range nodes[path] { + if node.Kind == graph.KindFile && node.ID == path { + source = node + break + } + } + caller := "" + if source != nil { + caller = source.RepoPrefix + } for _, e := range outgoing[path] { if e.Kind != graph.EdgeImports { continue @@ -32,11 +43,6 @@ func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files [ if importPath == "" || !trackedGoImport(cr.goModulePaths, importPath) && e.Meta["go_import"] == nil { continue } - source := cr.graph.GetNode(e.From) - caller := "" - if source != nil { - caller = source.RepoPrefix - } match := cr.goModuleImportCandidate(importPath, caller, candidateWorkspaceID(source)) if match != nil && match.ID == e.To || match == nil && e.Meta["go_import"] == nil { continue @@ -201,6 +207,18 @@ func (cr *CrossRepoResolver) applyGoModuleImportReindexes(batch []graph.EdgeRein if stats == nil { return } + targetIDs := make([]string, 0, len(batch)) + seenTargets := make(map[string]bool) + for _, row := range batch { + if row.Edge.CrossRepo && !seenTargets[row.Edge.To] { + seenTargets[row.Edge.To] = true + targetIDs = append(targetIDs, row.Edge.To) + } + } + var targets map[string]*graph.Node + if len(targetIDs) > 0 { + targets = cr.graph.GetNodesByIDs(targetIDs) + } for _, row := range batch { original := graph.EdgeIdentityFor(row.Edge) original.To = row.OldTo @@ -223,7 +241,7 @@ func (cr *CrossRepoResolver) applyGoModuleImportReindexes(batch []graph.EdgeRein } if row.Edge.CrossRepo { stats.CrossRepoEdges++ - if target := cr.graph.GetNode(row.Edge.To); target != nil { + if target := targets[row.Edge.To]; target != nil { stats.ByRepo[target.RepoPrefix]++ } } diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index c696cac1e..1b9de3d87 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -720,3 +720,36 @@ func TestGoModuleImportRestorePreservesVendoredBinding(t *testing.T) { }) } } + +func TestGoModuleImportUsesBatchedFileAndStatsNodes(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &resolverBatchCountingStore{Store: goImportReviewStore(t, backend)} + g.AddBatch([]*graph.Node{{ID: "consumer/main.go", FilePath: "consumer/main.go", Kind: graph.KindFile, RepoPrefix: "consumer", Language: "go"}, {ID: "provider/lib.go", FilePath: "provider/lib.go", Kind: graph.KindFile, RepoPrefix: "provider", Language: "go"}}, nil) + const count = 64 + for i := 0; i < count; i++ { + g.AddEdge(&graph.Edge{From: "consumer/main.go", FilePath: "consumer/main.go", To: "external::example.org/lib", Kind: graph.EdgeImports, Line: i + 1}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"provider": "example.org/lib"}) + for pass := 0; pass < 2; pass++ { + g.getNodeCalls, g.getFileNodesByPathsCalls = 0, 0 + stats := &CrossRepoStats{ByRepo: map[string]int{}} + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"consumer/main.go"}, stats, nil)) + require.Zero(t, g.getNodeCalls, "reuse loaded caller files and batch statistics targets") + require.Equal(t, 1, g.getFileNodesByPathsCalls) + rows := outEdgesOfKind(g, "consumer/main.go", graph.EdgeImports) + require.Len(t, rows, count) + for _, e := range rows { + require.Equal(t, "provider/lib.go", e.To) + } + if pass == 0 { + require.Equal(t, count, stats.ByRepo["provider"]) + } else { + require.Zero(t, stats.Resolved) + require.Empty(t, stats.ByRepo) + } + } + }) + } +} From 150d23ce0664543d7fcb5e420f18b12fec977e39 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 11:59:53 -0700 Subject: [PATCH 089/111] fix(indexer): match cold output across tracked-import lifecycles Compare complete Cargo and Python edge payloads, and cover rename and back and untrack-only. Repair the provenance, terminal-stamp, cold-order and purge-retry gaps this exposed. Fallback and synthesis run before the tracked-provider bridge, as in a cold index, so bridge refusals still win. After provider removal, consumers match a consumer-only cold index. --- .../indexer/cargo_import_lifecycle_test.go | 77 +++++++++++-- internal/indexer/go_module_lifecycle_test.go | 2 +- internal/indexer/multi.go | 10 +- internal/indexer/npm_lifecycle_test.go | 2 +- .../indexer/python_import_fallback_test.go | 2 +- .../indexer/python_import_lifecycle_test.go | 89 ++++++++++++--- .../python_import_warm_recovery_test.go | 2 +- .../indexer/python_package_reexport_test.go | 14 ++- internal/indexer/repository_untrack.go | 44 +++++++- .../indexer/repository_untrack_retry_test.go | 106 ++++++++++++++++++ internal/indexer/workspace_resolve.go | 20 +++- internal/resolver/cross_repo.go | 7 ++ internal/resolver/resolver.go | 4 + internal/resolver/terminal.go | 56 +++++++++ 14 files changed, 394 insertions(+), 41 deletions(-) diff --git a/internal/indexer/cargo_import_lifecycle_test.go b/internal/indexer/cargo_import_lifecycle_test.go index 7cc2d752f..653b0fcb3 100644 --- a/internal/indexer/cargo_import_lifecycle_test.go +++ b/internal/indexer/cargo_import_lifecycle_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "os" "path/filepath" "slices" @@ -24,7 +25,7 @@ import ( func TestTrackedCargoImportLifecycle(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { for _, providerFirst := range []bool{false, true} { - for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "provider_gains_empty_module", "no_new_file_noop", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track", "deferred_track"} { + for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "provider_gains_empty_module", "no_new_file_noop", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "rename_and_back", "untrack_only", "no_change", "track", "deferred_track"} { if operation == "deferred_track" && providerFirst { continue // This case starts with an already tracked consumer. } @@ -127,6 +128,19 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { }, calls) } assertImport() + initial := cargoLifecycleBindings(g, "") + initialMarkers := map[string]string{} + for _, node := range g.GetFileNodes("consumer/src/lib.rs") { + for _, edge := range g.GetOutEdges(node.ID) { + key := "cargo_call_stub" + if edge.Kind == graph.EdgeImports { + key = "cargo_import_stub" + } + if marker, ok := edge.Meta[key].(string); ok { + initialMarkers[fmt.Sprint(edge.Kind, edge.Line)] = marker + } + } + } logs.TakeAll() switch operation { case "consumer_save": @@ -231,12 +245,46 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { result, noChangeErr := mi.IncrementalReindexRepo("provider", nil) require.NoError(t, noChangeErr) require.Zero(t, result.StaleFileCount) + case "rename_and_back": + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper_renamed() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) + require.NoError(t, err) + for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 8 { + require.False(t, edge.CrossRepo) + require.Equal(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)], edge.Meta["cargo_call_stub"]) + } + } + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) + case "untrack_only": + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") case "untrack_retrack": _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") require.NoError(t, err) _, err = mi.TrackRepoCtx(context.Background(), entries[1]) } require.NoError(t, err) + if operation == "rename_and_back" { + require.Equal(t, initial, cargoLifecycleBindings(g, "")) + } + if operation == "untrack_only" { + for _, node := range g.GetFileNodes("consumer/src/lib.rs") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind != graph.EdgeImports && edge.Kind != graph.EdgeCalls { + continue + } + require.False(t, edge.CrossRepo) + require.False(t, strings.HasPrefix(edge.To, "provider/")) + key := "cargo_call_stub" + if edge.Kind == graph.EdgeImports { + key = "cargo_import_stub" + } + require.NotEmpty(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)]) + require.Equal(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)], edge.Meta[key]) + } + } + } if operation == "provider_delete_util_batched" || operation == "forced_evict_util" { var parked []*graph.Edge for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { @@ -256,7 +304,12 @@ func TestTrackedCargoImportLifecycle(t *testing.T) { t.Cleanup(func() { require.NoError(t, db.Close()) }) cold = db } - coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + coldConfig := cm + if operation == "untrack_only" { + coldConfig = newTestConfigManager(t) + coldConfig.Global().Repos = []config.RepoEntry{entries[0]} + } + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), coldConfig, zap.NewNop()) t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) _, err = coldIndexer.IndexAll() require.NoError(t, err) @@ -276,16 +329,20 @@ func cargoLifecycleBindings(g graph.Store, operation string) []string { var bindings []string for _, node := range g.GetFileNodes("consumer/src/lib.rs") { for _, edge := range g.GetOutEdges(node.ID) { - if edge.Kind == graph.EdgeImports || edge.Kind == graph.EdgeCalls { - var stub any - if edge.Kind == graph.EdgeCalls { - stub = edge.Meta["cargo_call_stub"] - if edge.Line == 8 && (operation == "provider_delete_util_batched" || operation == "forced_evict_util") { - stub = nil // Cold has no prior bind; the exact parked marker is asserted separately. - } + if edge.Kind != graph.EdgeImports && edge.Kind != graph.EdgeCalls { + continue + } + copy := *edge + parked := operation == "untrack_only" || ((operation == "provider_delete_util_batched" || operation == "forced_evict_util") && ((edge.Kind == graph.EdgeImports && edge.Line == 3) || (edge.Kind == graph.EdgeCalls && edge.Line == 8))) + if parked && !edge.CrossRepo { + copy.Meta = maps.Clone(edge.Meta) + key := "cargo_call_stub" + if edge.Kind == graph.EdgeImports { + key = "cargo_import_stub" } - bindings = append(bindings, fmt.Sprintf("%s -%s-> %s cross=%v line=%d stub=%v", edge.From, edge.Kind, edge.To, edge.CrossRepo, edge.Line, stub)) + delete(copy.Meta, key) // Historical parked identity is asserted separately. } + bindings = append(bindings, builderRenderEdge(©)) } } slices.Sort(bindings) diff --git a/internal/indexer/go_module_lifecycle_test.go b/internal/indexer/go_module_lifecycle_test.go index 3fffefb44..d38f2b6c8 100644 --- a/internal/indexer/go_module_lifecycle_test.go +++ b/internal/indexer/go_module_lifecycle_test.go @@ -294,7 +294,7 @@ func TestNonGoRepoEvictionSkipsModuleRestub(t *testing.T) { g.AddNode(&graph.Node{ID: "js/main.js::Hello", RepoPrefix: "js", FilePath: "js/main.js", Kind: graph.KindFunction, Language: "javascript", Name: "Hello"}) mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) - _, _, err := mi.purgeRepositoryPayload("js", &RepoMetadata{RootPath: t.TempDir()}) + _, _, err := mi.purgeRepositoryPayload("js", &RepoMetadata{RootPath: t.TempDir()}, nil) require.NoError(t, err) assert.Zero(t, g.definitionScans, "non-Go eviction must not run the module incoming-edge scan") assert.Nil(t, g.GetNode("js/main.js::Hello")) diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 2288e16f9..824b84acd 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -63,6 +63,8 @@ type RepoMetadata struct { // retries from repeating successful payload/vector phases and keeps the stable // mutation lane closed until every external side effect commits. type repositoryUntrackState struct { + bridgeConsumerFiles []string + retainAdmission bool mu sync.Mutex metadata *RepoMetadata @@ -992,11 +994,9 @@ func (mi *MultiIndexer) newMasterResolver(useLSP bool) *resolver.Resolver { } master := resolver.New(mi.graph) master.SetLogger(mi.logger) - // The master resolve is the only pass with whole-graph evidence, so it is - // the one allowed to durably flag terminally-unresolved edges (permanently - // external / stdlib / definition-less) that a later scoped warm resolve can - // skip. A scoped/file pass leaves the flag alone; only a full ResolveAll - // stamps and self-heals terminal state. + // Full resolution owns terminal stamps. Completed tracked-package catch-up + // may finish marked callers using the same global name-count proof through + // ReconcileBridgeCallerTerminals; ordinary file passes only carry stamps. master.SetStampTerminal(true) // The pre-enrichment queryability pass uses resolver-time LSP precision. // The post-enrichment catch-up disables it because semantic providers just diff --git a/internal/indexer/npm_lifecycle_test.go b/internal/indexer/npm_lifecycle_test.go index 0122f9abf..c89b4e365 100644 --- a/internal/indexer/npm_lifecycle_test.go +++ b/internal/indexer/npm_lifecycle_test.go @@ -50,7 +50,7 @@ func TestTrackedNpmImportsSurviveEviction(t *testing.T) { evictRepoCurrentGeneration(g, "lib") case "untrack": mi := &MultiIndexer{graph: g} - _, _, err := mi.purgeRepositoryPayload("lib", nil) + _, _, err := mi.purgeRepositoryPayload("lib", nil, nil) require.NoError(t, err) } var targets []string diff --git a/internal/indexer/python_import_fallback_test.go b/internal/indexer/python_import_fallback_test.go index 2ccf239b8..3e5c650ee 100644 --- a/internal/indexer/python_import_fallback_test.go +++ b/internal/indexer/python_import_fallback_test.go @@ -70,7 +70,7 @@ func TestTrackedPythonNewModuleThroughFallback(t *testing.T) { t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) _, err = coldIndexer.IndexAll() require.NoError(t, err) - require.Equal(t, pythonLifecycleBindings(cold, false), pythonLifecycleBindings(g, false)) + require.Equal(t, pythonLifecycleBindings(cold, ""), pythonLifecycleBindings(g, "")) }) } } diff --git a/internal/indexer/python_import_lifecycle_test.go b/internal/indexer/python_import_lifecycle_test.go index 0acecbcb5..40cc49c3e 100644 --- a/internal/indexer/python_import_lifecycle_test.go +++ b/internal/indexer/python_import_lifecycle_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "os" "path/filepath" "slices" @@ -24,7 +25,7 @@ import ( func TestTrackedPythonImportLifecycle(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { for _, providerFirst := range []bool{false, true} { - for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "provider_gains_empty_module", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "no_change", "track", "deferred_track"} { + for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "provider_gains_empty_module", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "rename_and_back", "untrack_only", "no_change", "track", "deferred_track"} { if operation == "deferred_track" && providerFirst { continue // This case starts with an already tracked consumer. } @@ -122,6 +123,19 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { }, calls) } assertImport() + initial := pythonLifecycleBindings(g, "") + initialMarkers := map[string]string{} + for _, node := range g.GetFileNodes("consumer/app.py") { + for _, edge := range g.GetOutEdges(node.ID) { + key := "python_call_stub" + if edge.Kind == graph.EdgeImports { + key = "python_import_stub" + } + if marker, ok := edge.Meta[key].(string); ok { + initialMarkers[fmt.Sprint(edge.Kind, edge.Line)] = marker + } + } + } logs.TakeAll() var scans *pythonLookupScanStore if operation == "provider_save" { @@ -168,7 +182,8 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { } } require.Len(t, parked, 1) - require.Equal(t, "unresolved::extern::PIL.util.helper::helper", parked[0].To) + require.Equal(t, "external-call::dep::PIL.util.helper", parked[0].To) + require.Equal(t, "unresolved::extern::PIL.util.helper::helper", parked[0].Meta["python_call_stub"]) require.False(t, parked[0].CrossRepo) writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) @@ -229,12 +244,46 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { result, noChangeErr := mi.IncrementalReindexRepo("provider", nil) require.NoError(t, noChangeErr) require.Zero(t, result.StaleFileCount) + case "rename_and_back": + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper_renamed(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + require.NoError(t, err) + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 10 { + require.False(t, edge.CrossRepo) + require.Equal(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)], edge.Meta["python_call_stub"]) + } + } + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + case "untrack_only": + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") case "untrack_retrack": _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") require.NoError(t, err) _, err = mi.TrackRepoCtx(context.Background(), entries[1]) } require.NoError(t, err) + if operation == "rename_and_back" { + require.Equal(t, initial, pythonLifecycleBindings(g, "")) + } + if operation == "untrack_only" { + for _, node := range g.GetFileNodes("consumer/app.py") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind != graph.EdgeImports && edge.Kind != graph.EdgeCalls { + continue + } + require.False(t, edge.CrossRepo) + require.False(t, strings.HasPrefix(edge.To, "provider/")) + key := "python_call_stub" + if edge.Kind == graph.EdgeImports { + key = "python_import_stub" + } + require.NotEmpty(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)]) + require.Equal(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)], edge.Meta[key]) + } + } + } if scans != nil { // The existing cross-repo pass scans imports once for reachability. require.Equal(t, 1, scans.importScans, "ordinary saves must not add a Python consumer-frontier scan") @@ -255,10 +304,11 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { require.Len(t, missingImport, 1) require.Equal(t, "module::pypi:PIL", missingImport[0].To) require.Equal(t, "PIL.util", missingImport[0].Meta["python_import"]) + require.Equal(t, "module::pypi:PIL", missingImport[0].Meta["python_import_stub"]) require.False(t, missingImport[0].CrossRepo) require.Len(t, missingCall, 1) - require.Equal(t, "unresolved::extern::PIL.util.helper::helper", missingCall[0].To) - require.Equal(t, missingCall[0].To, missingCall[0].Meta["python_call_stub"]) + require.Equal(t, "external-call::dep::PIL.util.helper", missingCall[0].To) + require.Equal(t, "unresolved::extern::PIL.util.helper::helper", missingCall[0].Meta["python_call_stub"]) require.False(t, missingCall[0].CrossRepo) } var cold graph.Store = graph.New() @@ -268,11 +318,16 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { t.Cleanup(func() { require.NoError(t, db.Close()) }) cold = db } - coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + coldConfig := cm + if operation == "untrack_only" { + coldConfig = newTestConfigManager(t) + coldConfig.Global().Repos = []config.RepoEntry{entries[0]} + } + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), coldConfig, zap.NewNop()) t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) _, err = coldIndexer.IndexAll() require.NoError(t, err) - require.Equal(t, pythonLifecycleBindings(cold, missingUtil), pythonLifecycleBindings(g, missingUtil), "after %s", operation) + require.Equal(t, pythonLifecycleBindings(cold, operation), pythonLifecycleBindings(g, operation), "after %s", operation) require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) require.Empty(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveAll").All()) }) @@ -284,20 +339,24 @@ func TestTrackedPythonImportLifecycle(t *testing.T) { const pythonLifecycleProvider = "class Image:\n def show(self): pass\ndef f(): pass\n" const pythonLifecycleConsumer = "from PIL.image import Image\nfrom PIL.image import f\nimport PIL.image as image\nfrom PIL.util import helper\ndef run():\n x = Image()\n x.show()\n f()\n image.f()\n helper()\n" -func pythonLifecycleBindings(g graph.Store, missingUtil bool) []string { +func pythonLifecycleBindings(g graph.Store, operation string) []string { var bindings []string for _, node := range g.GetFileNodes("consumer/app.py") { for _, edge := range g.GetOutEdges(node.ID) { - if edge.Kind == graph.EdgeImports || edge.Kind == graph.EdgeCalls { - if missingUtil && (edge.Kind == graph.EdgeImports && edge.Line == 4 || edge.Kind == graph.EdgeCalls && edge.Line == 10) { - continue // These exact parked sites are asserted in the deletion case. - } - var stub any - if edge.Kind == graph.EdgeCalls { - stub = edge.Meta["python_call_stub"] + if edge.Kind != graph.EdgeImports && edge.Kind != graph.EdgeCalls { + continue + } + copy := *edge + parked := operation == "untrack_only" || ((operation == "provider_delete_util_batched" || operation == "forced_evict_util") && ((edge.Kind == graph.EdgeImports && edge.Line == 4) || (edge.Kind == graph.EdgeCalls && edge.Line == 10))) + if parked && !edge.CrossRepo { + copy.Meta = maps.Clone(edge.Meta) + key := "python_call_stub" + if edge.Kind == graph.EdgeImports { + key = "python_import_stub" } - bindings = append(bindings, fmt.Sprintf("%s -%s-> %s cross=%v line=%d stub=%v", edge.From, edge.Kind, edge.To, edge.CrossRepo, edge.Line, stub)) + delete(copy.Meta, key) // Historical parked identity is asserted separately. } + bindings = append(bindings, builderRenderEdge(©)) } } slices.Sort(bindings) diff --git a/internal/indexer/python_import_warm_recovery_test.go b/internal/indexer/python_import_warm_recovery_test.go index 195b8db46..bbcd7ade1 100644 --- a/internal/indexer/python_import_warm_recovery_test.go +++ b/internal/indexer/python_import_warm_recovery_test.go @@ -93,7 +93,7 @@ func TestTrackedPythonWarmFileRecovery(t *testing.T) { cold := pythonLocalStore(t, backend) _, err = newMI(cold).IndexAll() require.NoError(t, err) - require.Equal(t, pythonLifecycleBindings(cold, false), pythonLifecycleBindings(g, false)) + require.Equal(t, pythonLifecycleBindings(cold, ""), pythonLifecycleBindings(g, "")) }) } } diff --git a/internal/indexer/python_package_reexport_test.go b/internal/indexer/python_package_reexport_test.go index 3a8b2d371..e29ed1fd0 100644 --- a/internal/indexer/python_package_reexport_test.go +++ b/internal/indexer/python_package_reexport_test.go @@ -63,6 +63,7 @@ func TestTrackedPythonPackageReExport(t *testing.T) { require.Len(t, imports, 1) require.Equal(t, "provider/src/itsdangerous/__init__.py", imports[0].To) require.True(t, imports[0].CrossRepo) + external := "external-call::dep::itsdangerous.URLSafeTimedSerializer" assertCall := func(target string, linked bool) { t.Helper() calls = nil @@ -75,7 +76,11 @@ func TestTrackedPythonPackageReExport(t *testing.T) { require.Equal(t, linked, calls[0].CrossRepo, "%#v", calls[0]) require.Equal(t, target, calls[0].To) require.NotEmpty(t, calls[0].Meta["python_call_stub"]) - require.NotContains(t, calls[0].Meta, "external_call") + if target == external { + require.Equal(t, true, calls[0].Meta["external_call"]) + } else { + require.NotContains(t, calls[0].Meta, "external_call") + } } stub := "unresolved::extern::itsdangerous.URLSafeTimedSerializer::URLSafeTimedSerializer" target := "provider/src/itsdangerous/url_safe.py::URLSafeTimedSerializer" @@ -96,7 +101,8 @@ func TestTrackedPythonPackageReExport(t *testing.T) { assertCall(target, true) _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") require.NoError(t, err) - assertCall(stub, false) + // A fresh consumer-only IndexAll materializes this dependency call. + assertCall(external, false) _, err = mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: provider, Name: "provider", Workspace: "shared"}) require.NoError(t, err) assertCall(target, true) @@ -197,6 +203,10 @@ func TestTrackedPythonPackageReExportColdPaths(t *testing.T) { require.Equal(t, "provider/src/itsdangerous/__init__.py", call.Meta["python_reexport_file"]) } require.Equal(t, expected, target, "call: %#v", call) + if ambiguous { + require.Equal(t, "unresolved::extern::itsdangerous.URLSafeTimedSerializer::URLSafeTimedSerializer", call.To) + require.NotContains(t, call.Meta, "external_call") + } if mode == "incremental" { incrementalTarget = target } else { diff --git a/internal/indexer/repository_untrack.go b/internal/indexer/repository_untrack.go index a7db767c7..b2f2ccf85 100644 --- a/internal/indexer/repository_untrack.go +++ b/internal/indexer/repository_untrack.go @@ -189,7 +189,7 @@ func (mi *MultiIndexer) untrackRepoCheckedRetainingAdmission( // retry does not repeat a completed destructive scan. refresh := func(sw *search.Swappable) error { if !state.payloadPurged { - removedNodes, removedEdges, purgeErr := mi.purgeRepositoryPayload(repoPrefix, state.metadata) + removedNodes, removedEdges, purgeErr := mi.purgeRepositoryPayload(repoPrefix, state.metadata, &state.bridgeConsumerFiles) if purgeErr != nil { return purgeErr } @@ -287,12 +287,16 @@ func (mi *MultiIndexer) finalizeCompletedRepositoryUntrackConfig( } func (mi *MultiIndexer) purgeRepositoryPayload( - repoPrefix string, meta *RepoMetadata, + repoPrefix string, meta *RepoMetadata, bridgeConsumers *[]string, ) (nodesRemoved, edgesRemoved int, err error) { restubTrackedNpmRepoIncoming(mi.graph, repoPrefix) parked := restubGoModuleRepoIncoming(mi.graph, repoPrefix) - restubTrackedCargoRepoIncoming(mi.graph, repoPrefix) - restubTrackedPythonRepoIncoming(mi.graph, repoPrefix) + var localConsumers []string + if bridgeConsumers == nil { + bridgeConsumers = &localConsumers + } + *bridgeConsumers = appendUniqueSorted(*bridgeConsumers, restubTrackedCargoRepoIncoming(mi.graph, repoPrefix)...) + *bridgeConsumers = appendUniqueSorted(*bridgeConsumers, restubTrackedPythonRepoIncoming(mi.graph, repoPrefix)...) if purger, ok := mi.graph.(interface{ PurgeRepo(string) error }); ok { if err := purger.PurgeRepo(repoPrefix); err != nil { return 0, 0, fmt.Errorf("purge repository payload: %w", err) @@ -300,6 +304,7 @@ func (mi *MultiIndexer) purgeRepositoryPayload( if err := mi.finalizeUntrackedGoModuleCalls(repoPrefix, parked); err != nil { return 0, 0, err } + mi.finalizeUntrackedBridgeConsumers(*bridgeConsumers) if meta != nil { return meta.NodeCount, meta.EdgeCount, nil } @@ -308,6 +313,9 @@ func (mi *MultiIndexer) purgeRepositoryPayload( nodesRemoved, edgesRemoved, err = evictRepoAllGenerations(mi.graph, repoPrefix) if err == nil { err = mi.finalizeUntrackedGoModuleCalls(repoPrefix, parked) + if err == nil { + mi.finalizeUntrackedBridgeConsumers(*bridgeConsumers) + } } return nodesRemoved, edgesRemoved, err } @@ -329,3 +337,31 @@ func (mi *MultiIndexer) finalizeUntrackedGoModuleCalls(removedRepo string, parke mi.reconcileRetargetedTestCalls(cr.TakeRetargetedTestCallFiles()) return nil } + +// Removal must settle parked callers too; an immediate retrack used to hide +// their unclassified terminals and unfinished provenance handoff. +func (mi *MultiIndexer) finalizeUntrackedBridgeConsumers(files []string) { + if len(files) == 0 { + return + } + mi.runMasterResolveFiles(files, false) + mi.synthesizeDemotedGoModuleCalls(files) // Scoped synthesis honors each caller's config for every language. + if master := mi.newMasterResolver(true); master != nil { + master.ReconcileBridgeCallerTerminals(files) + } + mu := mi.graph.ResolveMutex() + mu.Lock() + defer mu.Unlock() + var batch []graph.EdgeReindex + for row := range graph.EdgesInScopeSeq(mi.graph, nil, files, graph.EdgeCalls) { + edge := row.Edge + if !graph.HasRestubProvenance(edge) { + continue + } + graph.RestoreRestubProvenance(edge) + batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: edge.To}) + } + if len(batch) > 0 { + mi.graph.ReindexEdges(batch) + } +} diff --git a/internal/indexer/repository_untrack_retry_test.go b/internal/indexer/repository_untrack_retry_test.go index b51350cb0..5ca497c69 100644 --- a/internal/indexer/repository_untrack_retry_test.go +++ b/internal/indexer/repository_untrack_retry_test.go @@ -473,3 +473,109 @@ func BenchmarkRepositoryCleanupFailureRetry(b *testing.B) { }) } } + +type bridgeRetryCleanupStore struct { + graph.Store + failed bool +} + +func (s *bridgeRetryCleanupStore) PurgeRepo(prefix string) error { + if !s.failed { + s.failed = true + return errInjectedRepositoryPurge + } + _, _, err := evictRepoAllGenerations(s.Store, prefix) + return err +} + +func (s *bridgeRetryCleanupStore) CountNodesByNameClass(names []string, kinds []graph.NodeKind) map[string]graph.NodeNameClassCount { + if counter, ok := s.Store.(graph.NodeNameClassCounter); ok { + return counter.CountNodesByNameClass(names, kinds) + } + allowed := map[graph.NodeKind]bool{} + for _, kind := range kinds { + allowed[kind] = true + } + out := map[string]graph.NodeNameClassCount{} + for _, name := range names { + count := graph.NodeNameClassCount{} + for _, node := range s.FindNodesByName(name) { + if graph.IsStub(node.ID) { + count.Stub++ + } else if allowed[node.Kind] { + count.Real++ + } + } + out[name] = count + } + return out +} + +type bridgeUnknownCounterStore struct{ graph.Store } + +func (s *bridgeUnknownCounterStore) PurgeRepo(prefix string) error { + return s.Store.(interface{ PurgeRepo(string) error }).PurgeRepo(prefix) +} + +type bridgeTerminalLSP struct{} + +func (bridgeTerminalLSP) SupportsPath(string) bool { return true } +func (bridgeTerminalLSP) Definition(string, int, string) (string, int, bool) { + panic("terminal reconciliation must not call LSP") +} + +func TestRepositoryCleanupRetainsBridgeCallersAcrossPurgeFailure(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"native_counter", "lsp_supported", "unknown_counter"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + var base graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(t.TempDir() + "/graph.sqlite") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }) + base = db + } + store := &bridgeRetryCleanupStore{Store: base} + source, target := "consumer/src/lib.rs::run", "provider/src/lib.rs::helper" + store.AddBatch([]*graph.Node{{ID: "consumer/src/lib.rs", Name: "lib.rs", FilePath: "consumer/src/lib.rs", Kind: graph.KindFile, RepoPrefix: "consumer", Language: "rust"}, {ID: source, Name: "run", FilePath: "consumer/src/lib.rs", Kind: graph.KindFunction, RepoPrefix: "consumer", Language: "rust"}, {ID: "provider/src/lib.rs", Name: "lib.rs", FilePath: "provider/src/lib.rs", Kind: graph.KindFile, RepoPrefix: "provider", Language: "rust"}, {ID: target, Name: "helper", FilePath: "provider/src/lib.rs", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}}, []*graph.Edge{{From: source, FilePath: "consumer/src/lib.rs", To: target, Kind: graph.EdgeCalls, Line: 2, CrossRepo: true, Origin: graph.OriginASTResolved, Confidence: 0.85, Meta: map[string]any{"cargo_call_stub": "unresolved::helper"}}}) + var activeStore graph.Store = store + if mode == "unknown_counter" { + activeStore = &bridgeUnknownCounterStore{Store: store} + } + mi := NewMultiIndexer(activeStore, newTestRegistry(), nil, newTestConfigManager(t), zap.NewNop()) + if mode == "lsp_supported" { + mi.resolverLSPHelper = bridgeTerminalLSP{} + } + mi.repos["consumer"] = &RepoMetadata{RepoPrefix: "consumer", RootPath: t.TempDir()} + mi.repos["provider"] = &RepoMetadata{RepoPrefix: "provider", RootPath: t.TempDir()} + if _, _, err := mi.UntrackRepoChecked(t.Context(), "provider"); !errors.Is(err, errInjectedRepositoryPurge) { + t.Fatalf("first untrack = %v", err) + } + if _, _, err := mi.UntrackRepoChecked(t.Context(), "provider"); err != nil { + t.Fatal(err) + } + rows := store.GetOutEdges(source) + if len(rows) != 1 { + t.Fatalf("calls = %d", len(rows)) + } + call := rows[0] + if call.To != "unresolved::helper" || call.CrossRepo || graph.HasRestubProvenance(call) || (call.Meta["resolve_terminal"] == true) != (mode == "native_counter") { + t.Fatalf("unsettled caller after purge retry: %s", builderRenderEdge(call)) + } + settled := builderRenderEdge(call) + if _, _, err := mi.UntrackRepoChecked(t.Context(), "provider"); err != nil { + t.Fatal(err) + } + if got := builderRenderEdge(store.GetOutEdges(source)[0]); got != settled { + t.Fatalf("no-change untrack altered caller: %s", got) + } + }) + } + } +} diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 7c28acf74..fac31f2ba 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -4,6 +4,7 @@ import ( "context" "path/filepath" "sort" + "strings" "sync" "sync/atomic" @@ -562,7 +563,24 @@ func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, ed return } cr.SetCargoImportRetargetRepo(cargoRetargetRepo...) - cr.ResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles) + var bridgeFiles []string + for _, file := range resolutionFiles { + if strings.HasSuffix(file, ".rs") || strings.HasSuffix(file, ".py") || strings.HasSuffix(file, ".pyi") { + bridgeFiles = append(bridgeFiles, file) + } + } + // Match cold ordering: tracked-provider refusal must remain authoritative + // after fallback has handled genuinely external calls. + if len(bridgeFiles) > 0 { + mi.runMasterResolveFiles(bridgeFiles, false) + mi.synthesizeDemotedGoModuleCalls(bridgeFiles) + } + _, admission := cr.ResolveMutationFrontiersBounded(resolutionFiles, edgeSourceFiles, definitionFiles) + if !admission.Refused && len(bridgeFiles) > 0 { + if master := mi.newMasterResolver(true); master != nil { + master.ReconcileBridgeCallerTerminals(bridgeFiles) + } + } mi.finishGoModuleCalls(cr) mi.reconcileRetargetedTestCalls(cr.TakeRetargetedTestCallFiles()) } diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 67d49e654..7a00c8ddc 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -1399,7 +1399,12 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b } } + stashed := graph.HasRestubProvenance(e) + if stashed { + graph.RestoreRestubProvenance(e) + } if e.To != oldTo { + clearEdgeTerminal(e) if e.Kind == graph.EdgeCalls && e.CrossRepo && cr.cargoCrates != nil && strings.HasSuffix(e.FilePath, ".rs") { if e.Meta == nil { e.Meta = make(map[string]any) @@ -1416,6 +1421,8 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b } *batch = append(*batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) cr.noteRetargetedCall(e) + } else if stashed { + *batch = append(*batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) } } diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 01dd058d4..6bc617547 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -4123,6 +4123,10 @@ func (r *Resolver) resolveEdge(e *graph.Edge, stats *ResolveStats) (oldTo string e.Meta[PythonCallStubMeta] = oldTo } } + if e.To != oldTo { + graph.RestoreRestubProvenance(e) + clearEdgeTerminal(e) + } return oldTo, e.To != oldTo } diff --git a/internal/resolver/terminal.go b/internal/resolver/terminal.go index c91c99140..0d611d80e 100644 --- a/internal/resolver/terminal.go +++ b/internal/resolver/terminal.go @@ -372,3 +372,59 @@ func (r *Resolver) reconcileTerminalStampsBulk(pending []*graph.Edge, counter gr } return changed, stamped, unstamped } + +// ReconcileBridgeCallerTerminals finishes a completed tracked-package mutation +// using the full-pass policy for its affected marked calls. Callers must finish +// bridge resolution first; refused/incomplete passes leave the calls retryable. +func (r *Resolver) ReconcileBridgeCallerTerminals(files []string) { + if len(files) == 0 { + return + } + r.mu.Lock() + defer r.mu.Unlock() + _, counted := r.graph.(graph.NodeNameClassCounter) + _, memory := r.graph.(*graph.Graph) + if !counted && !memory { + return + } // Unknown adapters cannot certify absence. + var pending []*graph.Edge + for row := range graph.EdgesInScopeSeq(r.graph, nil, files, graph.EdgeCalls) { + edge := row.Edge + if edge == nil || !graph.IsUnresolvedTarget(edge.To) || (edge.Meta[CargoCallStubMeta] == nil && edge.Meta[PythonCallStubMeta] == nil) { + continue + } + if r.lspHelper != nil && r.lspHelper.SupportsPath(edge.FilePath) { + continue + } + pending = append(pending, edge) + } + var changed []*graph.Edge + if counter, ok := r.graph.(graph.NodeNameClassCounter); ok { + changed, _, _ = r.reconcileTerminalStampsBulk(pending, counter) + } else { + for _, edge := range pending { + reason, terminal := r.classifyTerminal(edge) + if terminal && !edgeTerminalFlag(edge) { + setEdgeTerminal(edge, reason) + changed = append(changed, edge) + } else if !terminal && edgeTerminalFlag(edge) { + clearEdgeTerminal(edge) + changed = append(changed, edge) + } + } + } + if len(changed) == 0 { + return + } + if p, ok := r.graph.(graph.EdgeTerminalStampPersister); ok { + p.PersistEdgeTerminalStamps(changed) + } else if p, ok := r.graph.(graph.EdgeMetaBatchPersister); ok { + p.PersistEdgeAttributesBatch(changed) + } else { + batch := make([]graph.EdgeReindex, 0, len(changed)) + for _, edge := range changed { + batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: edge.To}) + } + r.graph.ReindexEdges(batch) + } +} From af33f3ddc9872bc6a00bddf8937d56de04032775 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 12:00:03 -0700 Subject: [PATCH 090/111] test(indexer): classify tracked-bridge metadata restaters Name the seven bridge resolution metadata keys and exercise them in a small mixed-language two-repository fixture. --- .../indexer/edit_delta_meta_key_class_test.go | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/internal/indexer/edit_delta_meta_key_class_test.go b/internal/indexer/edit_delta_meta_key_class_test.go index 15c55c454..951837819 100644 --- a/internal/indexer/edit_delta_meta_key_class_test.go +++ b/internal/indexer/edit_delta_meta_key_class_test.go @@ -51,7 +51,14 @@ var metaKeysRestatedOutsideTheFingerprint = map[string]string{ "import_path": "modules scanner / resolver external_call_attribution (module nodes)", // Python imports are restated by whole resolves, file saves, and tracked binding. - "python_import": "resolver attributeNonGoModuleImports / attributePythonModuleImportsForFiles / resolveTrackedPythonImports (exact dotted import spelling)", + "python_import": "resolver attributeNonGoModuleImports / attributePythonModuleImportsForFiles / resolveTrackedPythonImports (exact dotted import spelling)", + "cargo_import_stub": "resolver bindCargoImport / restubTrackedCargoImportsFromView (exact pre-bind import target)", + "cargo_import_module": "resolver bindCargoImport / resolveCargoImports (matched module; cleared by Cargo restub)", + "cargo_call_stub": "resolver CrossRepoResolver.resolveEdge / reuseBindingMetaKeys carry (fresh call spelling)", + "python_import_stub": "resolver resolveTrackedPythonImports / restubTrackedPythonImportsFromView (exact pre-bind target)", + "python_call_stub": "resolver Resolver.resolveEdge / CrossRepoResolver.resolveEdge / reuseBindingMetaKeys carry", + "python_reexport_file": "resolver resolvePythonPackageReExport / resolutionMetaAdded prior-binding carry (cleared by Python restub)", + "go_import_fallback": "resolver resolveGoModuleImports (original import target retained in copied Meta)", // Test classification after extraction (test_edges.go). "is_test": "test pass (test_edges.go)", "is_test_file": "test pass (test_edges.go)", @@ -203,3 +210,42 @@ func TestPythonImportSpellingEditColdEquivalent(t *testing.T) { }) } } + +func TestTrackedBridgeMetadataKeysHaveRestaters(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + trees := map[string]map[string]string{ + provider: {"go.mod": "module example.org/library\n", "lib.go": "package library\nfunc Hello() {}\n", "Cargo.toml": "[package]\nname='bridge-crate'\nversion='0.1.0'\n", "src/lib.rs": "pub fn helper() {}\n", "pyproject.toml": "[project]\nname='image-dist'\n", "pkg/__init__.py": "from .impl import Factory\n", "pkg/impl.py": "class Factory: pass\n"}, + consumer: {"go.mod": "module example.org/caller\nrequire example.org/library v1.0.0\n", "main.go": "package caller\nimport library \"example.org/library\"\nfunc Run() { library.Hello() }\n", "Cargo.toml": "[package]\nname='consumer'\nversion='0.1.0'\n[dependencies]\nbridge-crate='1'\n", "src/lib.rs": "use bridge_crate::helper;\npub fn run() { helper(); }\n", "pyproject.toml": "[project]\ndependencies=['image-dist']\n", "app.py": "from pkg import Factory\ndef run(): return Factory()\n"}, + } + for root, tree := range trees { + for file, source := range tree { + writeWorkspaceFile(t, root, file, source) + } + } + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + mi.registry.Register(languages.NewRustExtractor()) + _, err := mi.IndexAll() + require.NoError(t, err) + check := func() { + present := map[string]bool{} + for _, edge := range g.AllEdges() { + for key := range edge.Meta { + present[key] = true + } + } + for _, key := range []string{"cargo_import_stub", "cargo_import_module", "cargo_call_stub", "python_import_stub", "python_call_stub", "python_reexport_file", "go_import_fallback"} { + require.True(t, present[key], "fixture must exercise %s", key) + require.NotEmpty(t, metaKeysRestatedOutsideTheFingerprint[key], "missing bridge restater for %s", key) + } + } + check() + result, err := mi.IncrementalReindexRepo("consumer", nil) + require.NoError(t, err) + require.Zero(t, result.StaleFileCount) + check() + }) + } +} From e9d21bee918a63129dd9a91b73cf8da335510ca2 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 14:24:24 -0700 Subject: [PATCH 091/111] fix(resolver): let a full pass complete deferred Go import recovery A full pass restores every import without a budget, so it no longer counts deferrals and can mark recovery complete. Keep the bounded deferral on mutation passes only. Gate the Cargo and Python terminal reconcile on the incoming refusal alone, not on a Go import deferral. --- internal/indexer/workspace_resolve.go | 2 +- internal/resolver/cross_repo.go | 2 +- internal/resolver/cross_repo_incremental.go | 6 ++++-- internal/resolver/go_module_import_seed_test.go | 15 ++++++++++++++- internal/resolver/go_modules_cross_repo.go | 11 +++++++---- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index fac31f2ba..140165415 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -576,7 +576,7 @@ func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, ed mi.synthesizeDemotedGoModuleCalls(bridgeFiles) } _, admission := cr.ResolveMutationFrontiersBounded(resolutionFiles, edgeSourceFiles, definitionFiles) - if !admission.Refused && len(bridgeFiles) > 0 { + if !admission.IncomingRefused && len(bridgeFiles) > 0 { if master := mi.newMasterResolver(true); master != nil { master.ReconcileBridgeCallerTerminals(bridgeFiles) } diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 7a00c8ddc..984170bed 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -424,7 +424,7 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS return stats, err } } - if err := cr.retryGoModuleDeps(ctx, stats); err != nil { + if err := cr.retryGoModuleDeps(ctx, stats, true); err != nil { return stats, err } // Fresh placeholder-source set per pass — same rationale as ResolveAll diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 64ad57b5a..a2928a5b1 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -38,7 +38,8 @@ func (cr *CrossRepoResolver) ResolveMutationFiles(resolutionFiles, materializati // one that refused to admit the incoming work it had. type CrossRepoIncomingAdmission struct { graph.IncomingSourceAdmission - Refusal error + Refusal error + IncomingRefused bool // Separates incoming proof from independent Go import recovery. } // ResolveMutationFrontiers preserves the distinct edge-source and changed- @@ -173,7 +174,7 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed // A newly tracked module also makes old dep/stdlib terminals eligible. // This indexed sweep runs at most once per module/config revision. if !admission.Refused { - if err := cr.retryGoModuleDeps(context.Background(), stats); err != nil { + if err := cr.retryGoModuleDeps(context.Background(), stats, false); err != nil { cr.logger.Error("cross-repo resolve: retry Go module terminals", zap.Error(err)) } } @@ -204,6 +205,7 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed } } + admission.IncomingRefused = admission.Refused if deferred := cr.goModuleImportsDeferred; deferred > 0 { admission.Refused = true admission.Dropped += deferred diff --git a/internal/resolver/go_module_import_seed_test.go b/internal/resolver/go_module_import_seed_test.go index 855886e4c..152748e3c 100644 --- a/internal/resolver/go_module_import_seed_test.go +++ b/internal/resolver/go_module_import_seed_test.go @@ -113,7 +113,7 @@ func TestGoModuleImportOversizedProviderDoesNotStarveNext(t *testing.T) { require.True(t, admission.Refused) require.Error(t, admission.Refusal) } else { - require.NoError(t, cr.recheckGoModuleBindings(t.Context(), nil, &CrossRepoStats{ByRepo: map[string]int{}})) + require.NoError(t, cr.recheckGoModuleBindings(t.Context(), nil, &CrossRepoStats{ByRepo: map[string]int{}}, false)) } imports := outEdgesOfKind(g, "consumer/main.go", graph.EdgeImports) require.Len(t, imports, 1, "the small provider must still restore its import") @@ -194,9 +194,21 @@ func TestGoModuleImportRecoveryBudgetRefusal(t *testing.T) { require.Error(t, admission.Refusal) require.GreaterOrEqual(t, admission.Dropped, 2) require.False(t, state.complete) + require.False(t, admission.IncomingRefused) // The explicit full pass is unbounded and repairs the deferred imports. _, err := cr.ResolveAllContext(t.Context()) require.NoError(t, err) + require.True(t, state.complete, "a full pass must finish the deferred revision") + for _, restart := range []bool{false, true} { + next := cr + if restart { + next = NewCrossRepo(g) + next.SetGoModules(modules) + next.SetGoModuleDepRetry(state, cr.goModuleRevision) + } + _, nextAdmission := next.ResolveMutationFrontiersBounded(nil, nil, []string{"consumer/f0.go"}) + require.False(t, nextAdmission.Refused, "completed revision must not reopen its bounded recovery") + } for i := 0; i < 2; i++ { rows := outEdgesOfKind(g, fmt.Sprintf("consumer/f%d.go", i), graph.EdgeImports) require.Len(t, rows, 1) @@ -204,6 +216,7 @@ func TestGoModuleImportRecoveryBudgetRefusal(t *testing.T) { } _, err = cr.ResolveAllContext(t.Context()) require.NoError(t, err) + require.True(t, state.complete) }) } } diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index 057b8b8fb..877aa2a5c 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -76,7 +76,7 @@ func (cr *CrossRepoResolver) SetGoModuleDepRetry(state *GoModuleDepRetry, revisi // retryGoModuleDeps visits only external terminals, filters by tracked module // identity, and bounds mutation batches. Symbol lookup is memoized per batch. -func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *CrossRepoStats) error { +func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *CrossRepoStats, importsRestored bool) error { if err := ctx.Err(); err != nil { return err } @@ -105,7 +105,7 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross if seedBindings || changedReplacements { previous = nil } - if err := cr.recheckGoModuleBindings(ctx, previous, stats); err != nil { + if err := cr.recheckGoModuleBindings(ctx, previous, stats, importsRestored); err != nil { return err } } @@ -522,7 +522,7 @@ func (cr *CrossRepoResolver) RetryGoModuleDeps(ctx context.Context) error { cr.mu.Lock() defer cr.mu.Unlock() cr.goModuleImportsDeferred = 0 - return cr.retryGoModuleDeps(ctx, stats) + return cr.retryGoModuleDeps(ctx, stats, false) }(); err != nil { return err } @@ -603,7 +603,7 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { // Re-check only marked incoming calls into providers affected by identity edits. // The compact definition stream avoids loading a whole repo's nodes or edges. -func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previous map[string]string, stats *CrossRepoStats) error { +func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previous map[string]string, stats *CrossRepoStats, importsRestored bool) error { changed := make(map[string]bool) for repo, module := range previous { if cr.goModules[repo] != module { @@ -662,6 +662,9 @@ func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previo if len(ids) > 0 { flush() } + if importsRestored { + return ctx.Err() + } // A changed module can make another provider's fallback imports bindable, // even when that provider is outside the saved-file frontier. importBudget := newGoModuleImportRecoveryBudget() From 22c3dd2fac2b9fd73ab1c18415d7eb0bdf243f30 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 14:24:42 -0700 Subject: [PATCH 092/111] fix(resolver): persist meta-only reindex rows in chunked passes A row that keeps its unresolved target needs a live source edge, not a node for the target. Real retargets still need a live concrete target. Both stores now keep the same restub provenance. --- internal/resolver/cross_repo.go | 4 +- .../resolver/cross_repo_meta_liveness_test.go | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 internal/resolver/cross_repo_meta_liveness_test.go diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 984170bed..6c18a2013 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -1442,7 +1442,7 @@ func (cr *CrossRepoResolver) filterLiveReindex(batch []graph.EdgeReindex) []grap sites = append(sites, graph.EdgeSite{ From: reindex.Edge.From, Line: reindex.Edge.Line, Kind: reindex.Edge.Kind, }) - if !isSyntheticResolveTarget(reindex.Edge.To) { + if reindex.Edge.To != reindex.OldTo && !isSyntheticResolveTarget(reindex.Edge.To) { targetIDs = append(targetIDs, reindex.Edge.To) } } @@ -1464,7 +1464,7 @@ func (cr *CrossRepoResolver) filterLiveReindex(batch []graph.EdgeReindex) []grap if !live { continue } - if !isSyntheticResolveTarget(edge.To) && targets[edge.To] == nil { + if edge.To != reindex.OldTo && !isSyntheticResolveTarget(edge.To) && targets[edge.To] == nil { edge.To = reindex.OldTo continue } diff --git a/internal/resolver/cross_repo_meta_liveness_test.go b/internal/resolver/cross_repo_meta_liveness_test.go new file mode 100644 index 000000000..e62244712 --- /dev/null +++ b/internal/resolver/cross_repo_meta_liveness_test.go @@ -0,0 +1,44 @@ +package resolver + +import ( + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "testing" +) + +func TestCrossRepoChunkedRestubMetadataPersists(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, chunk := range []string{"0", "1"} { + t.Run(backend+"/chunk="+chunk, func(t *testing.T) { + t.Setenv("GORTEX_RESOLVE_CHUNK", chunk) + g := goImportReviewStore(t, backend) + call := &graph.Edge{From: "caller/main.py::run", To: "provider/gone.py::missing", Kind: graph.EdgeCalls, FilePath: "caller/main.py", Line: 2, Confidence: 0.85, Origin: graph.OriginASTResolved} + graph.StashRestubProvenance(call) + call.To = "unresolved::missing" + g.AddBatch([]*graph.Node{{ID: "caller/main.py", Kind: graph.KindFile, FilePath: "caller/main.py", RepoPrefix: "caller", Language: "python"}, {ID: call.From, Name: "run", Kind: graph.KindFunction, FilePath: call.FilePath, RepoPrefix: "caller", Language: "python"}}, []*graph.Edge{call}) + for i := 0; i < 2; i++ { + _, err := NewCrossRepo(g).ResolveAllContext(t.Context()) + require.NoError(t, err) + rows := g.GetOutEdges(call.From) + require.Len(t, rows, 1) + require.Equal(t, "unresolved::missing", rows[0].To) + require.False(t, graph.HasRestubProvenance(rows[0])) + require.Empty(t, rows[0].Origin) + require.Zero(t, rows[0].Confidence) + } + }) + } + } +} + +func TestCrossRepoMetaReindexStillRequiresLiveEdge(t *testing.T) { + g := graph.New() + cr := NewCrossRepo(g) + call := &graph.Edge{From: "a/main.py::run", To: "unresolved::missing", Kind: graph.EdgeCalls, FilePath: "a/main.py", Line: 2} + require.Empty(t, cr.filterLiveReindex([]graph.EdgeReindex{{Edge: call, OldTo: call.To}})) + g.AddEdge(call) + copy := *call + copy.To = "b/gone.py::missing" + require.Empty(t, cr.filterLiveReindex([]graph.EdgeReindex{{Edge: ©, OldTo: call.To}})) + require.Equal(t, call.To, copy.To) +} From 176e84f370c705a50681fe8f2b23d2bf59a9b077 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 14:24:54 -0700 Subject: [PATCH 093/111] perf(indexer): skip the second incoming resolve on Python and Rust saves Receipt and watcher callers pass the files they already resolved, and the bridge step does only the remaining outgoing work. A Python save now runs one incoming resolve instead of two, on both stores. --- internal/indexer/incremental_watcher_batch.go | 5 +- internal/indexer/multi.go | 4 +- .../indexer/python_save_resolve_work_test.go | 52 +++++++++++++++++++ internal/indexer/workspace_resolve.go | 21 ++++++-- 4 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 internal/indexer/python_save_resolve_work_test.go diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index 6c1a3b3a2..a13fa818e 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -555,6 +555,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( crossRepoFiles = appendUniqueSorted(crossRepoFiles, result.DerivedInvalidation.Files...) } idx := mi.GetIndexer(repoPrefix) + var masterResolvedFiles []string if needed && len(files) > 0 && idx != nil { resolvedFiles := idx.runIncrementalResolutionCatchup(files, batch, func(frontier []string) { if idx.incrementalResolveFilesHook != nil { @@ -563,12 +564,14 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } mi.runMasterResolveFilesWithEvidence(frontier, false, idx.resolver) }) + masterResolvedFiles = resolvedFiles crossRepoFiles = appendUniqueSorted(crossRepoFiles, resolvedFiles...) if receipt != nil && receipt.Complete { mi.runMasterResolveNames(vanishedReceiptNames(mi.graph, receipt)) } } else if needed && len(files) > 0 { mi.runMasterResolveFiles(files, false) + masterResolvedFiles = files if receipt != nil && receipt.Complete { mi.runMasterResolveNames(vanishedReceiptNames(mi.graph, receipt)) } @@ -604,7 +607,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( idx.observeIncrementalCatchup("cross_repo", crossRepoFiles) } if len(crossRepoFiles) > 0 { - mi.runCrossRepoResolveFiles(crossRepoFiles, cargoRetargetRepo) + mi.runCrossRepoResolveMutationFrontiers(crossRepoFiles, crossRepoFiles, crossRepoFiles, masterResolvedFiles, cargoRetargetRepo) } else { mi.runCrossRepoResolve(false) } diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 824b84acd..56bc7ec09 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -960,8 +960,10 @@ func (mi *MultiIndexer) resolveDeferredMutations(receipt *graph.MutationReceipt, return deferredResolveSkipped, true } + var masterResolvedFiles []string if receipt.ResolutionRelevant { mi.runMasterResolveFiles(resolutionFiles, false) + masterResolvedFiles = resolutionFiles // Evicted definitions' pending references live outside the file // frontier (their name is no longer declared in any frontier // file); rebind them by the names the receipt recorded that no @@ -971,7 +973,7 @@ func (mi *MultiIndexer) resolveDeferredMutations(receipt *graph.MutationReceipt, // Resolve only files that can create or bind unresolved edges. Resolved // edge sources still materialise their cross_repo_* generation without // repeating the substantially larger name-resolution frontier. - mi.runCrossRepoResolveMutationFrontiers(resolutionFiles, receipt.ChangedFiles, receipt.DefinitionFiles) + mi.runCrossRepoResolveMutationFrontiers(resolutionFiles, receipt.ChangedFiles, receipt.DefinitionFiles, masterResolvedFiles) return deferredResolveExact, true } if !fallbackNeeded { diff --git a/internal/indexer/python_save_resolve_work_test.go b/internal/indexer/python_save_resolve_work_test.go new file mode 100644 index 000000000..80d94bef3 --- /dev/null +++ b/internal/indexer/python_save_resolve_work_test.go @@ -0,0 +1,52 @@ +package indexer + +import ( + "context" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + "path/filepath" + "testing" +) + +func TestPythonSaveRunsOneIncomingResolve(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"receipt", "point"} { + t.Run(backend+"/"+route, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'dist'\n") + writeWorkspaceFile(t, provider, "pkg/__init__.py", "") + writeWorkspaceFile(t, provider, "pkg/image.py", "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['dist']\n") + source := "from pkg.image import f\ndef run(): return f()\n" + writeWorkspaceFile(t, consumer, "app.py", source) + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + core, logs := observer.New(zap.InfoLevel) + mi.logger = zap.New(core) + save := func() { + t.Helper() + if route == "point" { + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "app.py"))) + } else { + _, err := mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "app.py")}) + require.NoError(t, err) + } + require.Equal(t, []string{"provider/pkg/image.py::f"}, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + } + writeWorkspaceFile(t, consumer, "app.py", source+"def added(): pass\n") + save() + require.Equal(t, 1, logs.FilterMessage("DEFERRED-TIMING master.ResolveFilesAndIncoming").Len()) + logs.TakeAll() + save() + require.LessOrEqual(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveFilesAndIncoming").Len(), 1) + }) + } + } +} diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 140165415..47ecd846e 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -551,10 +551,10 @@ func (mi *MultiIndexer) runCrossRepoResolveContext(ctx context.Context, reconcil // and global topology-writer gate, so the graph cannot expose a half-resolved // batch between same-repo, semantic, and cross-repository phases. func (mi *MultiIndexer) runCrossRepoResolveFiles(filePaths []string, cargoRetargetRepo ...string) { - mi.runCrossRepoResolveMutationFrontiers(filePaths, filePaths, filePaths, cargoRetargetRepo...) + mi.runCrossRepoResolveMutationFrontiers(filePaths, filePaths, filePaths, nil, cargoRetargetRepo...) } -func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles []string, cargoRetargetRepo ...string) { +func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles, masterResolvedFiles []string, cargoRetargetRepo ...string) { if len(resolutionFiles) == 0 && len(edgeSourceFiles) == 0 && len(definitionFiles) == 0 { return } @@ -571,8 +571,23 @@ func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, ed } // Match cold ordering: tracked-provider refusal must remain authoritative // after fallback has handled genuinely external calls. + prepared := make(map[string]bool, len(masterResolvedFiles)) + for _, file := range masterResolvedFiles { + prepared[file] = true + } + var outgoingFiles []string + for _, file := range bridgeFiles { + if !prepared[file] { + outgoingFiles = append(outgoingFiles, file) + } + } + if len(outgoingFiles) > 0 { + if master := mi.newMasterResolver(false); master != nil { + master.ResolveFilesOutgoing(outgoingFiles) + mi.reconcileRetargetedTestCalls(master.TakeRetargetedTestCallFiles()) + } + } if len(bridgeFiles) > 0 { - mi.runMasterResolveFiles(bridgeFiles, false) mi.synthesizeDemotedGoModuleCalls(bridgeFiles) } _, admission := cr.ResolveMutationFrontiersBounded(resolutionFiles, edgeSourceFiles, definitionFiles) From 7ab22ce332608a2cbcb030cc7af14c8ec120012f Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 14:25:20 -0700 Subject: [PATCH 094/111] perf(resolver): run Go import recovery only for Go inputs Unrelated non-Go saves no longer seed imports or retry module dependencies. Go source saves and Go build-list inputs (go.mod, go.sum, go.work, go.work.sum, vendor/modules.txt) still seed once per revision and retry. --- internal/resolver/cross_repo_incremental.go | 51 +++++++++++++++---- .../resolver/go_module_import_seed_test.go | 47 +++++++++++++++++ 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index a2928a5b1..e71425f6e 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -2,6 +2,7 @@ package resolver import ( "context" + "path" "strings" "go.uber.org/zap" @@ -66,10 +67,35 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed return stats, admission } + providerRepos := make(map[string]bool) + goMutation := false + goManifestMutation := false + for _, files := range [][]string{resolutionFiles, edgeSourceFiles} { + for _, file := range files { + if goModuleManifestPath(file) { + goManifestMutation = true + goMutation = true + } + if strings.HasSuffix(file, ".go") { + goMutation = true + } + } + } + for _, file := range definitionFiles { + if goModuleManifestPath(file) { + goManifestMutation = true + goMutation = true + } + repo := graph.RepoPrefixOfID(file) + if strings.HasSuffix(file, ".go") && !strings.HasSuffix(file, "_test.go") && cr.goModules[repo] != "" { + providerRepos[repo] = true + goMutation = true + } + } importBudget := newGoModuleImportRecoveryBudget() - // Provider recovery needs the lazy seed before the pass lock is taken. + // Provider and build-list recovery need the seed before the pass lock. // Ordinary consumer frontiers keep local resolution ahead of the bridge. - if len(definitionFiles) > 0 { + if len(providerRepos) > 0 || goManifestMutation { if err := cr.seedGoModuleImportRegistry(context.Background()); err != nil { cr.logger.Warn("cross-repo resolve: seed Go module imports deferred", zap.Error(err)) } @@ -173,20 +199,14 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed } // A newly tracked module also makes old dep/stdlib terminals eligible. // This indexed sweep runs at most once per module/config revision. - if !admission.Refused { + if goMutation && !admission.Refused { if err := cr.retryGoModuleDeps(context.Background(), stats, false); err != nil { cr.logger.Error("cross-repo resolve: retry Go module terminals", zap.Error(err)) } } // Import recovery has its own ceiling: a popular provider must not starve // incoming calls or the revision-keyed call-terminal retry above. - repos := make(map[string]bool) - for _, file := range definitionFiles { - repo := graph.RepoPrefixOfID(file) - if cr.goModules[repo] == "" || repos[repo] { - continue - } - repos[repo] = true + for repo := range providerRepos { ids := cr.goModuleImportIDs(repo) if len(ids) > importBudget.Remaining() { cr.goModuleImportsDeferred += len(ids) @@ -239,3 +259,14 @@ func DetectCrossRepoEdgesForMutationFiles(g graph.Store, edgeSourceFiles, defini } return materializeCrossRepoCandidates(g, crossRepoCandidatesForMutationFiles(g, edgeSourceFiles, definitionFiles)) } + +// Keep the Go build-list inputs aligned with the indexer's generation input +// boundary. Source-only suffix checks miss workspace and vendor changes. +func goModuleManifestPath(file string) bool { + file = strings.ReplaceAll(file, "\\", "/") + switch path.Base(file) { + case "go.mod", "go.sum", "go.work", "go.work.sum": + return true + } + return file == "vendor/modules.txt" || strings.HasSuffix(file, "/vendor/modules.txt") +} diff --git a/internal/resolver/go_module_import_seed_test.go b/internal/resolver/go_module_import_seed_test.go index 152748e3c..2cab45954 100644 --- a/internal/resolver/go_module_import_seed_test.go +++ b/internal/resolver/go_module_import_seed_test.go @@ -221,3 +221,50 @@ func TestGoModuleImportRecoveryBudgetRefusal(t *testing.T) { } } } + +func TestGoModuleImportUnrelatedMutationSkipsRecovery(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, file := range []string{"other/main.py", "other/main.ts", "provider/README.md", "provider/lib_test.go"} { + t.Run(backend+"/"+file, func(t *testing.T) { + g := &goImportCountingStore{Store: goImportReviewStore(t, backend)} + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: graph.RepoPrefixOfID(file)}) + state := &GoModuleDepRetry{} + for i := 0; i < 2; i++ { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"provider": "example.org/lib"}) + cr.SetGoModuleDepRetry(state, 1) + packageView := map[string]graph.FileNodeIdentity{"provider/lib.go": {ID: "provider/lib.go", FilePath: "provider/lib.go", RepoPrefix: "provider"}} + cr.goModuleImportState().packages["provider"] = packageView + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{file, file}) + require.False(t, admission.Refused) + require.Zero(t, g.importScans, "non-production-Go mutations must not seed the import registry") + require.Equal(t, packageView, state.packages["provider"], "unrelated saves must not invalidate provider projections") + } + }) + } + } +} + +func TestGoModuleManifestFrontiersRunRetry(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, file := range []string{"provider/go.mod", "provider/go.sum", "provider/go.work", "provider/go.work.sum", "provider/vendor/modules.txt", "provider/nested/vendor/modules.txt"} { + for route := range 3 { + t.Run(fmt.Sprintf("%s/%s/frontier_%d", backend, file, route), func(t *testing.T) { + g := &goImportCountingStore{Store: goImportReviewStore(t, backend)} + state := &GoModuleDepRetry{} + frontiers := [3][]string{} + frontiers[route] = []string{file, file} + for range 2 { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"provider": "example.org/lib"}) + cr.SetGoModuleDepRetry(state, 1) + _, admission := cr.ResolveMutationFrontiersBounded(frontiers[0], frontiers[1], frontiers[2]) + require.False(t, admission.Refused) + require.True(t, state.complete, "Go build-list input must enter revision retry") + } + require.Equal(t, 1, g.importScans, "manifest retry must seed imports once for revision recovery") + }) + } + } + } +} From 0d22d1c1772a30d1f2439ad27fdca909ee345ddf Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 14:25:31 -0700 Subject: [PATCH 095/111] perf(indexer): keep Python and Cargo manifest views across passes Snapshots check size and mtime with the two-second racy rule and take the stamp before the first stat. File inventory, roots, config, manifests and glob directories invalidate them, and an unreadable directory marks the snapshot unknown. Stable passes read no manifests. --- internal/indexer/bridge_manifest_snapshot.go | 248 +++++++++++++++++ .../indexer/bridge_manifest_snapshot_test.go | 257 ++++++++++++++++++ internal/indexer/cargo_crate.go | 188 +++++++------ internal/indexer/cold_manifest_census.go | 6 + internal/indexer/incremental_batch.go | 6 + internal/indexer/indexer.go | 5 + internal/indexer/multi.go | 3 + internal/indexer/python_tracked_import.go | 35 ++- internal/indexer/workspace_resolve.go | 2 +- 9 files changed, 638 insertions(+), 112 deletions(-) create mode 100644 internal/indexer/bridge_manifest_snapshot.go create mode 100644 internal/indexer/bridge_manifest_snapshot_test.go diff --git a/internal/indexer/bridge_manifest_snapshot.go b/internal/indexer/bridge_manifest_snapshot.go new file mode 100644 index 000000000..df4c8ba1d --- /dev/null +++ b/internal/indexer/bridge_manifest_snapshot.go @@ -0,0 +1,248 @@ +package indexer + +import ( + "maps" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/zzet/gortex/internal/resolver" +) + +var readBridgeManifestDir = os.ReadDir + +// A file-set revision changes only when accepted files are added, removed, or +// replaced as a full inventory. Ordinary symbol/edge edits keep local tops valid. +type bridgeManifestInventory struct { + indexer *Indexer + revision uint64 + workspace string +} + +type bridgeManifestStamp struct { + size, modified int64 + kind os.FileMode + exists, valid bool +} + +func statBridgeManifest(file string) bridgeManifestStamp { + info, err := os.Stat(file) + if os.IsNotExist(err) { + return bridgeManifestStamp{valid: true} + } + if err != nil { + return bridgeManifestStamp{} + } + return bridgeManifestStamp{size: info.Size(), modified: info.ModTime().UnixNano(), kind: info.Mode().Type(), exists: true, valid: true} +} + +type bridgeManifestSnapshot struct { + revision uint64 + roots map[string]string + inventory map[string]bridgeManifestInventory + trees map[string]manifestTree + stampedAt time.Time + reusable bool + // Cargo discovers nearest manifest boundaries on demand after publication. + mu sync.Mutex + files map[string]bridgeManifestStamp + unknown bool + python *resolver.PythonPackageLookup + pythonBuilt bool + pythonEntries map[string]*pythonManifestEntry + cargo resolver.CargoCrateLookup +} + +func (mi *MultiIndexer) newBridgeManifestSnapshot() *bridgeManifestSnapshot { + s := &bridgeManifestSnapshot{ + roots: map[string]string{}, inventory: map[string]bridgeManifestInventory{}, + trees: map[string]manifestTree{}, files: map[string]bridgeManifestStamp{}, + stampedAt: time.Now(), reusable: true, + } + if mi.configMgr != nil { + s.revision = mi.configMgr.Revision() + } + for repo, meta := range mi.AllMetadata() { + if meta == nil || meta.RootPath == "" { + continue + } + s.roots[repo] = meta.RootPath + idx := mi.GetIndexer(repo) + if idx == nil { + // A hand-built registry cannot certify graph file inventory changes. + s.reusable = false + } else { + s.inventory[repo] = bridgeManifestInventory{idx, idx.fileSetRevision.Load(), idx.WorkspaceID()} + } + tree := mi.manifestTreeFor(repo, meta.RootPath) + if tree.sourced() { + s.reusable = false + } + s.trees[repo] = &bridgeManifestTree{manifestTree: tree, snapshot: s} + } + return s +} + +func (s *bridgeManifestSnapshot) matches(next *bridgeManifestSnapshot) bool { + if s == nil || s.revision != next.revision || !maps.Equal(s.roots, next.roots) || !maps.Equal(s.inventory, next.inventory) || !next.reusable { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if !s.reusable || s.unknown { + return false + } + for file, stamp := range s.files { + // A recently written file can change again without changing size/mtime on + // a coarse filesystem clock. Never certify such an observation as stable. + if !stamp.valid || stamp.exists && !time.Unix(0, stamp.modified).Before(s.stampedAt.Add(-manifestRacyWindow)) || statBridgeManifest(file) != stamp { + return false + } + } + return true +} + +func (s *bridgeManifestSnapshot) watch(file string) bridgeManifestStamp { + s.mu.Lock() + defer s.mu.Unlock() + stamp, seen := s.files[file] + if !seen { + stamp = statBridgeManifest(file) + s.files[file] = stamp + if !stamp.valid { + s.unknown = true + } + } + return stamp +} + +func rejectBridgeManifest(tree manifestTree) { + if watched, ok := tree.(*bridgeManifestTree); ok { + watched.snapshot.mu.Lock() + watched.snapshot.unknown = true + watched.snapshot.mu.Unlock() + } +} + +// Watch the same files/glob directory prefixes the native builders use. This +// records stat evidence only, never file bytes or directory content hashes. +type bridgeManifestTree struct { + manifestTree + snapshot *bridgeManifestSnapshot +} + +func (t *bridgeManifestTree) watch(rel string) bridgeManifestStamp { + if t.sourced() { + return bridgeManifestStamp{} + } + return t.snapshot.watch(filepath.Join(t.root(), filepath.FromSlash(rel))) +} +func (t *bridgeManifestTree) readFile(rel string) ([]byte, bool) { + stamp := t.watch(rel) + src, ok := t.manifestTree.readFile(rel) + if !ok && stamp.exists { + rejectBridgeManifest(t) + } + return src, ok +} +func (t *bridgeManifestTree) isFile(rel string) bool { + t.watch(rel) + return t.manifestTree.isFile(rel) +} +func (t *bridgeManifestTree) watchGlob(pattern string) { + if t.sourced() { + return + } + watchDir := func(rel string) { + stamp := t.watch(rel) + if stamp.exists && stamp.kind.IsDir() { + if _, err := readBridgeManifestDir(filepath.Join(t.root(), filepath.FromSlash(rel))); err != nil { + rejectBridgeManifest(t) + } + } + } + watchDir(".") + parts := strings.Split(filepath.ToSlash(pattern), "/") + for depth := 1; depth <= len(parts); depth++ { + prefix := strings.Join(parts[:depth], "/") + if !strings.ContainsAny(prefix, `*?[\`) { + watchDir(prefix) + } + for _, dir := range t.manifestTree.matchDirs(prefix) { + watchDir(dir) + } + } +} +func (t *bridgeManifestTree) matchFiles(pattern string) []string { + t.watchGlob(pattern) + return t.manifestTree.matchFiles(pattern) +} +func (t *bridgeManifestTree) matchDirs(pattern string) []string { + t.watchGlob(pattern) + return t.manifestTree.matchDirs(pattern) +} + +func (mi *MultiIndexer) pythonManifestSnapshot() *bridgeManifestSnapshot { + next := mi.newBridgeManifestSnapshot() + mi.bridgeManifestMu.Lock() + defer mi.bridgeManifestMu.Unlock() + if mi.pythonPackageCache.matches(next) { + return mi.pythonPackageCache + } + next.pythonEntries = readPythonPackageEntries(next.trees) + mi.pythonPackageCache = next + return next +} + +func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { + snapshot := mi.pythonManifestSnapshot() + mi.bridgeManifestMu.Lock() + defer mi.bridgeManifestMu.Unlock() + if !snapshot.pythonBuilt { + snapshot.python = mi.buildPythonPackageLookup(snapshot.trees, snapshot.pythonEntries) + snapshot.pythonBuilt = true + } + return snapshot.python +} + +func (mi *MultiIndexer) cargoManifestSnapshot() *bridgeManifestSnapshot { + next := mi.newBridgeManifestSnapshot() + mi.bridgeManifestMu.Lock() + defer mi.bridgeManifestMu.Unlock() + if mi.cargoPackageCache.matches(next) { + return mi.cargoPackageCache + } + next.cargo = mi.buildCargoCrateLookup(next.trees) + mi.cargoPackageCache = next + return next +} + +func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { + // Direct recovery callers need to know whether a bridge is possible before + // scanning imports. Keep their cheap root admission and defer manifest reads. + roots := 0 + for repo, meta := range mi.AllMetadata() { + if meta != nil && meta.RootPath != "" && mi.manifestTreeFor(repo, meta.RootPath).isFile("Cargo.toml") { + roots++ + } + } + if roots < 2 { + return nil + } + return mi.lazyCargoCrateLookup() +} + +// Like npm, resolver construction does no Cargo work before the first use. +func (mi *MultiIndexer) lazyCargoCrateLookup() resolver.CargoCrateLookup { + var once sync.Once + var lookup resolver.CargoCrateLookup + return func(file, name string) []resolver.CargoCrate { + once.Do(func() { lookup = mi.cargoManifestSnapshot().cargo }) + if lookup == nil { + return nil + } + return lookup(file, name) + } +} diff --git a/internal/indexer/bridge_manifest_snapshot_test.go b/internal/indexer/bridge_manifest_snapshot_test.go new file mode 100644 index 000000000..6487e112e --- /dev/null +++ b/internal/indexer/bridge_manifest_snapshot_test.go @@ -0,0 +1,257 @@ +package indexer + +import ( + "io/fs" + "iter" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "go.uber.org/zap" +) + +type bridgeSnapshotCountStore struct { + graph.Store + files, nodes int +} + +func (s *bridgeSnapshotCountStore) GetNode(id string) *graph.Node { + s.nodes++ + return s.Store.GetNode(id) +} +func (s *bridgeSnapshotCountStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { + s.files++ + return graph.FileNodeIdentitiesSeq(s.Store, repos) +} + +func backdateBridgeFixture(t *testing.T, root string) { + t.Helper() + old := time.Now().Add(-time.Hour) + require.NoError(t, filepath.WalkDir(root, func(file string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + return os.Chtimes(file, old, old) + })) +} + +func TestPythonManifestSnapshotReusesParsedLookup(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + writeWorkspaceFile(t, consumer, "app.py", "import PIL\n") + g := &bridgeSnapshotCountStore{Store: graph.New()} + for repo, files := range map[string][]string{"provider": {"src/PIL/__init__.py"}, "consumer": {"app.py"}} { + for _, file := range files { + g.AddNode(&graph.Node{ID: repo + "/" + file, FilePath: repo + "/" + file, Kind: graph.KindFile, RepoPrefix: repo}) + } + } + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{}, indexers: map[string]*Indexer{}} + for repo, root := range map[string]string{"provider": provider, "consumer": consumer} { + idx := &Indexer{graph: g, repoPrefix: repo, logger: zap.NewNop()} + idx.SetRootPath(root) + initial := map[string]int64{} + if repo == "consumer" { + initial["app.py"] = 1 + } + idx.SetFileMtimes(initial) + mi.indexers[repo] = idx + mi.repos[repo] = &RepoMetadata{RootPath: root} + backdateBridgeFixture(t, root) + } + priorRead := readDiskFile + reads := 0 + readDiskFile = func(file string) ([]byte, bool) { reads++; return priorRead(file) } + t.Cleanup(func() { readDiskFile = priorRead }) + first := mi.pythonPackageLookup() + require.NotNil(t, first) + require.Len(t, first.Providers("consumer", "PIL"), 1) + reads, g.files, g.nodes = 0, 0, 0 + second := mi.pythonPackageLookup() + require.NotNil(t, second) + require.Len(t, second.Providers("consumer", "PIL"), 1) + require.Zero(t, reads, "unchanged pass must reuse parsed manifests") + require.Zero(t, g.nodes, "unchanged provider layout must not be probed") + require.Zero(t, g.files, "unchanged consumer inventory must not be scanned again") + // A structural edit that preserves the file set keeps the snapshot. + mi.indexers["consumer"].recordFileMtimeValue("app.py", 1) + second = mi.pythonPackageLookup() + require.Len(t, second.Providers("consumer", "PIL"), 1) + // A new local namespace blocks the foreign package on the next pass. + g.AddNode(&graph.Node{ID: "consumer/src/PIL/local.py", FilePath: "consumer/src/PIL/local.py", Kind: graph.KindFile, RepoPrefix: "consumer"}) + mi.indexers["consumer"].recordFileMtimeValue("src/PIL/local.py", 1) + require.Empty(t, mi.pythonPackageLookup().Providers("consumer", "PIL")) +} + +func TestCargoManifestSnapshotReusesParsedLookup(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'pillow'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\npillow = '1'\n") + mi := &MultiIndexer{graph: graph.New(), repos: map[string]*RepoMetadata{}, indexers: map[string]*Indexer{}} + for repo, root := range map[string]string{"provider": provider, "consumer": consumer} { + idx := &Indexer{} + idx.SetRootPath(root) + mi.indexers[repo] = idx + mi.repos[repo] = &RepoMetadata{RootPath: root} + backdateBridgeFixture(t, root) + } + priorRead := readDiskFile + reads := 0 + readDiskFile = func(file string) ([]byte, bool) { reads++; return priorRead(file) } + t.Cleanup(func() { readDiskFile = priorRead }) + first := mi.lazyCargoCrateLookup() + require.NotNil(t, first) + require.Zero(t, reads) + require.Len(t, first("consumer/src/lib.rs", "pillow"), 1) + reads = 0 + second := mi.cargoCrateLookup() + require.NotNil(t, second) + require.Len(t, second("consumer/src/lib.rs", "pillow"), 1) + require.Zero(t, reads, "unchanged Cargo lookup must reuse parsed manifests") + // A same-size dependency rename must invalidate the snapshot. + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\nmissing = '1'\n") + require.Empty(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow")) + // Restoring the manifest restores the dependency without a restart. + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\npillow = '1'\n") + require.Len(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow"), 1) +} + +func TestBridgeManifestSnapshotRacyBoundary(t *testing.T) { + stampTime := time.Unix(1700000000, 0) + for _, tc := range []struct { + name string + offset time.Duration + reusable bool + }{ + {"old", -time.Second, true}, {"boundary", 0, false}, {"recent", time.Second, false}, + } { + t.Run(tc.name, func(t *testing.T) { + file := filepath.Join(t.TempDir(), "pyproject.toml") + require.NoError(t, os.WriteFile(file, []byte("aaaa"), 0600)) + modified := stampTime.Add(-manifestRacyWindow).Add(tc.offset) + require.NoError(t, os.Chtimes(file, modified, modified)) + s := &bridgeManifestSnapshot{stampedAt: stampTime, reusable: true, files: map[string]bridgeManifestStamp{}} + s.watch(file) + // Same size/mtime can hide a second write on a coarse clock. + require.NoError(t, os.WriteFile(file, []byte("bbbb"), 0600)) + require.NoError(t, os.Chtimes(file, modified, modified)) + require.Equal(t, tc.reusable, s.matches(s)) + s.unknown = true + require.False(t, s.matches(s), "a failed read must not be reused") + }) + } +} + +func TestCargoManifestSnapshotWorkspaceAndBoundaryChanges(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[workspace]\nmembers = ['crates/*']\n") + require.NoError(t, os.MkdirAll(filepath.Join(provider, "crates/empty"), 0700)) + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\npillow = '1'\n") + mi := &MultiIndexer{repos: map[string]*RepoMetadata{}, indexers: map[string]*Indexer{}} + for repo, root := range map[string]string{"provider": provider, "consumer": consumer} { + idx := &Indexer{} + idx.SetRootPath(root) + mi.indexers[repo] = idx + mi.repos[repo] = &RepoMetadata{RootPath: root} + backdateBridgeFixture(t, root) + } + require.Empty(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow")) + writeWorkspaceFile(t, provider, "crates/empty/Cargo.toml", "[package]\nname = 'pillow'\n") + require.Len(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow"), 1) + backdateBridgeFixture(t, provider) + backdateBridgeFixture(t, consumer) + require.Len(t, mi.cargoCrateLookup()("consumer/tools/child/src/lib.rs", "pillow"), 1) + // A newly introduced nonmember boundary blocks inheritance from the root. + writeWorkspaceFile(t, consumer, "tools/child/Cargo.toml", "[package]\nname = 'child'\n") + require.Empty(t, mi.cargoCrateLookup()("consumer/tools/child/src/lib.rs", "pillow")) + require.NoError(t, os.Remove(filepath.Join(consumer, "tools/child/Cargo.toml"))) + require.Len(t, mi.cargoCrateLookup()("consumer/tools/child/src/lib.rs", "pillow"), 1) +} + +func TestBridgeManifestSnapshotInventoryAndRoots(t *testing.T) { + idx := &Indexer{graph: graph.New(), repoPrefix: "repo", logger: zap.NewNop()} + idx.SetRootPath(t.TempDir()) + mi := &MultiIndexer{repos: map[string]*RepoMetadata{"repo": {RootPath: idx.RootPath()}}, indexers: map[string]*Indexer{"repo": idx}} + idx.SetFileMtimes(map[string]int64{"a.py": 1}) + s := mi.newBridgeManifestSnapshot() + idx.recordFileMtimeValue("a.py", 2) + require.True(t, s.matches(mi.newBridgeManifestSnapshot()), "ordinary edits preserve file inventory") + idx.recordFileMtimeValue("b.py", 2) + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "additions change inventory") + s = mi.newBridgeManifestSnapshot() + idx.invalidateColdFileMtimes([]string{"b.py"}) + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "removals change inventory") + s = mi.newBridgeManifestSnapshot() + idx.SetFileMtimes(map[string]int64{"a.py": 2}) + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "full rebuild invalidates provider identities") + s = mi.newBridgeManifestSnapshot() + mi.indexers["repo"] = &Indexer{} + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "retrack replaces the owner") + s = mi.newBridgeManifestSnapshot() + mi.repos["repo"] = &RepoMetadata{RootPath: t.TempDir()} + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "a new root is not the old snapshot") +} + +func TestBridgeManifestSnapshotSourceAuthority(t *testing.T) { + provider, consumer, committed := t.TempDir(), t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "PIL/__init__.py", "") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\npillow = '1'\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + writeWorkspaceFile(t, committed, "Cargo.toml", "[package]\nname = 'app'\n") + writeWorkspaceFile(t, committed, "pyproject.toml", "[project]\nname = 'app'\n") + g := graph.New() + g.AddNode(&graph.Node{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", RepoPrefix: "provider", Kind: graph.KindFile}) + idx := &Indexer{} + idx.SetRootPath(consumer) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}, indexers: map[string]*Indexer{"consumer": idx}} + require.Len(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow"), 1) + require.NotNil(t, mi.pythonPackageLookup()) + idx.SetContentSource(snapshotOf(t, committed)) + require.Empty(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow")) + require.Nil(t, mi.pythonPackageLookup(), "dirty dependency must not escape the installed source") + idx.SetContentSource(nil) + require.Len(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow"), 1) + require.NotNil(t, mi.pythonPackageLookup()) +} + +func TestPythonManifestSnapshotFailedGlobIsUnknown(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname='library'\n") + writeWorkspaceFile(t, consumer, "requirements-private.txt", "library\n") + g := &pythonLookupScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeImports, To: "external::pkg.extra"}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + priorRead := readDiskFile + readDiskFile = func(file string) ([]byte, bool) { + if filepath.Base(file) == "requirements-private.txt" { + return nil, false + } + return priorRead(file) + } + t.Cleanup(func() { readDiskFile = priorRead }) + // stat can succeed even though enumeration fails; mirror Glob's empty result. + priorDir := readBridgeManifestDir + readBridgeManifestDir = func(dir string) ([]os.DirEntry, error) { + if dir == consumer { + return nil, fs.ErrPermission + } + return priorDir(dir) + } + t.Cleanup(func() { readBridgeManifestDir = priorDir }) + snapshot := mi.newBridgeManifestSnapshot() + snapshot.trees["consumer"] = &bridgeManifestTree{manifestTree: failingGlobManifestTree{manifestTree: newDiskManifestTree(consumer)}, snapshot: snapshot} + snapshot.pythonEntries = readPythonPackageEntries(snapshot.trees) + require.True(t, snapshot.hasUnknownManifest(), "failed glob cannot certify an absent declaration") + require.True(t, snapshot.declaresPythonBridge()) +} + +type failingGlobManifestTree struct{ manifestTree } + +func (t failingGlobManifestTree) matchFiles(string) []string { return nil } +func (t failingGlobManifestTree) matchDirs(string) []string { return nil } diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go index 41119c893..a6d096fcc 100644 --- a/internal/indexer/cargo_crate.go +++ b/internal/indexer/cargo_crate.go @@ -22,6 +22,7 @@ func readCargoManifest(tree manifestTree, dir string) map[string]any { } var manifest map[string]any if toml.Unmarshal(src, &manifest) != nil { + rejectBridgeManifest(tree) return nil } return manifest @@ -29,19 +30,16 @@ func readCargoManifest(tree manifestTree, dir string) map[string]any { // cargoCrateLookup snapshots root/member manifests once per cross-repo pass. // Only explicit workspace member globs are expanded; Cargo is never executed. -func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { +func (mi *MultiIndexer) buildCargoCrateLookup(trees map[string]manifestTree) resolver.CargoCrateLookup { type crate struct { dir, name, root string deps map[string]string imports map[string][]resolver.CargoCrate } roots := map[string]manifestTree{} - for prefix, meta := range mi.AllMetadata() { - if meta != nil && meta.RootPath != "" { - tree := mi.manifestTreeFor(prefix, meta.RootPath) - if tree.isFile("Cargo.toml") { - roots[prefix] = tree - } + for prefix, tree := range trees { + if tree.isFile("Cargo.toml") { + roots[prefix] = tree } } if len(roots) < 2 { @@ -51,104 +49,102 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { packages := map[string][]*crate{} ownerCache := map[string]*crate{} var ownerMu sync.Mutex - var once sync.Once - return func(file, name string) []resolver.CargoCrate { - once.Do(func() { - for prefix, tree := range roots { - manifest := readCargoManifest(tree, ".") - src, _ := tree.readFile("Cargo.toml") - members, exclude := modules.ParseCargoWorkspace(src) - dirs := []string{"."} - skip := map[string]bool{".": true} - for _, pattern := range exclude { - for _, dir := range tree.matchDirs(pattern) { - skip[dir] = true - } + + for prefix, tree := range roots { + manifest := readCargoManifest(tree, ".") + src, _ := tree.readFile("Cargo.toml") + members, exclude := modules.ParseCargoWorkspace(src) + dirs := []string{"."} + skip := map[string]bool{".": true} + for _, pattern := range exclude { + for _, dir := range tree.matchDirs(pattern) { + skip[dir] = true + } + } + for _, pattern := range members { + for _, dir := range tree.matchDirs(pattern) { + if !path.IsAbs(dir) && !hasDotDotSegment(dir) && !skip[dir] && tree.isFile(path.Join(dir, "Cargo.toml")) { + dirs = append(dirs, dir) + skip[dir] = true } - for _, pattern := range members { - for _, dir := range tree.matchDirs(pattern) { - if !path.IsAbs(dir) && !hasDotDotSegment(dir) && !skip[dir] && tree.isFile(path.Join(dir, "Cargo.toml")) { - dirs = append(dirs, dir) - skip[dir] = true - } - } + } + } + inherited := cargoTable(cargoTable(manifest, "workspace"), "dependencies") + for _, dir := range dirs { + m := manifest + if dir != "." { + m = readCargoManifest(tree, dir) + } + if m == nil { + continue + } + pkg, _ := cargoTable(m, "package")["name"].(string) + if pkg == "" { + continue + } + lib := cargoTable(m, "lib") + crateName, _ := lib["name"].(string) + if crateName == "" { + crateName = strings.ReplaceAll(pkg, "-", "_") + } + libPath, _ := lib["path"].(string) + if libPath == "" { + libPath = "src/lib.rs" + } + if path.IsAbs(libPath) || hasDotDotSegment(libPath) { + continue + } + deps := map[string]string{} + tables := []map[string]any{m} + for _, target := range cargoTable(m, "target") { + if table, ok := target.(map[string]any); ok { + tables = append(tables, table) } - inherited := cargoTable(cargoTable(manifest, "workspace"), "dependencies") - for _, dir := range dirs { - m := manifest - if dir != "." { - m = readCargoManifest(tree, dir) - } - if m == nil { - continue - } - pkg, _ := cargoTable(m, "package")["name"].(string) - if pkg == "" { - continue - } - lib := cargoTable(m, "lib") - crateName, _ := lib["name"].(string) - if crateName == "" { - crateName = strings.ReplaceAll(pkg, "-", "_") - } - libPath, _ := lib["path"].(string) - if libPath == "" { - libPath = "src/lib.rs" - } - if path.IsAbs(libPath) || hasDotDotSegment(libPath) { - continue - } - deps := map[string]string{} - tables := []map[string]any{m} - for _, target := range cargoTable(m, "target") { - if table, ok := target.(map[string]any); ok { - tables = append(tables, table) + } + for _, table := range tables { + for _, key := range []string{"dependencies", "dev-dependencies", "build-dependencies"} { + for alias, value := range cargoTable(table, key) { + dep, isTable := value.(map[string]any) + if isTable && dep["workspace"] == true { + value = inherited[alias] + dep, isTable = value.(map[string]any) } - } - for _, table := range tables { - for _, key := range []string{"dependencies", "dev-dependencies", "build-dependencies"} { - for alias, value := range cargoTable(table, key) { - dep, isTable := value.(map[string]any) - if isTable && dep["workspace"] == true { - value = inherited[alias] - dep, isTable = value.(map[string]any) - } - _, isVersion := value.(string) - if !isTable && !isVersion { - continue - } - target := alias - if packageName, ok := dep["package"].(string); ok { - target = packageName - } - // Conflicting target-specific aliases cannot identify one package. - if old, exists := deps[alias]; exists && old != target { - target = "" - } - deps[alias] = target - } + _, isVersion := value.(string) + if !isTable && !isVersion { + continue } - } - c := &crate{dir: path.Join(prefix, dir), name: crateName, root: path.Join(prefix, dir, libPath), deps: deps} - if owners[c.dir] == nil { - owners[c.dir] = c - packages[pkg] = append(packages[pkg], c) + target := alias + if packageName, ok := dep["package"].(string); ok { + target = packageName + } + // Conflicting target-specific aliases cannot identify one package. + if old, exists := deps[alias]; exists && old != target { + target = "" + } + deps[alias] = target } } } - for _, owner := range owners { - owner.imports = map[string][]resolver.CargoCrate{} - for alias, pkg := range owner.deps { - for _, provider := range packages[pkg] { - name := provider.name - if alias != pkg { - name = strings.ReplaceAll(alias, "-", "_") - } - owner.imports[name] = append(owner.imports[name], resolver.CargoCrate{Root: provider.root, Name: provider.name}) - } + c := &crate{dir: path.Join(prefix, dir), name: crateName, root: path.Join(prefix, dir, libPath), deps: deps} + if owners[c.dir] == nil { + owners[c.dir] = c + packages[pkg] = append(packages[pkg], c) + } + } + } + for _, owner := range owners { + owner.imports = map[string][]resolver.CargoCrate{} + for alias, pkg := range owner.deps { + for _, provider := range packages[pkg] { + name := provider.name + if alias != pkg { + name = strings.ReplaceAll(alias, "-", "_") } + owner.imports[name] = append(owner.imports[name], resolver.CargoCrate{Root: provider.root, Name: provider.name}) } - }) + } + } + return func(file, name string) []resolver.CargoCrate { // Cache the nearest manifest boundary, including nonmembers and excludes. // A nested crate must never borrow an ancestor package's dependencies. ownerMu.Lock() diff --git a/internal/indexer/cold_manifest_census.go b/internal/indexer/cold_manifest_census.go index 21b27553e..98d74984d 100644 --- a/internal/indexer/cold_manifest_census.go +++ b/internal/indexer/cold_manifest_census.go @@ -165,6 +165,9 @@ func (idx *Indexer) invalidateColdFileMtimes(paths []string) { idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() for _, path := range paths { + if _, exists := idx.fileMtimes[path]; exists { + idx.fileSetRevision.Add(1) + } delete(idx.fileMtimes, path) } idx.mtimeMu.Unlock() @@ -195,6 +198,9 @@ func (idx *Indexer) publishColdFileMtimes(mtimes map[string]int64, authoritative idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() for path, mtime := range mtimes { + if _, exists := idx.fileMtimes[path]; !exists { + idx.fileSetRevision.Add(1) + } idx.fileMtimes[path] = mtime } idx.mtimeMu.Unlock() diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 474739bf7..1c6c212af 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -1884,6 +1884,9 @@ func (idx *Indexer) recordFileReadVersionsBatched(receipts []fileReadReceipt) (f idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() for path, mtime := range mtimes { + if _, exists := idx.fileMtimes[path]; !exists { + idx.fileSetRevision.Add(1) + } idx.fileMtimes[path] = mtime } idx.mtimeMu.Unlock() @@ -2591,6 +2594,9 @@ func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInva idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() for _, relPath := range deleted { + if _, exists := idx.fileMtimes[relPath]; exists { + idx.fileSetRevision.Add(1) + } delete(idx.fileMtimes, relPath) } idx.mtimeMu.Unlock() diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 5f422a8b8..ac9223f50 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -489,6 +489,7 @@ type Indexer struct { parseErrors []IndexError fileMtimes map[string]int64 fileMtimesShared bool + fileSetRevision atomic.Uint64 // Any failed sidecar write can leave the durable keyset incomplete even // while the immutable in-memory snapshot is current. Receipt paging must // fall back to that snapshot until a full authoritative replace repairs it. @@ -5722,6 +5723,9 @@ func (idx *Indexer) recordFileReadVersion(relPath, absPath string, version fileR func (idx *Indexer) recordFileMtimeValue(relPath string, mtime int64) { idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() + if _, exists := idx.fileMtimes[relPath]; !exists { + idx.fileSetRevision.Add(1) + } idx.fileMtimes[relPath] = mtime idx.mtimeMu.Unlock() if w, ok := idx.graph.(graph.FileMtimeWriter); ok { @@ -7010,6 +7014,7 @@ func (idx *Indexer) pruneDeletedFileMtimes(deleted []string) { func (idx *Indexer) SetFileMtimes(mtimes map[string]int64) { idx.mtimeMu.Lock() defer idx.mtimeMu.Unlock() + idx.fileSetRevision.Add(1) idx.fileMtimes = make(map[string]int64, len(mtimes)) idx.fileMtimesShared = false for k, v := range mtimes { diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index 56bc7ec09..82825c802 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -122,6 +122,9 @@ type MultiIndexer struct { goModuleRetry resolver.GoModuleDepRetry npmPackageMu sync.Mutex npmPackageCache *npmPackageSnapshot + bridgeManifestMu sync.Mutex + pythonPackageCache *bridgeManifestSnapshot + cargoPackageCache *bridgeManifestSnapshot // pendingRepositoryUntracks is guarded by mu. Entries are installed before // a live repo is hidden and removed only after payload, vector, config, and diff --git a/internal/indexer/python_tracked_import.go b/internal/indexer/python_tracked_import.go index a5873b239..08b37e01d 100644 --- a/internal/indexer/python_tracked_import.go +++ b/internal/indexer/python_tracked_import.go @@ -4,6 +4,7 @@ import ( "path" "regexp" "strings" + "sync" "github.com/pelletier/go-toml/v2" "github.com/zzet/gortex/internal/graph" @@ -20,21 +21,17 @@ func normalizePythonDistribution(name string) string { // pythonPackageLookup is built only after a non-stdlib Python import is found. // Providers require indexed root/src modules or regular packages; consumers also // block foreign bindings with their own indexed namespace directories. -func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { - roots := mi.AllMetadata() - if len(roots) < 2 { +type pythonManifestEntry struct { + name string + deps map[string]bool +} + +func readPythonPackageEntries(trees map[string]manifestTree) map[string]*pythonManifestEntry { + if len(trees) < 2 { return nil } - type entry struct { - name string - deps map[string]bool - } - entries := map[string]*entry{} - for repo, meta := range roots { - if meta == nil || meta.RootPath == "" { - continue - } - tree := mi.manifestTreeFor(repo, meta.RootPath) + entries := map[string]*pythonManifestEntry{} + for repo, tree := range trees { src, _ := tree.readFile("pyproject.toml") var manifest struct { Project struct{ Name string } `toml:"project"` @@ -47,6 +44,7 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { } // A malformed manifest supplies no identity or dependencies. if len(src) > 0 && toml.Unmarshal(src, &manifest) != nil { + rejectBridgeManifest(tree) continue } name := manifest.Project.Name @@ -69,8 +67,12 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { for _, dep := range specs { deps[normalizePythonDistribution(dep.Path)] = true } - entries[repo] = &entry{name: normalizePythonDistribution(name), deps: deps} + entries[repo] = &pythonManifestEntry{name: normalizePythonDistribution(name), deps: deps} } + return entries +} + +func (mi *MultiIndexer) buildPythonPackageLookup(trees map[string]manifestTree, entries map[string]*pythonManifestEntry) *resolver.PythonPackageLookup { // Only declared distributions can supply providers. Discover top-level names // from their root/src directory entries, then probe exact indexed node IDs. packages := map[string]map[string][]graph.FileNodeIdentity{} @@ -85,7 +87,7 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { if !declared { continue } - tree := mi.manifestTreeFor(repo, roots[repo].RootPath) + tree := trees[repo] for _, dir := range []string{"", "src"} { files := tree.matchFiles(path.Join(dir, "*.py")) for _, pkg := range tree.matchDirs(path.Join(dir, "*")) { @@ -117,6 +119,7 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { return nil } localTops := map[string]map[string]bool{} + var localMu sync.Mutex return &resolver.PythonPackageLookup{ HasTop: func(top string) bool { return len(packages[top]) > 0 }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { @@ -124,6 +127,8 @@ func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { if consumer == nil || len(packages[top]) == 0 { return nil } + localMu.Lock() + defer localMu.Unlock() local, seen := localTops[caller] if !seen { local = map[string]bool{} diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 47ecd846e..9681d92ec 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -498,7 +498,7 @@ func (mi *MultiIndexer) newCrossRepoResolver() *resolver.CrossRepoResolver { cr.SetNpmAliasResolver(mi.npmAliasResolver()) cr.SetNpmDependencyLookup(mi.npmDependencyLookup()) cr.SetNpmPackageLookup(mi.npmPackageLookup()) - cr.SetCargoCrateLookup(mi.cargoCrateLookup()) + cr.SetCargoCrateLookup(mi.lazyCargoCrateLookup()) if mi.IsMultiRepo() { cr.SetPythonPackageLookup(mi.pythonPackageLookup) } From 3b64f9d101c4983a7df1c179385cb9344e865778 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 14:25:38 -0700 Subject: [PATCH 096/111] perf(indexer): skip added-file import scans without a declared bridge When the manifest snapshots show no tracked Python or Cargo dependency, added files no longer scan every import edge. An unknown snapshot still scans. Gaining a dependency reopens recovery and matches a cold index; workspaces with a declared bridge still scan per added file. --- .../added_file_declaration_gate_test.go | 133 ++++++++++++++++++ internal/indexer/bridge_manifest_snapshot.go | 26 +++- internal/indexer/cargo_crate.go | 11 +- internal/indexer/cargo_import_restub.go | 22 +++ internal/indexer/incremental_watcher_batch.go | 2 +- internal/indexer/python_import_restub.go | 11 +- 6 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 internal/indexer/added_file_declaration_gate_test.go diff --git a/internal/indexer/added_file_declaration_gate_test.go b/internal/indexer/added_file_declaration_gate_test.go new file mode 100644 index 000000000..fd5a24c52 --- /dev/null +++ b/internal/indexer/added_file_declaration_gate_test.go @@ -0,0 +1,133 @@ +package indexer + +import ( + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/parser/languages" + "path/filepath" + "strings" + "testing" +) + +func TestPythonAddedFileDeclarationGate(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'library'\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\nname = 'app'\n") + g := &pythonLookupScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: "external::pkg.extra", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + for range 2 { + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + } + require.Zero(t, g.importScans, "no declaration means no import scan") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['library']\n") + require.Equal(t, []string{"consumer/app.py"}, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + require.Equal(t, 1, g.importScans) + require.Zero(t, g.nodeReads, "admission must not build the provider lookup") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\nname = 'app'\n") + g.importScans = 0 + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/next.py"})) + require.Zero(t, g.importScans) + // Existing but unreadable snapshot evidence must open admission. + prior := readDiskFile + readDiskFile = func(file string) ([]byte, bool) { return nil, false } + t.Cleanup(func() { readDiskFile = prior }) + require.Equal(t, []string{"consumer/app.py"}, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + require.Equal(t, 1, g.importScans) +} + +func TestCargoAddedFileDeclarationGate(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'library'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n") + g := &cargoWarmScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "external::library/extra", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + for range 2 { + require.Empty(t, mi.cargoAddedFileConsumers("provider")) + } + require.Zero(t, g.lightScans) + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\nlibrary = '1'\n") + require.Equal(t, []string{"consumer/src/lib.rs"}, mi.cargoAddedFileConsumers("provider")) + require.Equal(t, 1, g.lightScans) + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n") + g.lightScans = 0 + require.Empty(t, mi.cargoAddedFileConsumers("provider")) + require.Zero(t, g.lightScans) + prior := readDiskFile + readDiskFile = func(file string) ([]byte, bool) { return nil, false } + t.Cleanup(func() { readDiskFile = prior }) + require.Equal(t, []string{"consumer/src/lib.rs"}, mi.cargoAddedFileConsumers("provider")) + require.Equal(t, 1, g.lightScans) +} + +func TestPythonAddedTestPackageDoesNotScan(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'library'\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['library']\n") + g := &pythonLookupScanStore{Store: graph.New()} + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/tests/__init__.py", "provider/src/test/helpers.py"})) + require.Zero(t, g.importScans) +} + +func TestAddedFileAfterDeclarationMatchesCold(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, lang := range []string{"python", "cargo"} { + t.Run(backend+"/"+lang, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + manifest, caller, added := "pyproject.toml", "app.py", "pkg/extra.py" + providerManifest, consumerManifest := "[project]\nname = 'library'\n", "[project]\nname = 'app'\n" + declaration := "dependencies = ['library']\n" + providerSource, consumerSource := "", "from pkg.extra import f\ndef run(): return f()\n" + if lang == "cargo" { + manifest, caller, added = "Cargo.toml", "src/lib.rs", "src/extra.rs" + providerManifest, consumerManifest = "[package]\nname = 'library'\n", "[package]\nname = 'app'\n" + declaration = "[dependencies]\nlibrary = '1'\n" + providerSource, consumerSource = "pub mod extra;\n", "use library::extra::f;\npub fn run() { f(); }\n" + writeWorkspaceFile(t, provider, "src/lib.rs", providerSource) + } else { + writeWorkspaceFile(t, provider, "pkg/__init__.py", providerSource) + } + writeWorkspaceFile(t, provider, manifest, providerManifest) + writeWorkspaceFile(t, consumer, manifest, consumerManifest) + writeWorkspaceFile(t, consumer, caller, consumerSource) + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + mi.registry.Register(languages.NewRustExtractor()) + _, err := mi.IndexAll() + require.NoError(t, err) + for _, target := range revisionCallTargets(g, "consumer/"+caller, graph.EdgeImports) { + require.False(t, strings.HasPrefix(target, "provider/")) + } + // The declaration changes without recreating the MultiIndexer or resolver registry. + writeWorkspaceFile(t, consumer, manifest, consumerManifest+declaration) + if lang == "cargo" { + providerSource = "pub fn f() {}\n" + } else { + providerSource = "def f(): pass\n" + } + writeWorkspaceFile(t, provider, added, providerSource) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, added)}) + require.NoError(t, err) + targets := revisionCallTargets(g, "consumer/"+caller, graph.EdgeImports) + require.Contains(t, targets, "provider/"+added) + cold, coldGraph := newPythonBridgeIdentityIndexer(t, backend, entries) + cold.registry.Register(languages.NewRustExtractor()) + _, err = cold.IndexAll() + require.NoError(t, err) + require.ElementsMatch(t, revisionCallTargets(coldGraph, "consumer/"+caller, graph.EdgeImports), targets) + require.ElementsMatch(t, revisionCallTargets(coldGraph, "consumer/"+caller+"::run", graph.EdgeCalls), revisionCallTargets(g, "consumer/"+caller+"::run", graph.EdgeCalls)) + // Removal closes admission for the next addition. Full/retrack recovery is separate. + writeWorkspaceFile(t, consumer, manifest, consumerManifest) + if lang == "cargo" { + require.Empty(t, mi.cargoAddedFileConsumers("provider")) + } else { + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/next.py"})) + } + require.NoError(t, mi.GetIndexer("provider").IndexFile(filepath.Join(provider, added))) + }) + } + } +} diff --git a/internal/indexer/bridge_manifest_snapshot.go b/internal/indexer/bridge_manifest_snapshot.go index df4c8ba1d..cb5c5643f 100644 --- a/internal/indexer/bridge_manifest_snapshot.go +++ b/internal/indexer/bridge_manifest_snapshot.go @@ -53,6 +53,7 @@ type bridgeManifestSnapshot struct { pythonBuilt bool pythonEntries map[string]*pythonManifestEntry cargo resolver.CargoCrateLookup + cargoDeclared bool } func (mi *MultiIndexer) newBridgeManifestSnapshot() *bridgeManifestSnapshot { @@ -214,7 +215,7 @@ func (mi *MultiIndexer) cargoManifestSnapshot() *bridgeManifestSnapshot { if mi.cargoPackageCache.matches(next) { return mi.cargoPackageCache } - next.cargo = mi.buildCargoCrateLookup(next.trees) + next.cargo, next.cargoDeclared = mi.buildCargoCrateLookup(next.trees) mi.cargoPackageCache = next return next } @@ -246,3 +247,26 @@ func (mi *MultiIndexer) lazyCargoCrateLookup() resolver.CargoCrateLookup { return lookup(file, name) } } + +func (s *bridgeManifestSnapshot) hasUnknownManifest() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.unknown +} + +func (s *bridgeManifestSnapshot) declaresPythonBridge() bool { + if s.hasUnknownManifest() { + return true + } + for provider, pkg := range s.pythonEntries { + if pkg.name == "" { + continue + } + for caller, consumer := range s.pythonEntries { + if caller != provider && consumer.deps[pkg.name] { + return true + } + } + } + return false +} diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go index a6d096fcc..3c362e085 100644 --- a/internal/indexer/cargo_crate.go +++ b/internal/indexer/cargo_crate.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/pelletier/go-toml/v2" + "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/modules" "github.com/zzet/gortex/internal/resolver" ) @@ -30,7 +31,7 @@ func readCargoManifest(tree manifestTree, dir string) map[string]any { // cargoCrateLookup snapshots root/member manifests once per cross-repo pass. // Only explicit workspace member globs are expanded; Cargo is never executed. -func (mi *MultiIndexer) buildCargoCrateLookup(trees map[string]manifestTree) resolver.CargoCrateLookup { +func (mi *MultiIndexer) buildCargoCrateLookup(trees map[string]manifestTree) (resolver.CargoCrateLookup, bool) { type crate struct { dir, name, root string deps map[string]string @@ -43,7 +44,7 @@ func (mi *MultiIndexer) buildCargoCrateLookup(trees map[string]manifestTree) res } } if len(roots) < 2 { - return nil + return nil, false } owners := map[string]*crate{} packages := map[string][]*crate{} @@ -132,10 +133,14 @@ func (mi *MultiIndexer) buildCargoCrateLookup(trees map[string]manifestTree) res } } } + declared := false for _, owner := range owners { owner.imports = map[string][]resolver.CargoCrate{} for alias, pkg := range owner.deps { for _, provider := range packages[pkg] { + if graph.RepoPrefixOfID(provider.root) != graph.RepoPrefixOfID(owner.dir) { + declared = true + } name := provider.name if alias != pkg { name = strings.ReplaceAll(alias, "-", "_") @@ -186,5 +191,5 @@ func (mi *MultiIndexer) buildCargoCrateLookup(trees map[string]manifestTree) res return nil } return owner.imports[name] - } + }, declared } diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go index aa0ee2401..d59234568 100644 --- a/internal/indexer/cargo_import_restub.go +++ b/internal/indexer/cargo_import_restub.go @@ -98,6 +98,28 @@ func (mi *MultiIndexer) cargoConsumerFilesForRepos(repos []string, retargetBound if lookup == nil || len(repos) == 0 { return nil } + return mi.cargoConsumerFilesForLookup(repos, retargetBound, lookup) +} + +func (mi *MultiIndexer) cargoAddedFileConsumers(repo string) []string { + snapshot := mi.cargoManifestSnapshot() + if snapshot.hasUnknownManifest() { + // Missing evidence must widen recovery, never discard a possible importer. + files := map[string]struct{}{} + for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { + if strings.HasSuffix(edge.FilePath, ".rs") { + files[edge.FilePath] = struct{}{} + } + } + return sortedStringKeys(files) + } + if !snapshot.cargoDeclared { + return nil + } + return mi.cargoConsumerFilesForLookup([]string{repo}, true, snapshot.cargo) +} + +func (mi *MultiIndexer) cargoConsumerFilesForLookup(repos []string, retargetBound bool, lookup resolver.CargoCrateLookup) []string { wanted := make(map[string]bool, len(repos)) for _, repo := range repos { wanted[repo] = true diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index a13fa818e..5bfbab4b0 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -545,7 +545,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.npmConsumerFiles...) crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.cargoConsumerFiles...) if batch.cargoAddedFile { - crossRepoFiles = appendUniqueSorted(crossRepoFiles, mi.cargoConsumerFilesForRepo(repoPrefix, true)...) + crossRepoFiles = appendUniqueSorted(crossRepoFiles, mi.cargoAddedFileConsumers(repoPrefix)...) cargoRetargetRepo = repoPrefix } crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.pythonConsumerFiles...) diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index 655bc83d5..96d59332b 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -206,5 +206,14 @@ func (mi *MultiIndexer) pythonAddedFileConsumers(files []string) []string { if !warming && len(mi.AllMetadata()) < 2 { return nil } - return pythonConsumerFilesForAddedFiles(mi.graph, files) + if !warming && !mi.pythonManifestSnapshot().declaresPythonBridge() { + return nil + } + eligible := make([]string, 0, len(files)) + for _, file := range files { + if !resolver.IsPythonTestPackageFile(file, graph.RepoPrefixOfID(file)) { + eligible = append(eligible, file) + } + } + return pythonConsumerFilesForAddedFiles(mi.graph, eligible) } From 18cf7e7cae99a69eb6115a1af66f5dec8274a273 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 14:25:54 -0700 Subject: [PATCH 097/111] perf(indexer): hydrate only foreign bridge sites when parking a repo Read light inbound identities and load full edges only for foreign import and call sites carrying a bridge marker. --- internal/indexer/bridge_import_restub.go | 38 +++++++++++ internal/indexer/bridge_incoming_work_test.go | 65 +++++++++++++++++++ internal/indexer/cargo_import_restub.go | 12 +--- internal/indexer/python_import_restub.go | 12 +--- 4 files changed, 107 insertions(+), 20 deletions(-) create mode 100644 internal/indexer/bridge_import_restub.go create mode 100644 internal/indexer/bridge_incoming_work_test.go diff --git a/internal/indexer/bridge_import_restub.go b/internal/indexer/bridge_import_restub.go new file mode 100644 index 000000000..3f41b8942 --- /dev/null +++ b/internal/indexer/bridge_import_restub.go @@ -0,0 +1,38 @@ +package indexer + +import "github.com/zzet/gortex/internal/graph" + +// Repository parking can only change foreign imports/calls. Read their compact +// keys first, then hydrate those sites after the identity cursor has closed. +func foreignBridgeIncoming(g graph.Store, repo string, ids []string) map[string][]*graph.Edge { + identities := graph.InEdgeIdentitiesByNodeIDs(g, ids) + selected := map[graph.EdgeIdentity]bool{} + seen := map[graph.EdgeSite]bool{} + var sites []graph.EdgeSite + for _, rows := range identities { + for _, row := range rows { + if graph.RepoPrefixOfID(row.From) == repo || row.Kind != graph.EdgeImports && row.Kind != graph.EdgeCalls { + continue + } + selected[row] = true + site := graph.EdgeSite{From: row.From, Line: row.Line, Kind: row.Kind} + if !seen[site] { + seen[site] = true + sites = append(sites, site) + } + } + } + incoming := map[string][]*graph.Edge{} + for start := 0; start < len(sites); start += 512 { + page := sites[start:min(start+512, len(sites))] + rows := g.GetEdgeCandidates(nil, page) + for _, site := range page { + for _, edge := range rows.Site(site.From, site.Line, site.Kind) { + if selected[graph.EdgeIdentityFor(edge)] { + incoming[edge.To] = append(incoming[edge.To], edge) + } + } + } + } + return incoming +} diff --git a/internal/indexer/bridge_incoming_work_test.go b/internal/indexer/bridge_incoming_work_test.go new file mode 100644 index 000000000..870285b18 --- /dev/null +++ b/internal/indexer/bridge_incoming_work_test.go @@ -0,0 +1,65 @@ +package indexer + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +type bridgeIncomingCountStore struct { + graph.Store + fullReads, lightReads, hydrated int +} + +func (s *bridgeIncomingCountStore) GetInEdgesByNodeIDs(ids []string) map[string][]*graph.Edge { + s.fullReads++ + return s.Store.GetInEdgesByNodeIDs(ids) +} +func (s *bridgeIncomingCountStore) GetInEdgeIdentitiesByNodeIDs(ids []string) map[string][]graph.EdgeIdentity { + s.lightReads++ + return graph.InEdgeIdentitiesByNodeIDs(s.Store, ids) +} +func (s *bridgeIncomingCountStore) GetEdgeCandidates(endpoints []graph.EdgeEndpoint, sites []graph.EdgeSite) graph.EdgeCandidateSet { + s.hydrated += len(sites) + return s.Store.GetEdgeCandidates(endpoints, sites) +} + +func TestBridgeRepoRestubHydratesOnlyForeignSites(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, lang := range []string{"python", "cargo"} { + t.Run(backend+"/"+lang, func(t *testing.T) { + g := &bridgeIncomingCountStore{Store: cargoWarmStore(t, backend)} + file, importKey, callKey := "provider/mod.py", resolver.PythonImportStubMeta, resolver.PythonCallStubMeta + restub := restubTrackedPythonRepoIncoming + if lang == "cargo" { + file, importKey, callKey, restub = "provider/mod.rs", resolver.CargoImportStubMeta, resolver.CargoCallStubMeta, restubTrackedCargoRepoIncoming + } + target := file + "::f" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "provider"}) + g.AddNode(&graph.Node{ID: target, FilePath: file, Kind: graph.KindFunction, RepoPrefix: "provider"}) + for i := range 40 { + g.AddEdge(&graph.Edge{From: fmt.Sprintf("provider/local%d::run", i), To: target, FilePath: "provider/local", Kind: graph.EdgeCalls, CrossRepo: true, Meta: map[string]any{callKey: "unresolved::f"}}) + g.AddEdge(&graph.Edge{From: fmt.Sprintf("foreign/other%d", i), To: file, FilePath: "foreign/other", Kind: graph.EdgeContains}) + } + g.AddEdge(&graph.Edge{From: "consumer/app", To: file, FilePath: "consumer/app", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{importKey: "external::pkg.mod"}}) + g.AddEdge(&graph.Edge{From: "consumer/app::run", To: target, FilePath: "consumer/app", Kind: graph.EdgeCalls, Line: 2, CrossRepo: true, ConfidenceLabel: "high", Meta: map[string]any{callKey: "unresolved::f"}}) + g.AddEdge(&graph.Edge{From: "other/app", To: file, FilePath: "other/app", Kind: graph.EdgeImports, Line: 3, CrossRepo: true}) + require.Equal(t, []string{"consumer/app"}, restub(g, "provider")) + require.Zero(t, g.fullReads, "repository parking must not fetch full inbound adjacency") + require.Positive(t, g.lightReads) + require.Equal(t, 3, g.hydrated, "only foreign import/call sites need metadata") + require.Equal(t, file, g.GetOutEdges("other/app")[0].To) + require.Equal(t, target, g.GetOutEdges("provider/local0::run")[0].To) + call := g.GetOutEdges("consumer/app::run")[0] + require.Equal(t, "unresolved::f", call.To) + require.False(t, call.CrossRepo) + require.Empty(t, call.ConfidenceLabel) + require.True(t, graph.HasRestubProvenance(call)) + require.Empty(t, restub(g, "provider")) + }) + } + } +} diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go index d59234568..c54677d52 100644 --- a/internal/indexer/cargo_import_restub.go +++ b/internal/indexer/cargo_import_restub.go @@ -16,16 +16,8 @@ func restubTrackedCargoRepoIncoming(g graph.Store, repo string) []string { } var files, ids []string flush := func() { - incoming := g.GetInEdgesByNodeIDs(ids) - evicted := make(map[string]struct{}) - for _, edges := range incoming { - for _, edge := range edges { - if edge != nil && graph.RepoPrefixOfID(edge.From) == repo { - evicted[edge.From] = struct{}{} - } - } - } - files = appendUniqueSorted(files, restubTrackedCargoImportsFromView(g, evicted, incoming, true)...) + incoming := foreignBridgeIncoming(g, repo, ids) + files = appendUniqueSorted(files, restubTrackedCargoImportsFromView(g, nil, incoming, true)...) ids = ids[:0] } for node := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFile, graph.KindFunction, graph.KindMethod) { diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index 96d59332b..dbc822c24 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -14,16 +14,8 @@ func restubTrackedPythonRepoIncoming(g graph.Store, repo string) []string { } var files, ids []string flush := func() { - incoming := g.GetInEdgesByNodeIDs(ids) - evicted := make(map[string]struct{}) - for _, edges := range incoming { - for _, edge := range edges { - if edge != nil && graph.RepoPrefixOfID(edge.From) == repo { - evicted[edge.From] = struct{}{} - } - } - } - files = appendUniqueSorted(files, restubTrackedPythonImportsFromView(g, evicted, incoming, true)...) + incoming := foreignBridgeIncoming(g, repo, ids) + files = appendUniqueSorted(files, restubTrackedPythonImportsFromView(g, nil, incoming, true)...) ids = ids[:0] } for node := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFile, graph.KindFunction, graph.KindMethod, graph.KindType) { From 316c397787ea1219f8812f39306c4d0bb46765c7 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 14:26:02 -0700 Subject: [PATCH 098/111] perf(resolver): reuse caller imports for Python re-export calls Read the caller's imports from the bridge index and memoize each package's imports for the pass. A later export edit on the same resolver still sees the new target. --- internal/resolver/cross_repo.go | 3 + internal/resolver/python_package_reexport.go | 21 ++++++- .../resolver/python_reexport_work_test.go | 57 +++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 internal/resolver/python_reexport_work_test.go diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 6c18a2013..792ba29d8 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -114,6 +114,7 @@ type CrossRepoResolver struct { importedDirsByFile map[string]map[string]struct{} jsTSBindingsByFile map[string]map[string]string bridgeImportsByFile map[string][]*graph.Edge + pythonReexportsByFile map[string][]*graph.Edge pythonLocalTypesByFile map[string]map[string]bool // depModuleIndex bridges Go imports to dep:: contract // nodes from the caller's go.mod. Same shape and rationale as @@ -871,6 +872,7 @@ func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]boo dirs := make(map[string]map[string]struct{}) cr.jsTSBindingsByFile = make(map[string]map[string]string) cr.bridgeImportsByFile = make(map[string][]*graph.Edge) + cr.pythonReexportsByFile = make(map[string][]*graph.Edge) cr.pythonLocalTypesByFile = make(map[string]map[string]bool) var pythonFiles []string // Materialise metadata-free import projections and batch-load only target @@ -993,6 +995,7 @@ func (cr *CrossRepoResolver) clearReachableReposIndex() { cr.importedDirsByFile = nil cr.jsTSBindingsByFile = nil cr.bridgeImportsByFile = nil + cr.pythonReexportsByFile = nil cr.pythonLocalTypesByFile = nil } diff --git a/internal/resolver/python_package_reexport.go b/internal/resolver/python_package_reexport.go index c4196822e..b5db5f2e9 100644 --- a/internal/resolver/python_package_reexport.go +++ b/internal/resolver/python_package_reexport.go @@ -15,7 +15,7 @@ func (cr *CrossRepoResolver) resolvePythonPackageReExport(e *graph.Edge, module, callerWS := cr.callerWorkspaceID(e) var match *graph.Node packageFile := "" - for _, imp := range cr.graph.GetOutEdges(cr.callerFileID(e)) { + for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { if imp.Kind != graph.EdgeImports || !imp.CrossRepo || !strings.HasSuffix(imp.To, "/__init__.py") { continue } @@ -38,7 +38,7 @@ func (cr *CrossRepoResolver) resolvePythonPackageReExport(e *graph.Edge, module, if !pythonModuleFileMatches(pkg.FilePath, spec) { continue } - for _, reexport := range cr.graph.GetOutEdges(pkg.ID) { + for _, reexport := range cr.pythonPackageReexports(pkg.ID) { if reexport.Kind != graph.EdgeImports { continue } @@ -82,3 +82,20 @@ func (cr *CrossRepoResolver) resolvePythonPackageReExport(e *graph.Edge, module, e.Meta[PythonReExportFileMeta] = packageFile return true } + +func (cr *CrossRepoResolver) pythonPackageReexports(file string) []*graph.Edge { + if rows, seen := cr.pythonReexportsByFile[file]; seen { + return rows + } + var imports []*graph.Edge + for _, edge := range cr.graph.GetOutEdges(file) { + if edge.Kind == graph.EdgeImports { + imports = append(imports, edge) + } + } + if cr.pythonReexportsByFile == nil { + cr.pythonReexportsByFile = map[string][]*graph.Edge{} + } + cr.pythonReexportsByFile[file] = imports + return imports +} diff --git a/internal/resolver/python_reexport_work_test.go b/internal/resolver/python_reexport_work_test.go new file mode 100644 index 000000000..6120328b3 --- /dev/null +++ b/internal/resolver/python_reexport_work_test.go @@ -0,0 +1,57 @@ +package resolver + +import ( + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "testing" +) + +type pythonReexportCountStore struct { + graph.Store + reads map[string]int +} + +func (s *pythonReexportCountStore) GetOutEdges(id string) []*graph.Edge { + s.reads[id]++ + return s.Store.GetOutEdges(id) +} + +func TestPythonPackageReExportReadsOncePerPass(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &pythonReexportCountStore{Store: goImportReviewStore(t, backend), reads: map[string]int{}} + file, pkg, impl := "consumer/app.py", "provider/pkg/__init__.py", "provider/pkg/impl.py" + for _, node := range []*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared"}, + {ID: pkg, FilePath: pkg, Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: "shared"}, + {ID: impl + "::Original", Name: "Original", FilePath: impl, Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}, + {ID: impl + "::Alternate", Name: "Alternate", FilePath: impl, Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}, + } { + g.AddNode(node) + } + g.AddEdge(&graph.Edge{From: file, To: pkg, FilePath: file, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg", "python_import": "pkg"}}) + g.AddEdge(&graph.Edge{From: pkg, To: impl, FilePath: pkg, Kind: graph.EdgeImports, Meta: map[string]any{"python_reexports": map[string]any{"Public": "Original"}, "python_reexport_stem": "pkg/impl"}}) + cr := NewCrossRepo(g) + cr.buildReachableReposIndex(nil) + g.reads = map[string]int{} + for range 4 { + edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{}} + require.True(t, cr.resolvePythonPackageReExport(edge, "pkg", "Public")) + require.Equal(t, impl+"::Original", edge.To) + } + require.Zero(t, g.reads[file], "caller bridge metadata is already loaded") + require.Equal(t, 1, g.reads[pkg], "package reexports are shared by the pass") + cr.clearReachableReposIndex() + row := g.GetOutEdges(pkg)[0] + row.Meta["python_reexports"] = map[string]any{"Public": "Alternate"} + g.ReindexEdges([]graph.EdgeReindex{{Edge: row, OldTo: row.To}}) + cr.buildReachableReposIndex(nil) + g.reads = map[string]int{} + edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{}} + require.True(t, cr.resolvePythonPackageReExport(edge, "pkg", "Public")) + require.Equal(t, impl+"::Alternate", edge.To) + require.Zero(t, g.reads[file]) + require.Equal(t, 1, g.reads[pkg]) + }) + } +} From b78de7fa43ed37e5d007506295176d2ba2a0b3a9 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 14:26:19 -0700 Subject: [PATCH 099/111] fix(indexer): stop failed registrations from keeping Python warmup open Count only persisted, unregistered graph repositories in an active batch. Configured repositories that never register no longer keep the extra import scans running. --- .../python_added_file_warm_gate_test.go | 60 +++++++++++++++++++ internal/indexer/python_import_restub.go | 18 +++--- 2 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 internal/indexer/python_added_file_warm_gate_test.go diff --git a/internal/indexer/python_added_file_warm_gate_test.go b/internal/indexer/python_added_file_warm_gate_test.go new file mode 100644 index 000000000..b0e9e291d --- /dev/null +++ b/internal/indexer/python_added_file_warm_gate_test.go @@ -0,0 +1,60 @@ +package indexer + +import ( + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "testing" +) + +func TestPythonUnregisteredConfigDoesNotKeepWarming(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &pythonLookupScanStore{Store: cargoWarmStore(t, backend)} + g.AddNode(&graph.Node{ID: "provider/pkg/__init__.py", FilePath: "provider/pkg/__init__.py", RepoPrefix: "provider", Kind: graph.KindFile}) + cm := newTestConfigManager(t) + cm.Global().Repos = []config.RepoEntry{{Name: "provider", Path: t.TempDir()}, {Name: "missing", Path: t.TempDir() + "/absent"}, {Name: "failed", Path: t.TempDir()}} + mi := &MultiIndexer{graph: g, configMgr: cm, repos: map[string]*RepoMetadata{"provider": {RootPath: cm.Global().Repos[0].Path}}} + for range 2 { + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + } + require.Zero(t, g.importScans, "unregistered config is not a live warmup") + for _, parallel := range []bool{false, true} { + if parallel { + mi.BeginParallelBatch() + } else { + mi.BeginBatch() + } + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + require.Zero(t, g.importScans, "a one-repo graph has no pending foreign consumers") + mi.ResetBatch() + } + }) + } +} + +func TestPythonAddedFileWarmGateRequiresPendingGraphRepo(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, parallel := range []bool{false, true} { + t.Run(backend+map[bool]string{false: "/serial", true: "/parallel"}[parallel], func(t *testing.T) { + g := &pythonLookupScanStore{Store: cargoWarmStore(t, backend)} + for _, file := range []string{"provider/pkg/__init__.py", "consumer/app.py"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, RepoPrefix: graph.RepoPrefixOfID(file), Kind: graph.KindFile}) + } + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: "external::pkg.extra", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: t.TempDir()}}} + if parallel { + mi.BeginParallelBatch() + } else { + mi.BeginBatch() + } + require.Equal(t, []string{"consumer/app.py"}, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + require.Equal(t, 1, g.importScans) + mi.ResetBatch() + g.importScans = 0 + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/next.py"})) + require.Zero(t, g.importScans, "a completed warmup must not leave admission open") + }) + } + } +} diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go index dbc822c24..e54782597 100644 --- a/internal/indexer/python_import_restub.go +++ b/internal/indexer/python_import_restub.go @@ -186,16 +186,20 @@ func (mi *MultiIndexer) pythonAddedFileConsumers(files []string) []string { if len(files) == 0 { return nil } - // Warmup can publish a provider receipt before its configured consumers - // are registered. Preserve that frontier until the workspace pass. + // A live warmup batch can publish provider additions before persisted + // consumers register. Raw config entries are not evidence of pending work. + metadata := mi.AllMetadata() warming := false - if mi.configMgr != nil { - if global := mi.configMgr.Global(); global != nil { - configuredRepos := global.ReposSnapshot() - warming = len(configuredRepos) > 1 && len(mi.AllMetadata()) < len(configuredRepos) + mode := mi.currentBatchMode() + if mode.deferGlobalPasses { + for _, repo := range mi.graph.RepoPrefixes() { + if metadata[repo] == nil { + warming = true + break + } } } - if !warming && len(mi.AllMetadata()) < 2 { + if !warming && len(metadata) < 2 { return nil } if !warming && !mi.pythonManifestSnapshot().declaresPythonBridge() { From 2a998c925e87d262465fc9f98b470f80088c0130 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 15:33:25 -0700 Subject: [PATCH 100/111] fix(resolver): fill Python re-export imports before the worker fan-out The re-export memo was written by resolveEdge workers during a full pass, which races and can crash the daemon. Fill it while building the reachable repo index, so workers only read it, like the other per-pass caches. --- internal/resolver/cross_repo.go | 19 ++++++++ internal/resolver/python_package_reexport.go | 19 +------- .../resolver/python_reexport_work_test.go | 47 ++++++++++++++++++- 3 files changed, 65 insertions(+), 20 deletions(-) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 792ba29d8..5e5f25dfa 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -977,6 +977,25 @@ func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]boo set[to.RepoPrefix] = struct{}{} } } + // Publish package exports before ResolveAllContext starts its workers. + // The map and its edge slices are read-only until this index is cleared. + for _, imports := range cr.bridgeImportsByFile { + for _, imp := range imports { + if !imp.CrossRepo || !strings.HasSuffix(imp.To, "/__init__.py") { + continue + } + if _, seen := cr.pythonReexportsByFile[imp.To]; seen { + continue + } + var reexports []*graph.Edge + for _, edge := range cr.graph.GetOutEdges(imp.To) { + if edge.Kind == graph.EdgeImports { + reexports = append(reexports, edge) + } + } + cr.pythonReexportsByFile[imp.To] = reexports + } + } if len(pythonFiles) > 0 { for file, nodes := range cr.graph.GetFileNodesByPaths(pythonFiles) { for _, node := range nodes { diff --git a/internal/resolver/python_package_reexport.go b/internal/resolver/python_package_reexport.go index b5db5f2e9..4ab70f363 100644 --- a/internal/resolver/python_package_reexport.go +++ b/internal/resolver/python_package_reexport.go @@ -38,7 +38,7 @@ func (cr *CrossRepoResolver) resolvePythonPackageReExport(e *graph.Edge, module, if !pythonModuleFileMatches(pkg.FilePath, spec) { continue } - for _, reexport := range cr.pythonPackageReexports(pkg.ID) { + for _, reexport := range cr.pythonReexportsByFile[pkg.ID] { if reexport.Kind != graph.EdgeImports { continue } @@ -82,20 +82,3 @@ func (cr *CrossRepoResolver) resolvePythonPackageReExport(e *graph.Edge, module, e.Meta[PythonReExportFileMeta] = packageFile return true } - -func (cr *CrossRepoResolver) pythonPackageReexports(file string) []*graph.Edge { - if rows, seen := cr.pythonReexportsByFile[file]; seen { - return rows - } - var imports []*graph.Edge - for _, edge := range cr.graph.GetOutEdges(file) { - if edge.Kind == graph.EdgeImports { - imports = append(imports, edge) - } - } - if cr.pythonReexportsByFile == nil { - cr.pythonReexportsByFile = map[string][]*graph.Edge{} - } - cr.pythonReexportsByFile[file] = imports - return imports -} diff --git a/internal/resolver/python_reexport_work_test.go b/internal/resolver/python_reexport_work_test.go index 6120328b3..1775e4fa3 100644 --- a/internal/resolver/python_reexport_work_test.go +++ b/internal/resolver/python_reexport_work_test.go @@ -1,6 +1,7 @@ package resolver import ( + "fmt" "github.com/stretchr/testify/require" "github.com/zzet/gortex/internal/graph" "testing" @@ -32,7 +33,10 @@ func TestPythonPackageReExportReadsOncePerPass(t *testing.T) { g.AddEdge(&graph.Edge{From: file, To: pkg, FilePath: file, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg", "python_import": "pkg"}}) g.AddEdge(&graph.Edge{From: pkg, To: impl, FilePath: pkg, Kind: graph.EdgeImports, Meta: map[string]any{"python_reexports": map[string]any{"Public": "Original"}, "python_reexport_stem": "pkg/impl"}}) cr := NewCrossRepo(g) + g.reads = map[string]int{} cr.buildReachableReposIndex(nil) + require.Len(t, cr.pythonReexportsByFile[pkg], 1, "publish reexports before workers start") + require.Equal(t, 1, g.reads[pkg]) g.reads = map[string]int{} for range 4 { edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{}} @@ -40,18 +44,57 @@ func TestPythonPackageReExportReadsOncePerPass(t *testing.T) { require.Equal(t, impl+"::Original", edge.To) } require.Zero(t, g.reads[file], "caller bridge metadata is already loaded") - require.Equal(t, 1, g.reads[pkg], "package reexports are shared by the pass") + require.Zero(t, g.reads[pkg], "workers only read the prefilled package index") cr.clearReachableReposIndex() row := g.GetOutEdges(pkg)[0] row.Meta["python_reexports"] = map[string]any{"Public": "Alternate"} g.ReindexEdges([]graph.EdgeReindex{{Edge: row, OldTo: row.To}}) + g.reads = map[string]int{} cr.buildReachableReposIndex(nil) + require.Len(t, cr.pythonReexportsByFile[pkg], 1, "publish reexports before workers start") + require.Equal(t, 1, g.reads[pkg]) g.reads = map[string]int{} edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{}} require.True(t, cr.resolvePythonPackageReExport(edge, "pkg", "Public")) require.Equal(t, impl+"::Alternate", edge.To) require.Zero(t, g.reads[file]) - require.Equal(t, 1, g.reads[pkg]) + require.Zero(t, g.reads[pkg]) + }) + } +} + +// The full pass must publish package exports before its resolveEdge fan-out. +// Run this result assertion with -race in fork CI as well. +func TestPythonPackageReExportFullPassWorkers(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + for i := range 4 { + pkg := fmt.Sprintf("provider/pkg%d/__init__.py", i) + impl := fmt.Sprintf("provider/pkg%d/impl.py", i) + g.AddNode(&graph.Node{ID: pkg, FilePath: pkg, Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: "shared"}) + g.AddNode(&graph.Node{ID: impl + "::Original", Name: "Original", FilePath: impl, Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}) + g.AddEdge(&graph.Edge{From: pkg, To: impl, FilePath: pkg, Kind: graph.EdgeImports, Meta: map[string]any{"python_reexports": map[string]any{"Public": "Original"}, "python_reexport_stem": fmt.Sprintf("pkg%d/impl", i)}}) + } + for i := range 64 { + file := fmt.Sprintf("consumer/app%d.py", i) + spec := fmt.Sprintf("pkg%d", i%4) + pkg := "provider/" + spec + "/__init__.py" + stub := "unresolved::extern::" + spec + "::Public" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared"}) + g.AddEdge(&graph.Edge{From: file, To: pkg, FilePath: file, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::" + spec, "python_import": spec}}) + g.AddEdge(&graph.Edge{From: file, To: stub, FilePath: file, Kind: graph.EdgeCalls, Line: 2, Meta: map[string]any{PythonCallStubMeta: stub}}) + } + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { return nil }) + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { return nil }) + cr.ResolveAll() + for i := range 64 { + calls := outEdgesOfKind(g, fmt.Sprintf("consumer/app%d.py", i), graph.EdgeCalls) + require.Len(t, calls, 1) + require.Equal(t, fmt.Sprintf("provider/pkg%d/impl.py::Original", i%4), calls[0].To) + require.True(t, calls[0].CrossRepo) + } }) } } From 1ce719658dd3aa1908d6e6f00f2ab2f828a94e12 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 15:33:34 -0700 Subject: [PATCH 101/111] fix(indexer): keep the Cargo lookup nil below two Cargo roots Install cargoCrateLookup, which keeps the cheap root count, so the Cargo gates skip Rust work again in workspaces with fewer than two Cargo roots and cold output is unchanged there. --- .../indexer/cargo_resolver_admission_test.go | 62 +++++++++++++++++++ internal/indexer/workspace_resolve.go | 2 +- 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 internal/indexer/cargo_resolver_admission_test.go diff --git a/internal/indexer/cargo_resolver_admission_test.go b/internal/indexer/cargo_resolver_admission_test.go new file mode 100644 index 000000000..311d9cef7 --- /dev/null +++ b/internal/indexer/cargo_resolver_admission_test.go @@ -0,0 +1,62 @@ +package indexer + +import ( + "iter" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "go.uber.org/zap" +) + +type cargoAdmissionStore struct { + graph.Store + scans int +} + +func (s *cargoAdmissionStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + return func(yield func(*graph.Edge) bool) { + if len(kinds) == 1 && kinds[0] == graph.EdgeImports { + s.scans++ + } + graph.EdgesLightSeq(s.Store, kinds...)(yield) + } +} + +func TestCargoResolverSingleRootAdmission(t *testing.T) { + for _, withCall := range []bool{false, true} { + name := "imports_only" + if withCall { + name = "cross_repo_call" + } + t.Run(name, func(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, first, "Cargo.toml", "[package]\nname='app'\n") + g := &cargoAdmissionStore{Store: graph.New()} + file, target := "first/src/lib.rs", "second/src/lib.rs" + for _, n := range []*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, Language: "rust", RepoPrefix: "first", WorkspaceID: "shared"}, + {ID: target, FilePath: target, Kind: graph.KindFile, Language: "rust", RepoPrefix: "second", WorkspaceID: "shared"}, + {ID: target + "::Run", FilePath: target, Name: "Run", Kind: graph.KindFunction, Language: "rust", RepoPrefix: "second", WorkspaceID: "shared"}, + } { + g.AddNode(n) + } + g.AddEdge(&graph.Edge{From: file, To: target, FilePath: file, Kind: graph.EdgeImports, Line: 1, CrossRepo: true}) + if withCall { + g.AddEdge(&graph.Edge{From: file, To: "unresolved::Run", FilePath: file, Kind: graph.EdgeCalls, Line: 2}) + } + mi := &MultiIndexer{graph: g, logger: zap.NewNop(), repos: map[string]*RepoMetadata{"first": {RootPath: first}, "second": {RootPath: second}}} + mi.newCrossRepoResolver().ResolveAll() + wantScans := 2 // Existing npm and Python import discovery on the minimal Store wrapper. + if withCall { + wantScans++ + } // Reachability for the pending call. + require.Equal(t, wantScans, g.scans, "Cargo must not consume another import scan below two roots") + edges := g.GetOutEdges(file) + require.Contains(t, edges, &graph.Edge{From: file, To: target, FilePath: file, Kind: graph.EdgeImports, Line: 1, CrossRepo: true}) + if withCall { + require.Contains(t, edges, &graph.Edge{From: file, To: target + "::Run", FilePath: file, Kind: graph.EdgeCalls, Line: 2, CrossRepo: true}) + } + }) + } +} diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 9681d92ec..47ecd846e 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -498,7 +498,7 @@ func (mi *MultiIndexer) newCrossRepoResolver() *resolver.CrossRepoResolver { cr.SetNpmAliasResolver(mi.npmAliasResolver()) cr.SetNpmDependencyLookup(mi.npmDependencyLookup()) cr.SetNpmPackageLookup(mi.npmPackageLookup()) - cr.SetCargoCrateLookup(mi.lazyCargoCrateLookup()) + cr.SetCargoCrateLookup(mi.cargoCrateLookup()) if mi.IsMultiRepo() { cr.SetPythonPackageLookup(mi.pythonPackageLookup) } From b7ff48a3c43cf6a7d525d127a10a7aa9dd4f8792 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 15:33:49 -0700 Subject: [PATCH 102/111] fix(indexer): mark only master-resolved files as prepared Pass the files the master resolver saw, not the union with affected files, so affected Python and Rust referrers still get the pre-bridge outgoing pass and match a cold index. --- internal/indexer/incremental_watcher_batch.go | 2 +- .../indexer/python_save_resolve_work_test.go | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index 5bfbab4b0..d3a9cb8e1 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -564,7 +564,7 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } mi.runMasterResolveFilesWithEvidence(frontier, false, idx.resolver) }) - masterResolvedFiles = resolvedFiles + masterResolvedFiles = files crossRepoFiles = appendUniqueSorted(crossRepoFiles, resolvedFiles...) if receipt != nil && receipt.Complete { mi.runMasterResolveNames(vanishedReceiptNames(mi.graph, receipt)) diff --git a/internal/indexer/python_save_resolve_work_test.go b/internal/indexer/python_save_resolve_work_test.go index 80d94bef3..22ecfcd78 100644 --- a/internal/indexer/python_save_resolve_work_test.go +++ b/internal/indexer/python_save_resolve_work_test.go @@ -50,3 +50,48 @@ func TestPythonSaveRunsOneIncomingResolve(t *testing.T) { } } } + +func TestPythonAffectedReferrerGetsBridgePreparation(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname='dist'\n") + writeWorkspaceFile(t, provider, "pkg/__init__.py", "") + writeWorkspaceFile(t, provider, "pkg/image.py", "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies=['dist']\n") + writeWorkspaceFile(t, consumer, "app.py", "from pkg.image import f\ndef run(): return f()\n") + writeWorkspaceFile(t, consumer, "defs.py", "def changed(): pass\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + idx := mi.GetIndexer("consumer") + require.NoError(t, idx.IndexFileNoResolve(filepath.Join(consumer, "app.py"))) + writeWorkspaceFile(t, consumer, "defs.py", "def changed(arg): pass\n") + require.NoError(t, idx.IndexFileNoResolve(filepath.Join(consumer, "defs.py"))) + core, logs := observer.New(zap.InfoLevel) + mi.logger = zap.New(core) + var masterFiles []string + idx.incrementalCatchupHook = func(kind string, files []string) { + if kind == "resolve" { + masterFiles = append(masterFiles, files...) + } + } + batch := &reparsePendingEnrichmentBatch{deferResolverCatchup: true, deferredAffectedFiles: map[string]struct{}{"consumer/app.py": {}}} + mi.resolveIncrementalRepoMutation("consumer", &IndexResult{DerivedInvalidation: DerivedInvalidationPlan{Files: []string{"consumer/defs.py"}}}, nil, batch) + require.Equal(t, []string{"consumer/defs.py"}, masterFiles) + outgoing := logs.FilterMessage("resolver: outgoing files phases").All() + require.Len(t, outgoing, 1, "affected referrers did not pass through the master callback") + require.EqualValues(t, 1, outgoing[0].ContextMap()["input_files"], "only the affected file needs preparation") + require.Equal(t, []string{"provider/pkg/image.py::f"}, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + cold, cg := newPythonBridgeIdentityIndexer(t, backend, entries) + for _, entry := range entries { + _, err := cold.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + require.Equal(t, pythonLifecycleBindings(cg, ""), pythonLifecycleBindings(g, "")) + }) + } +} From 3937c79a4718eb35a19a7d5e2724bf3474e9373f Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 15:34:07 -0700 Subject: [PATCH 103/111] fix(indexer): run the full pass after a deferred Go import recovery When a mutation pass defers Go import recovery without an incoming refusal, request the existing full cross-repo fallback, so later Go saves do not repeat the all-repo recheck until a restart. --- internal/indexer/go_module_deferral_test.go | 62 +++++++++++++++++++++ internal/indexer/workspace_resolve.go | 6 ++ 2 files changed, 68 insertions(+) create mode 100644 internal/indexer/go_module_deferral_test.go diff --git a/internal/indexer/go_module_deferral_test.go b/internal/indexer/go_module_deferral_test.go new file mode 100644 index 000000000..d383d5296 --- /dev/null +++ b/internal/indexer/go_module_deferral_test.go @@ -0,0 +1,62 @@ +package indexer + +import ( + "fmt" + "iter" + "os" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +type goDeferralStore struct { + graph.Store + rechecks int +} + +func (s *goDeferralStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { + if len(kinds) == 4 { + s.rechecks++ + } + return graph.RepoNodeIdentitiesSeq(s.Store, repos, kinds...) +} + +func TestGoModuleMutationDeferralRunsFullFallback(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "go.mod", "module example.org/library\n") + writeWorkspaceFile(t, consumer, "go.mod", "module example.org/app\n") + g := &goDeferralStore{Store: pythonLocalStore(t, backend)} + g.AddNode(&graph.Node{ID: "provider/lib.go", FilePath: "provider/lib.go", Name: "lib.go", Kind: graph.KindFile, Language: "go", RepoPrefix: "provider", WorkspaceID: "shared"}) + for i := range 2 { + file := fmt.Sprintf("consumer/f%d.go", i) + g.AddBatch([]*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, Language: "go", RepoPrefix: "consumer", WorkspaceID: "shared"}, + {ID: file + "::import::example.org/library", FilePath: file, Kind: graph.KindImport, Language: "go", RepoPrefix: "consumer", WorkspaceID: "shared", StartLine: 1, Meta: map[string]any{"path": "example.org/library"}}, + }, []*graph.Edge{{From: file, FilePath: file, To: "dep::example.org/library", Kind: graph.EdgeImports, Line: 1}}) + } + core, logs := observer.New(zap.InfoLevel) + mi := &MultiIndexer{graph: g, logger: zap.New(core), repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + mi.runCrossRepoResolveMutationFrontiers(nil, nil, []string{"consumer/f0.go"}, nil) + deferrals := logs.FilterMessage("Go module import identity recheck deferred to full pass").Len() + if os.Getenv("GX_SFR16_SMALL_GO_BUDGET") == "1" { + require.Equal(t, 1, deferrals, "the native budget overlay must exercise refusal") + } + require.Equal(t, deferrals, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len(), "every independent Go deferral needs a full continuation") + for i := range 2 { + require.Contains(t, g.GetOutEdges(fmt.Sprintf("consumer/f%d.go", i)), &graph.Edge{From: fmt.Sprintf("consumer/f%d.go", i), FilePath: fmt.Sprintf("consumer/f%d.go", i), To: "provider/lib.go", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/library", "go_import_fallback": "dep::example.org/library"}}) + } + require.Equal(t, 1+deferrals, g.rechecks, "one bounded identity recheck and, only after refusal, one full recovery recheck") + before := g.rechecks + for range 2 { + mi.runCrossRepoResolveMutationFrontiers(nil, nil, []string{"consumer/f0.go"}, nil) + } + require.Equal(t, before, g.rechecks, "later Go mutations must not repeat the all-repo recheck") + require.Equal(t, deferrals, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len(), "completed revisions must not request another full pass") + }) + } +} diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 47ecd846e..6ef795a73 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -598,6 +598,12 @@ func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, ed } mi.finishGoModuleCalls(cr) mi.reconcileRetargetedTestCalls(cr.TakeRetargetedTestCallFiles()) + if admission.Refused && !admission.IncomingRefused { + // The bounded import retry cannot finish this revision. Reuse the + // full fallback before another mutation repeats its identity sweep. + mi.logger.Info("cross-repo resolve: completing deferred Go module imports") + mi.runCrossRepoResolve(false) + } } // crossWorkspaceLookup builds a resolver.CrossWorkspaceDepLookup from From 1d67d5d2d31ae49279eedd80dc4cd59413e9f160 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 15:34:20 -0700 Subject: [PATCH 104/111] fix(indexer): compare the owner's repo prefix for root Cargo crates A root package has no slash in its directory, so every root crate with a dependency counted as a bridge. Path dependencies on its own workspace members no longer do. --- .../added_file_declaration_gate_test.go | 23 +++++++++++++++++++ internal/indexer/cargo_crate.go | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/internal/indexer/added_file_declaration_gate_test.go b/internal/indexer/added_file_declaration_gate_test.go index fd5a24c52..cda7ddd8e 100644 --- a/internal/indexer/added_file_declaration_gate_test.go +++ b/internal/indexer/added_file_declaration_gate_test.go @@ -131,3 +131,26 @@ func TestAddedFileAfterDeclarationMatchesCold(t *testing.T) { } } } + +func TestCargoRootOwnWorkspaceDependencyIsNotBridge(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + own := "[package]\nname='app'\n[workspace]\nmembers=['member']\n[dependencies]\nmember={path='member'}\n" + writeWorkspaceFile(t, first, "Cargo.toml", own) + writeWorkspaceFile(t, first, "member/Cargo.toml", "[package]\nname='member'\n") + writeWorkspaceFile(t, second, "Cargo.toml", "[package]\nname='other'\n") + g := &cargoWarmScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "first/src/lib.rs", FilePath: "first/src/lib.rs", To: "external::other/extra", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"first": {RootPath: first}, "second": {RootPath: second}}} + require.False(t, mi.cargoManifestSnapshot().cargoDeclared) + require.Empty(t, mi.cargoAddedFileConsumers("first")) + require.Zero(t, g.lightScans, "a local workspace dependency is not a tracked bridge") + writeWorkspaceFile(t, first, "Cargo.toml", own+"other='1'\n") + require.True(t, mi.cargoManifestSnapshot().cargoDeclared) + require.Equal(t, []string{"first/src/lib.rs"}, mi.cargoAddedFileConsumers("second")) + require.Equal(t, 1, g.lightScans) + writeWorkspaceFile(t, first, "Cargo.toml", own) + require.False(t, mi.cargoManifestSnapshot().cargoDeclared) + g.lightScans = 0 + require.Empty(t, mi.cargoAddedFileConsumers("first")) + require.Zero(t, g.lightScans) +} diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go index 3c362e085..95f2cfa6d 100644 --- a/internal/indexer/cargo_crate.go +++ b/internal/indexer/cargo_crate.go @@ -138,7 +138,7 @@ func (mi *MultiIndexer) buildCargoCrateLookup(trees map[string]manifestTree) (re owner.imports = map[string][]resolver.CargoCrate{} for alias, pkg := range owner.deps { for _, provider := range packages[pkg] { - if graph.RepoPrefixOfID(provider.root) != graph.RepoPrefixOfID(owner.dir) { + if graph.RepoPrefixOfID(provider.root) != graph.RepoPrefixOfID(owner.root) { declared = true } name := provider.name From 89cd86b93c9c09b7d46823b8b32a38a3d2a473f0 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 15:34:29 -0700 Subject: [PATCH 105/111] fix(resolver): drain re-queued Go module work on any save Run the Go module retry whenever re-queued batches are pending, without reseeding, so a dropped reindex or failed untrack retry does not wait for the next Go save or full pass. --- internal/resolver/cross_repo_incremental.go | 3 + .../resolver/go_module_pending_drain_test.go | 61 +++++++++++++++++++ internal/resolver/go_modules_cross_repo.go | 36 +++++++---- 3 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 internal/resolver/go_module_pending_drain_test.go diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index e71425f6e..537365504 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -204,6 +204,9 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed cr.logger.Error("cross-repo resolve: retry Go module terminals", zap.Error(err)) } } + if !goMutation { + cr.retryPendingGoModuleEdges(stats) + } // Import recovery has its own ceiling: a popular provider must not starve // incoming calls or the revision-keyed call-terminal retry above. for repo := range providerRepos { diff --git a/internal/resolver/go_module_pending_drain_test.go b/internal/resolver/go_module_pending_drain_test.go new file mode 100644 index 000000000..fdc6e330f --- /dev/null +++ b/internal/resolver/go_module_pending_drain_test.go @@ -0,0 +1,61 @@ +package resolver + +import ( + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "iter" + "testing" +) + +type goPendingDrainStore struct { + *goModuleDroppingStore + identitySweeps int +} + +func (s *goPendingDrainStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { + s.identitySweeps++ + return graph.RepoNodeIdentitiesSeq(s.Store, repos, kinds...) +} + +func TestGoModulePendingDrainsOnNonGoMutation(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &goPendingDrainStore{goModuleDroppingStore: &goModuleDroppingStore{Store: goImportReviewStore(t, backend), drop: true}} + g.AddNode(&graph.Node{ID: "a/main.go::Caller", FilePath: "a/main.go", Kind: graph.KindFunction, Name: "Caller", Language: "go", RepoPrefix: "a", WorkspaceID: "shared"}) + g.AddNode(&graph.Node{ID: "b/lib.go::Fn", FilePath: "b/lib.go", Kind: graph.KindFunction, Name: "Fn", Language: "go", RepoPrefix: "b", WorkspaceID: "shared"}) + edge := &graph.Edge{From: "a/main.go::Caller", FilePath: "a/main.go", To: "dep::example.org/lib::Fn", Kind: graph.EdgeCalls, Line: 1} + g.AddEdge(edge) + state := &GoModuleDepRetry{} + makeResolver := func() *CrossRepoResolver { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.org/lib"}) + cr.SetGoModuleDepRetry(state, 1) + return cr + } + require.NoError(t, makeResolver().ResolveDeferredGoModuleEdges([]*graph.Edge{edge})) + require.Len(t, state.pending, 1) + g.drop = true // The next drain must preserve a second refusal. + g.identitySweeps = 0 + _, admission := makeResolver().ResolveMutationFrontiersBounded(nil, nil, []string{"a/README.md"}) + require.False(t, admission.Refused) + require.Len(t, state.pending, 1) + require.False(t, g.drop, "the queued batch must have been attempted") + require.Equal(t, "dep::example.org/lib::Fn", g.GetOutEdges(edge.From)[0].To) + _, admission = makeResolver().ResolveMutationFrontiersBounded(nil, nil, []string{"a/app.py"}) + require.False(t, admission.Refused) + require.Empty(t, state.pending) + var calls []*graph.Edge + for _, row := range g.GetOutEdges(edge.From) { + if row.Kind == graph.EdgeCalls { + calls = append(calls, row) + } + } + require.Len(t, calls, 1) + require.Equal(t, "b/lib.go::Fn", calls[0].To) + require.True(t, calls[0].CrossRepo) + require.False(t, state.complete, "a queue drain must not certify the legacy sweep") + makeResolver().ResolveMutationFrontiersBounded(nil, nil, []string{"a/app.ts"}) + require.Zero(t, g.identitySweeps, "non-Go saves may drain, not reseed or recheck all repositories") + }) + } +} diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index 877aa2a5c..232a1e6a0 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -109,19 +109,7 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross return err } } - state.mu.Lock() - pending := state.pending - state.pending = nil - state.mu.Unlock() - var deferred []*graph.Edge - for _, edge := range pending { - deferred = append(deferred, edge) - if len(deferred) == externalCallMutationChunk { - cr.resolveGoModuleEdgesLocked(deferred, stats) - deferred = deferred[:0] - } - } - cr.resolveGoModuleEdgesLocked(deferred, stats) + cr.retryPendingGoModuleEdges(stats) state.mu.Lock() if len(cr.goModules) == 0 { state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, cr.goModuleImportsDeferred == 0 @@ -234,6 +222,28 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross return nil } +// retryPendingGoModuleEdges drains exact dropped batches without opening the +// revision-wide registry seed or legacy binding sweep. The resolve mutex is held. +func (cr *CrossRepoResolver) retryPendingGoModuleEdges(stats *CrossRepoStats) { + state := cr.goModuleRetry + if state == nil { + return + } + state.mu.Lock() + pending := state.pending + state.pending = nil + state.mu.Unlock() + var deferred []*graph.Edge + for _, edge := range pending { + deferred = append(deferred, edge) + if len(deferred) == externalCallMutationChunk { + cr.resolveGoModuleEdgesLocked(deferred, stats) + deferred = deferred[:0] + } + } + cr.resolveGoModuleEdgesLocked(deferred, stats) +} + // goModuleIncomingEdges probes only package-qualified placeholders owned by the // changed Go definitions. Bare-name frontier keys cannot find these imports. func (cr *CrossRepoResolver) goModuleIncomingEdges(nodesByFile map[string][]*graph.Node, definitionFiles []string) []*graph.Edge { From 4a2f602245e849036793ac752ad4315a2cba2a79 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 15:34:38 -0700 Subject: [PATCH 106/111] docs(indexer): describe the manifest snapshot invalidation rule Fix comments left stale by the snapshot refactor and state when a snapshot is reused: stat stamps, the racy window, the file-set revision and the config revision. --- internal/indexer/bridge_manifest_snapshot.go | 11 ++++++++++- internal/indexer/cargo_crate.go | 5 +++-- internal/indexer/python_tracked_import.go | 5 ++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/internal/indexer/bridge_manifest_snapshot.go b/internal/indexer/bridge_manifest_snapshot.go index cb5c5643f..445be546a 100644 --- a/internal/indexer/bridge_manifest_snapshot.go +++ b/internal/indexer/bridge_manifest_snapshot.go @@ -56,6 +56,11 @@ type bridgeManifestSnapshot struct { cargoDeclared bool } +// newBridgeManifestSnapshot captures config revision, repo roots and file-set +// revisions before manifest reads. Parsed views survive resolver passes only +// while these inputs and every watched size/mtime/mode stamp remain unchanged. +// A positive stamp must be strictly older than stampedAt - manifestRacyWindow; +// unknown evidence, source-backed trees and missing inventory owners forbid reuse. func (mi *MultiIndexer) newBridgeManifestSnapshot() *bridgeManifestSnapshot { s := &bridgeManifestSnapshot{ roots: map[string]string{}, inventory: map[string]bridgeManifestInventory{}, @@ -197,6 +202,9 @@ func (mi *MultiIndexer) pythonManifestSnapshot() *bridgeManifestSnapshot { return next } +// pythonPackageLookup builds providers on first use of the reusable manifest +// snapshot. Providers require indexed root/src modules or regular packages; +// consumer-local indexed namespaces block foreign bindings. func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { snapshot := mi.pythonManifestSnapshot() mi.bridgeManifestMu.Lock() @@ -235,7 +243,8 @@ func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { return mi.lazyCargoCrateLookup() } -// Like npm, resolver construction does no Cargo work before the first use. +// lazyCargoCrateLookup defers manifest parsing until the first lookup. +// cargoCrateLookup performs cheap root-count admission before installing it. func (mi *MultiIndexer) lazyCargoCrateLookup() resolver.CargoCrateLookup { var once sync.Once var lookup resolver.CargoCrateLookup diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go index 95f2cfa6d..e38b5e9e7 100644 --- a/internal/indexer/cargo_crate.go +++ b/internal/indexer/cargo_crate.go @@ -29,8 +29,9 @@ func readCargoManifest(tree manifestTree, dir string) map[string]any { return manifest } -// cargoCrateLookup snapshots root/member manifests once per cross-repo pass. -// Only explicit workspace member globs are expanded; Cargo is never executed. +// buildCargoCrateLookup parses root/member manifests for one reusable snapshot. +// bridgeManifestSnapshot owns cross-pass invalidation. Only explicit workspace +// member globs are expanded; Cargo is never executed. func (mi *MultiIndexer) buildCargoCrateLookup(trees map[string]manifestTree) (resolver.CargoCrateLookup, bool) { type crate struct { dir, name, root string diff --git a/internal/indexer/python_tracked_import.go b/internal/indexer/python_tracked_import.go index 08b37e01d..1a740ffa5 100644 --- a/internal/indexer/python_tracked_import.go +++ b/internal/indexer/python_tracked_import.go @@ -18,9 +18,8 @@ func normalizePythonDistribution(name string) string { return pythonDistributionSeparators.ReplaceAllString(strings.ToLower(name), "-") } -// pythonPackageLookup is built only after a non-stdlib Python import is found. -// Providers require indexed root/src modules or regular packages; consumers also -// block foreign bindings with their own indexed namespace directories. +// pythonManifestEntry holds a distribution identity and declared dependencies. +// Declaration admission reads these entries without discovering provider files. type pythonManifestEntry struct { name string deps map[string]bool From 7139130dbe265b571bef845b48e976a4bb988df1 Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 16:23:01 -0700 Subject: [PATCH 107/111] fix(resolver): skip over-budget provider recovery without refusing the pass A provider save above the recovery budget logs and skips, as before the full-pass continuation, and no longer marks the pass refused or reopens the revision. Only the revision-wide consumer recheck can request the one full continuation. Consumers of a provider with more than 16,384 registered imports keep stale import targets until a full pass. --- internal/resolver/cross_repo_incremental.go | 5 ++- .../resolver/go_module_import_seed_test.go | 36 ++++++++++++++----- internal/resolver/go_module_imports_test.go | 4 +++ 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 537365504..28a7f6e53 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -208,16 +208,15 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed cr.retryPendingGoModuleEdges(stats) } // Import recovery has its own ceiling: a popular provider must not starve - // incoming calls or the revision-keyed call-terminal retry above. + // incoming calls or the revision-keyed call-terminal retry above. A skipped + // provider stays stale until a full pass; it must not reopen the revision. for repo := range providerRepos { ids := cr.goModuleImportIDs(repo) if len(ids) > importBudget.Remaining() { - cr.goModuleImportsDeferred += len(ids) cr.logger.Warn("cross-repo resolve: Go module imports deferred to full pass", zap.Int("imports", len(ids)), zap.Int("remaining", importBudget.Remaining())) continue } if err := importBudget.Charge(len(ids)); err != nil { - cr.goModuleImportsDeferred += len(ids) cr.logger.Warn("cross-repo resolve: Go module imports deferred to full pass", zap.Error(err)) continue } diff --git a/internal/resolver/go_module_import_seed_test.go b/internal/resolver/go_module_import_seed_test.go index 2cab45954..1f7eaf5f5 100644 --- a/internal/resolver/go_module_import_seed_test.go +++ b/internal/resolver/go_module_import_seed_test.go @@ -110,8 +110,9 @@ func TestGoModuleImportOversizedProviderDoesNotStarveNext(t *testing.T) { state.imports["b-small"] = map[string]struct{}{id: {}} if route == "frontier" { _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"a-big/lib.go", "b-small/lib.go"}) - require.True(t, admission.Refused) - require.Error(t, admission.Refusal) + require.False(t, admission.Refused) + require.NoError(t, admission.Refusal) + require.True(t, state.complete) } else { require.NoError(t, cr.recheckGoModuleBindings(t.Context(), nil, &CrossRepoStats{ByRepo: map[string]int{}}, false)) } @@ -168,7 +169,7 @@ func TestGoModuleImportRecoveryBudgetRefusal(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { for _, route := range []string{"frontier", "identity"} { t.Run(backend+"/"+route, func(t *testing.T) { - g := goImportReviewStore(t, backend) + g := &goImportCountingStore{Store: goImportReviewStore(t, backend)} g.AddNode(&graph.Node{ID: "provider/lib.go", Name: "lib.go", FilePath: "provider/lib.go", Kind: graph.KindFile, RepoPrefix: "provider", Language: "go"}) ids := map[string]struct{}{} for i := 0; i < 2; i++ { @@ -189,12 +190,29 @@ func TestGoModuleImportRecoveryBudgetRefusal(t *testing.T) { if route == "identity" { changed = "consumer/f0.go" } - _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{changed}) - require.True(t, admission.Refused) - require.Error(t, admission.Refusal) - require.GreaterOrEqual(t, admission.Dropped, 2) - require.False(t, state.complete) - require.False(t, admission.IncomingRefused) + if route == "frontier" { + for _, file := range []string{changed, changed, "consumer/f0.go"} { + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{file}) + require.False(t, admission.Refused, "a provider skip must not request a full continuation") + require.NoError(t, admission.Refusal) + require.Zero(t, admission.Dropped) + require.True(t, state.complete, "a provider skip must not reopen the revision") + } + require.Zero(t, g.bindingRechecks, "two provider saves and a third Go save must not recheck all repositories") + for i := range 2 { + rows := outEdgesOfKind(g, fmt.Sprintf("consumer/f%d.go", i), graph.EdgeImports) + require.Len(t, rows, 1) + require.Equal(t, "dep::example.org/lib", rows[0].To, "provider skips preserve stale imports until a full pass") + } + } else { + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{changed}) + require.True(t, admission.Refused) + require.Error(t, admission.Refusal) + require.GreaterOrEqual(t, admission.Dropped, 2) + require.False(t, state.complete) + require.False(t, admission.IncomingRefused) + require.Equal(t, 1, g.bindingRechecks) + } // The explicit full pass is unbounded and repairs the deferred imports. _, err := cr.ResolveAllContext(t.Context()) require.NoError(t, err) diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index 1b9de3d87..5fdaa2fbb 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -161,6 +161,7 @@ type goImportCountingStore struct { fileScans, hydrated int edgeLoads int importScans, importRows int + bindingRechecks int afterBatch func() } @@ -169,6 +170,9 @@ func (s *goImportCountingStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[g return graph.FileNodeIdentitiesSeq(s.Store, repos) } func (s *goImportCountingStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { + if len(kinds) == 4 { + s.bindingRechecks++ + } return func(yield func(graph.RepoNodeIdentity) bool) { imports := len(kinds) == 1 && kinds[0] == graph.KindImport && repos == nil if imports { From 4be92d0d7ae8ebb25aa37f51168fed1955d2443a Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 16:23:15 -0700 Subject: [PATCH 108/111] test(resolver): prove the Go deferral continuation without an env switch Replace the env-gated indexer test with an external resolver integration test that shrinks the recovery budget through export_test.go and drives the public indexer entry point. It fails on both stores when the full continuation is removed. --- internal/indexer/go_module_deferral_test.go | 62 -------- internal/resolver/export_test.go | 17 +++ .../go_module_deferral_integration_test.go | 132 ++++++++++++++++++ 3 files changed, 149 insertions(+), 62 deletions(-) delete mode 100644 internal/indexer/go_module_deferral_test.go create mode 100644 internal/resolver/export_test.go create mode 100644 internal/resolver/go_module_deferral_integration_test.go diff --git a/internal/indexer/go_module_deferral_test.go b/internal/indexer/go_module_deferral_test.go deleted file mode 100644 index d383d5296..000000000 --- a/internal/indexer/go_module_deferral_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package indexer - -import ( - "fmt" - "iter" - "os" - "testing" - - "github.com/stretchr/testify/require" - "github.com/zzet/gortex/internal/graph" - "go.uber.org/zap" - "go.uber.org/zap/zaptest/observer" -) - -type goDeferralStore struct { - graph.Store - rechecks int -} - -func (s *goDeferralStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { - if len(kinds) == 4 { - s.rechecks++ - } - return graph.RepoNodeIdentitiesSeq(s.Store, repos, kinds...) -} - -func TestGoModuleMutationDeferralRunsFullFallback(t *testing.T) { - for _, backend := range []string{"memory", "sqlite"} { - t.Run(backend, func(t *testing.T) { - provider, consumer := t.TempDir(), t.TempDir() - writeWorkspaceFile(t, provider, "go.mod", "module example.org/library\n") - writeWorkspaceFile(t, consumer, "go.mod", "module example.org/app\n") - g := &goDeferralStore{Store: pythonLocalStore(t, backend)} - g.AddNode(&graph.Node{ID: "provider/lib.go", FilePath: "provider/lib.go", Name: "lib.go", Kind: graph.KindFile, Language: "go", RepoPrefix: "provider", WorkspaceID: "shared"}) - for i := range 2 { - file := fmt.Sprintf("consumer/f%d.go", i) - g.AddBatch([]*graph.Node{ - {ID: file, FilePath: file, Kind: graph.KindFile, Language: "go", RepoPrefix: "consumer", WorkspaceID: "shared"}, - {ID: file + "::import::example.org/library", FilePath: file, Kind: graph.KindImport, Language: "go", RepoPrefix: "consumer", WorkspaceID: "shared", StartLine: 1, Meta: map[string]any{"path": "example.org/library"}}, - }, []*graph.Edge{{From: file, FilePath: file, To: "dep::example.org/library", Kind: graph.EdgeImports, Line: 1}}) - } - core, logs := observer.New(zap.InfoLevel) - mi := &MultiIndexer{graph: g, logger: zap.New(core), repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} - mi.runCrossRepoResolveMutationFrontiers(nil, nil, []string{"consumer/f0.go"}, nil) - deferrals := logs.FilterMessage("Go module import identity recheck deferred to full pass").Len() - if os.Getenv("GX_SFR16_SMALL_GO_BUDGET") == "1" { - require.Equal(t, 1, deferrals, "the native budget overlay must exercise refusal") - } - require.Equal(t, deferrals, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len(), "every independent Go deferral needs a full continuation") - for i := range 2 { - require.Contains(t, g.GetOutEdges(fmt.Sprintf("consumer/f%d.go", i)), &graph.Edge{From: fmt.Sprintf("consumer/f%d.go", i), FilePath: fmt.Sprintf("consumer/f%d.go", i), To: "provider/lib.go", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/library", "go_import_fallback": "dep::example.org/library"}}) - } - require.Equal(t, 1+deferrals, g.rechecks, "one bounded identity recheck and, only after refusal, one full recovery recheck") - before := g.rechecks - for range 2 { - mi.runCrossRepoResolveMutationFrontiers(nil, nil, []string{"consumer/f0.go"}, nil) - } - require.Equal(t, before, g.rechecks, "later Go mutations must not repeat the all-repo recheck") - require.Equal(t, deferrals, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len(), "completed revisions must not request another full pass") - }) - } -} diff --git a/internal/resolver/export_test.go b/internal/resolver/export_test.go new file mode 100644 index 000000000..b5640c784 --- /dev/null +++ b/internal/resolver/export_test.go @@ -0,0 +1,17 @@ +package resolver + +import "github.com/zzet/gortex/internal/graph" + +// SetGoModuleImportRecoveryBudgetForTest changes only the bounded import budget. +// External resolver tests use it to exercise the real indexer continuation. +func SetGoModuleImportRecoveryBudgetForTest(remaining int) (restore func()) { + previous := newGoModuleImportRecoveryBudget + newGoModuleImportRecoveryBudget = func() *graph.IncomingSourceBudget { + budget := &graph.IncomingSourceBudget{} + if err := budget.Charge(graph.MaxIncomingSourceCandidateRows - remaining); err != nil { + panic(err) + } + return budget + } + return func() { newGoModuleImportRecoveryBudget = previous } +} diff --git a/internal/resolver/go_module_deferral_integration_test.go b/internal/resolver/go_module_deferral_integration_test.go new file mode 100644 index 000000000..c3e8bd0b4 --- /dev/null +++ b/internal/resolver/go_module_deferral_integration_test.go @@ -0,0 +1,132 @@ +package resolver_test + +import ( + "context" + "fmt" + "iter" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/resolver" + "github.com/zzet/gortex/internal/search" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +type goDeferralStore struct { + graph.Store + rechecks int + importSeeds int +} + +func (s *goDeferralStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { + if len(kinds) == 4 { + s.rechecks++ + } + if repos == nil && len(kinds) == 1 && kinds[0] == graph.KindImport { + s.importSeeds++ + } + return graph.RepoNodeIdentitiesSeq(s.Store, repos, kinds...) +} + +func newGoDeferralIndexer(t *testing.T, backend string) (*indexer.MultiIndexer, *goDeferralStore, *observer.ObservedLogs) { + t.Helper() + var store graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store = db + } + g := &goDeferralStore{Store: store} + cm, err := config.NewConfigManager(filepath.Join(t.TempDir(), "config.yaml")) + require.NoError(t, err) + reg := parser.NewRegistry() + reg.Register(languages.NewGoExtractor()) + core, logs := observer.New(zap.InfoLevel) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.New(core)) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + mi.BeginBatch() + for _, repo := range []struct{ name, module string }{{"provider", "example.org/library"}, {"consumer", "example.org/app"}} { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "go.mod"), []byte("module "+repo.module+"\n"), 0600)) + _, err := mi.TrackRepo(config.RepoEntry{Path: root, Name: repo.name, Workspace: "shared"}) + require.NoError(t, err) + } + mi.ResetBatch() + // Persisted graph identities are enough for the restart recovery boundary; + // no source parse or dependency fetch is needed to construct this fixture. + g.AddNode(&graph.Node{ID: "provider/lib.go", FilePath: "provider/lib.go", Name: "lib.go", Kind: graph.KindFile, Language: "go", RepoPrefix: "provider", WorkspaceID: "shared"}) + for i := range 2 { + file := fmt.Sprintf("consumer/f%d.go", i) + g.AddBatch([]*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, Language: "go", RepoPrefix: "consumer", WorkspaceID: "shared"}, + {ID: file + "::import::example.org/library", FilePath: file, Kind: graph.KindImport, Language: "go", RepoPrefix: "consumer", WorkspaceID: "shared", StartLine: 1, Meta: map[string]any{"path": "example.org/library"}}, + }, []*graph.Edge{{From: file, FilePath: file, To: "dep::example.org/library", Kind: graph.EdgeImports, Line: 1}}) + } + g.rechecks = 0 + g.importSeeds = 0 + logs.TakeAll() + return mi, g, logs +} + +func requireGoDeferralImports(t *testing.T, g graph.Store) { + t.Helper() + for i := range 2 { + file := fmt.Sprintf("consumer/f%d.go", i) + require.Contains(t, g.GetOutEdges(file), &graph.Edge{From: file, FilePath: file, To: "provider/lib.go", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/library", "go_import_fallback": "dep::example.org/library"}}) + } +} + +func TestGoModuleMutationDeferralRunsFullFallback(t *testing.T) { + t.Cleanup(resolver.SetGoModuleImportRecoveryBudgetForTest(1)) + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, logs := newGoDeferralIndexer(t, backend) + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/f0.go"}, nil)) + require.Equal(t, 1, logs.FilterMessage("Go module import identity recheck deferred to full pass").Len(), "the committed test must exercise the bounded refusal") + require.Equal(t, 1, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len()) + requireGoDeferralImports(t, g) + require.Equal(t, 2, g.rechecks, "one bounded recheck and one full continuation") + require.Equal(t, 2, g.importSeeds, "the bounded revision seeds once and the full continuation reseeds once") + for range 2 { + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/f0.go"}, nil)) + } + require.Equal(t, 2, g.rechecks, "later Go saves must not repeat the all-repository recheck") + require.Equal(t, 2, g.importSeeds, "later saves must not start another full pass or reseed") + require.Equal(t, 1, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len()) + requireGoDeferralImports(t, g) + }) + } +} + +func TestGoModuleProviderSkipsDoNotRunFullFallback(t *testing.T) { + t.Cleanup(resolver.SetGoModuleImportRecoveryBudgetForTest(1)) + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, logs := newGoDeferralIndexer(t, backend) + mi.RunGlobalResolve() + requireGoDeferralImports(t, g) + g.rechecks = 0 + g.importSeeds = 0 + logs.TakeAll() + for _, file := range []string{"provider/lib.go", "provider/lib.go", "consumer/f0.go"} { + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{file}, nil)) + } + require.Equal(t, 2, logs.FilterMessage("cross-repo resolve: Go module imports deferred to full pass").Len()) + require.Zero(t, logs.FilterMessage("Go module import identity recheck deferred to full pass").Len()) + require.Zero(t, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len()) + require.Zero(t, g.rechecks, "two provider saves and a third Go save must not recheck all repositories") + require.Zero(t, g.importSeeds, "a full pass always reseeds imports, so no reseed proves no full pass ran") + requireGoDeferralImports(t, g) + }) + } +} From e78972f5b30de4a27ec851e5a0347914ca1c7e4e Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 16:23:31 -0700 Subject: [PATCH 109/111] perf(resolver): batch the Python re-export package prefill Collect the distinct package IDs and read their edges with one GetOutEdgesByNodeIDs call per full pass instead of one point read per __init__.py. --- internal/resolver/cross_repo.go | 21 ++++--- .../resolver/python_reexport_work_test.go | 60 +++++++++++++++++-- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 5e5f25dfa..c6e68afaf 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -979,21 +979,28 @@ func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]boo } // Publish package exports before ResolveAllContext starts its workers. // The map and its edge slices are read-only until this index is cleared. + packageIDs := make(map[string]struct{}) for _, imports := range cr.bridgeImportsByFile { for _, imp := range imports { - if !imp.CrossRepo || !strings.HasSuffix(imp.To, "/__init__.py") { - continue - } - if _, seen := cr.pythonReexportsByFile[imp.To]; seen { - continue + if imp.CrossRepo && strings.HasSuffix(imp.To, "/__init__.py") { + packageIDs[imp.To] = struct{}{} } + } + } + if len(packageIDs) > 0 { + ids := make([]string, 0, len(packageIDs)) + for id := range packageIDs { + ids = append(ids, id) + } + rows := cr.graph.GetOutEdgesByNodeIDs(ids) + for _, id := range ids { var reexports []*graph.Edge - for _, edge := range cr.graph.GetOutEdges(imp.To) { + for _, edge := range rows[id] { if edge.Kind == graph.EdgeImports { reexports = append(reexports, edge) } } - cr.pythonReexportsByFile[imp.To] = reexports + cr.pythonReexportsByFile[id] = reexports } } if len(pythonFiles) > 0 { diff --git a/internal/resolver/python_reexport_work_test.go b/internal/resolver/python_reexport_work_test.go index 1775e4fa3..53bb6f4b7 100644 --- a/internal/resolver/python_reexport_work_test.go +++ b/internal/resolver/python_reexport_work_test.go @@ -4,19 +4,44 @@ import ( "fmt" "github.com/stretchr/testify/require" "github.com/zzet/gortex/internal/graph" + "strings" + "sync" "testing" ) type pythonReexportCountStore struct { graph.Store - reads map[string]int + mu sync.Mutex + reads map[string]int + batches [][]string + batchCalls int } func (s *pythonReexportCountStore) GetOutEdges(id string) []*graph.Edge { + s.mu.Lock() s.reads[id]++ + s.mu.Unlock() return s.Store.GetOutEdges(id) } +func (s *pythonReexportCountStore) GetOutEdgesByNodeIDs(ids []string) map[string][]*graph.Edge { + s.mu.Lock() + s.batchCalls++ + s.mu.Unlock() + var packages []string + for _, id := range ids { + if strings.HasSuffix(id, "/__init__.py") { + packages = append(packages, id) + } + } + if len(packages) > 0 { + s.mu.Lock() + s.batches = append(s.batches, packages) + s.mu.Unlock() + } + return s.Store.GetOutEdgesByNodeIDs(ids) +} + func TestPythonPackageReExportReadsOncePerPass(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { t.Run(backend, func(t *testing.T) { @@ -36,7 +61,8 @@ func TestPythonPackageReExportReadsOncePerPass(t *testing.T) { g.reads = map[string]int{} cr.buildReachableReposIndex(nil) require.Len(t, cr.pythonReexportsByFile[pkg], 1, "publish reexports before workers start") - require.Equal(t, 1, g.reads[pkg]) + require.Zero(t, g.reads[pkg], "prefill must not issue package point reads") + require.Equal(t, [][]string{{pkg}}, g.batches) g.reads = map[string]int{} for range 4 { edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{}} @@ -50,9 +76,11 @@ func TestPythonPackageReExportReadsOncePerPass(t *testing.T) { row.Meta["python_reexports"] = map[string]any{"Public": "Alternate"} g.ReindexEdges([]graph.EdgeReindex{{Edge: row, OldTo: row.To}}) g.reads = map[string]int{} + g.batches = nil cr.buildReachableReposIndex(nil) require.Len(t, cr.pythonReexportsByFile[pkg], 1, "publish reexports before workers start") - require.Equal(t, 1, g.reads[pkg]) + require.Zero(t, g.reads[pkg], "prefill must not issue package point reads") + require.Equal(t, [][]string{{pkg}}, g.batches) g.reads = map[string]int{} edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{}} require.True(t, cr.resolvePythonPackageReExport(edge, "pkg", "Public")) @@ -68,9 +96,11 @@ func TestPythonPackageReExportReadsOncePerPass(t *testing.T) { func TestPythonPackageReExportFullPassWorkers(t *testing.T) { for _, backend := range []string{"memory", "sqlite"} { t.Run(backend, func(t *testing.T) { - g := goImportReviewStore(t, backend) + g := &pythonReexportCountStore{Store: goImportReviewStore(t, backend), reads: map[string]int{}} + var packages []string for i := range 4 { pkg := fmt.Sprintf("provider/pkg%d/__init__.py", i) + packages = append(packages, pkg) impl := fmt.Sprintf("provider/pkg%d/impl.py", i) g.AddNode(&graph.Node{ID: pkg, FilePath: pkg, Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: "shared"}) g.AddNode(&graph.Node{ID: impl + "::Original", Name: "Original", FilePath: impl, Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}) @@ -89,6 +119,11 @@ func TestPythonPackageReExportFullPassWorkers(t *testing.T) { cr.SetPythonPackageLookup(func() *PythonPackageLookup { return nil }) cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { return nil }) cr.ResolveAll() + require.Len(t, g.batches, 1, "one package batch must precede all workers") + require.ElementsMatch(t, packages, g.batches[0], "duplicate caller imports must not duplicate package IDs") + for _, pkg := range packages { + require.Zero(t, g.reads[pkg], "prefill and workers must not issue package point reads") + } for i := range 64 { calls := outEdgesOfKind(g, fmt.Sprintf("consumer/app%d.py", i), graph.EdgeCalls) require.Len(t, calls, 1) @@ -98,3 +133,20 @@ func TestPythonPackageReExportFullPassWorkers(t *testing.T) { }) } } + +func TestPythonPackageReExportNonPackageSkipsBatch(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &pythonReexportCountStore{Store: goImportReviewStore(t, backend), reads: map[string]int{}} + file, target := "consumer/app.py", "provider/module.py" + g.AddNode(&graph.Node{ID: target, FilePath: target, Kind: graph.KindFile, RepoPrefix: "provider"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::module"}}) + cr := NewCrossRepo(g) + cr.buildReachableReposIndex(nil) + require.Empty(t, g.batches) + require.Zero(t, g.batchCalls, "no package IDs means no batch query, even an empty one") + require.Empty(t, g.reads) + require.Empty(t, cr.pythonReexportsByFile) + }) + } +} From 40cbfbaf1d7e87acdd56c3fbfb74798dc88a2a5f Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 16:55:52 -0700 Subject: [PATCH 110/111] fix(resolver): skip over-budget providers in a stable order Iterate provider repos in sorted order so a shared budget always skips the same provider, and name the skipped repo in the warning: its consumers stay stale until the next full pass. --- internal/resolver/cross_repo_incremental.go | 8 ++-- .../go_module_deferral_integration_test.go | 2 +- .../resolver/go_module_import_seed_test.go | 47 +++++++++++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 28a7f6e53..03e539c32 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -2,7 +2,9 @@ package resolver import ( "context" + "maps" "path" + "slices" "strings" "go.uber.org/zap" @@ -210,14 +212,14 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed // Import recovery has its own ceiling: a popular provider must not starve // incoming calls or the revision-keyed call-terminal retry above. A skipped // provider stays stale until a full pass; it must not reopen the revision. - for repo := range providerRepos { + for _, repo := range slices.Sorted(maps.Keys(providerRepos)) { ids := cr.goModuleImportIDs(repo) if len(ids) > importBudget.Remaining() { - cr.logger.Warn("cross-repo resolve: Go module imports deferred to full pass", zap.Int("imports", len(ids)), zap.Int("remaining", importBudget.Remaining())) + cr.logger.Warn("cross-repo resolve: provider consumers stay stale until next full pass", zap.String("repo", repo), zap.Int("imports", len(ids)), zap.Int("remaining", importBudget.Remaining())) continue } if err := importBudget.Charge(len(ids)); err != nil { - cr.logger.Warn("cross-repo resolve: Go module imports deferred to full pass", zap.Error(err)) + cr.logger.Warn("cross-repo resolve: provider consumers stay stale until next full pass", zap.String("repo", repo), zap.Error(err)) continue } for start := 0; start < len(ids); start += externalCallMutationChunk { diff --git a/internal/resolver/go_module_deferral_integration_test.go b/internal/resolver/go_module_deferral_integration_test.go index c3e8bd0b4..e39efd23f 100644 --- a/internal/resolver/go_module_deferral_integration_test.go +++ b/internal/resolver/go_module_deferral_integration_test.go @@ -121,7 +121,7 @@ func TestGoModuleProviderSkipsDoNotRunFullFallback(t *testing.T) { for _, file := range []string{"provider/lib.go", "provider/lib.go", "consumer/f0.go"} { require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{file}, nil)) } - require.Equal(t, 2, logs.FilterMessage("cross-repo resolve: Go module imports deferred to full pass").Len()) + require.Equal(t, 2, logs.FilterMessage("cross-repo resolve: provider consumers stay stale until next full pass").Len()) require.Zero(t, logs.FilterMessage("Go module import identity recheck deferred to full pass").Len()) require.Zero(t, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len()) require.Zero(t, g.rechecks, "two provider saves and a third Go save must not recheck all repositories") diff --git a/internal/resolver/go_module_import_seed_test.go b/internal/resolver/go_module_import_seed_test.go index 1f7eaf5f5..affd4b892 100644 --- a/internal/resolver/go_module_import_seed_test.go +++ b/internal/resolver/go_module_import_seed_test.go @@ -8,6 +8,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/graph/store_sqlite" @@ -286,3 +288,48 @@ func TestGoModuleManifestFrontiersRunRetry(t *testing.T) { } } } + +func TestGoModuleImportProviderBudgetOrder(t *testing.T) { + t.Cleanup(SetGoModuleImportRecoveryBudgetForTest(1)) + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + modules := map[string]string{"a-provider": "example.org/a", "b-provider": "example.org/b", "consumer": "example.org/app"} + cr := NewCrossRepo(g) + cr.SetGoModules(modules) + state := cr.goModuleImportState() + state.modules, state.complete, state.importsSeeded = modules, true, true + for i, repo := range []string{"a-provider", "b-provider"} { + file := fmt.Sprintf("consumer/f%d.go", i) + id := file + "::import::" + modules[repo] + g.AddBatch([]*graph.Node{ + {ID: repo + "/lib.go", Name: "lib.go", FilePath: repo + "/lib.go", Kind: graph.KindFile, RepoPrefix: repo, Language: "go"}, + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "go"}, + {ID: id, FilePath: file, StartLine: 1, Kind: graph.KindImport, RepoPrefix: "consumer", Language: "go", Meta: map[string]any{"path": modules[repo]}}, + }, []*graph.Edge{{From: file, FilePath: file, To: "dep::" + modules[repo], Kind: graph.EdgeImports, Line: 1}}) + state.imports[repo] = map[string]struct{}{id: {}} + } + core, logs := observer.New(zap.WarnLevel) + cr.SetLogger(zap.New(core)) + for range 8 { + for i, repo := range []string{"a-provider", "b-provider"} { + rows := outEdgesOfKind(g, fmt.Sprintf("consumer/f%d.go", i), graph.EdgeImports) + require.Len(t, rows, 1) + oldTo := rows[0].To + rows[0].To, rows[0].CrossRepo, rows[0].Meta = "dep::"+modules[repo], false, nil + g.ReindexEdges([]graph.EdgeReindex{{Edge: rows[0], OldTo: oldTo}}) + } + stats, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b-provider/lib.go", "a-provider/lib.go"}) + require.False(t, admission.Refused) + require.True(t, state.complete) + require.Equal(t, 1, stats.Resolved) + require.Equal(t, "a-provider/lib.go", outEdgesOfKind(g, "consumer/f0.go", graph.EdgeImports)[0].To) + require.Equal(t, "dep::example.org/b", outEdgesOfKind(g, "consumer/f1.go", graph.EdgeImports)[0].To) + warnings := logs.TakeAll() + require.Len(t, warnings, 1) + require.Equal(t, "cross-repo resolve: provider consumers stay stale until next full pass", warnings[0].Message) + require.Equal(t, "b-provider", warnings[0].ContextMap()["repo"]) + } + }) + } +} From 4998f597f091ceb6135edc05d90283cbefb9587d Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Mon, 26 Oct 2026 16:56:25 -0700 Subject: [PATCH 111/111] test(resolver): count all-repo rechecks by the exact binding kinds Share the recheck kind list between production and the counting stores, so a changed kind list fails the zero-recheck assertions instead of passing without measuring anything. --- internal/resolver/export_test.go | 11 ++++++++++- .../resolver/go_module_deferral_integration_test.go | 13 ++++++++++++- internal/resolver/go_module_imports_test.go | 13 ++++++++++++- internal/resolver/go_modules_cross_repo.go | 4 +++- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/internal/resolver/export_test.go b/internal/resolver/export_test.go index b5640c784..bf2bea459 100644 --- a/internal/resolver/export_test.go +++ b/internal/resolver/export_test.go @@ -1,6 +1,10 @@ package resolver -import "github.com/zzet/gortex/internal/graph" +import ( + "slices" + + "github.com/zzet/gortex/internal/graph" +) // SetGoModuleImportRecoveryBudgetForTest changes only the bounded import budget. // External resolver tests use it to exercise the real indexer continuation. @@ -15,3 +19,8 @@ func SetGoModuleImportRecoveryBudgetForTest(remaining int) (restore func()) { } return func() { newGoModuleImportRecoveryBudget = previous } } + +// IsGoModuleBindingRecheckForTest shares the production kind list with recorders. +func IsGoModuleBindingRecheckForTest(kinds []graph.NodeKind) bool { + return slices.Equal(kinds, goModuleBindingRecheckKinds) +} diff --git a/internal/resolver/go_module_deferral_integration_test.go b/internal/resolver/go_module_deferral_integration_test.go index e39efd23f..e5c4fb326 100644 --- a/internal/resolver/go_module_deferral_integration_test.go +++ b/internal/resolver/go_module_deferral_integration_test.go @@ -28,7 +28,7 @@ type goDeferralStore struct { } func (s *goDeferralStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { - if len(kinds) == 4 { + if resolver.IsGoModuleBindingRecheckForTest(kinds) { s.rechecks++ } if repos == nil && len(kinds) == 1 && kinds[0] == graph.KindImport { @@ -130,3 +130,14 @@ func TestGoModuleProviderSkipsDoNotRunFullFallback(t *testing.T) { }) } } + +func TestGoModuleDeferralCounterExactBindingKinds(t *testing.T) { + g := &goDeferralStore{Store: graph.New()} + for range g.RepoNodeIdentitiesSeq(nil, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { + } + require.Equal(t, 1, g.rechecks) + g.rechecks = 0 + for range g.RepoNodeIdentitiesSeq(nil, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindImport) { + } + require.Zero(t, g.rechecks, "four unrelated kinds must not count as the binding recheck") +} diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go index 5fdaa2fbb..dd93fced8 100644 --- a/internal/resolver/go_module_imports_test.go +++ b/internal/resolver/go_module_imports_test.go @@ -170,7 +170,7 @@ func (s *goImportCountingStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[g return graph.FileNodeIdentitiesSeq(s.Store, repos) } func (s *goImportCountingStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { - if len(kinds) == 4 { + if IsGoModuleBindingRecheckForTest(kinds) { s.bindingRechecks++ } return func(yield func(graph.RepoNodeIdentity) bool) { @@ -757,3 +757,14 @@ func TestGoModuleImportUsesBatchedFileAndStatsNodes(t *testing.T) { }) } } + +func TestGoModuleImportCounterExactBindingKinds(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + for range g.RepoNodeIdentitiesSeq(nil, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { + } + require.Equal(t, 1, g.bindingRechecks) + g.bindingRechecks = 0 + for range g.RepoNodeIdentitiesSeq(nil, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindImport) { + } + require.Zero(t, g.bindingRechecks, "four unrelated kinds must not count as the binding recheck") +} diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index 232a1e6a0..0aeac2542 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -611,6 +611,8 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { return nil } +var goModuleBindingRecheckKinds = []graph.NodeKind{graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile} + // Re-check only marked incoming calls into providers affected by identity edits. // The compact definition stream avoids loading a whole repo's nodes or edges. func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previous map[string]string, stats *CrossRepoStats, importsRestored bool) error { @@ -660,7 +662,7 @@ func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previo cr.resolveGoModuleEdgesLocked(edges, stats) ids = ids[:0] } - for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { + for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, goModuleBindingRecheckKinds...) { if err := ctx.Err(); err != nil { return err }