diff --git a/cmd/gortex/daemon_state.go b/cmd/gortex/daemon_state.go index 8f097b733..86f8adff0 100644 --- a/cmd/gortex/daemon_state.go +++ b/cmd/gortex/daemon_state.go @@ -367,6 +367,13 @@ func (frontier *warmupDeltaFrontier) invalidate() { func (frontier *warmupDeltaFrontier) record(result *indexer.IndexResult) { frontier.mu.Lock() defer frontier.mu.Unlock() + if result != nil { + for _, file := range result.CrossRepoResolveFiles { + if file != "" { + frontier.crossRepoFiles[file] = struct{}{} + } + } + } if result == nil || result.RepoPrefix == "" || result.FullRetrack || len(result.FailedFiles) > 0 { frontier.exact = false return @@ -374,11 +381,6 @@ func (frontier *warmupDeltaFrontier) record(result *indexer.IndexResult) { plan := frontier.plans[result.RepoPrefix] plan.Merge(result.DerivedInvalidation) frontier.plans[result.RepoPrefix] = plan - for _, file := range result.CrossRepoResolveFiles { - if file != "" { - frontier.crossRepoFiles[file] = struct{}{} - } - } } func (frontier *warmupDeltaFrontier) snapshot(changedRepos int, scopeUnknown bool) (map[string]indexer.DerivedInvalidationPlan, []string, bool) { diff --git a/cmd/gortex/daemon_warm_cargo_frontier_test.go b/cmd/gortex/daemon_warm_cargo_frontier_test.go new file mode 100644 index 000000000..bbb23a5b4 --- /dev/null +++ b/cmd/gortex/daemon_warm_cargo_frontier_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/indexer" +) + +func TestWarmupCargoConsumersStayOutsideDerivedPlan(t *testing.T) { + for _, full := range []bool{false, true} { + frontier := newWarmupDeltaFrontier() + frontier.record(&indexer.IndexResult{ + RepoPrefix: "provider", FullRetrack: full, + DerivedInvalidation: indexer.DerivedInvalidationPlan{Files: []string{"provider/src/lib.rs"}}, + CrossRepoResolveFiles: []string{"consumer/src/lib.rs"}, + }) + plans, files, exact := frontier.snapshot(1, false) + require.Contains(t, files, "consumer/src/lib.rs") + require.Equal(t, !full, exact) + for _, plan := range plans { + require.NotContains(t, plan.Files, "consumer/src/lib.rs") + } + } +} diff --git a/cmd/gortex/daemon_warm_python_frontier_test.go b/cmd/gortex/daemon_warm_python_frontier_test.go new file mode 100644 index 000000000..fc4a825bc --- /dev/null +++ b/cmd/gortex/daemon_warm_python_frontier_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/indexer" +) + +func TestWarmupPythonConsumersStayOutsideDerivedPlan(t *testing.T) { + for _, full := range []bool{false, true} { + frontier := newWarmupDeltaFrontier() + frontier.record(&indexer.IndexResult{ + RepoPrefix: "provider", FullRetrack: full, + DerivedInvalidation: indexer.DerivedInvalidationPlan{Files: []string{"provider/src/PIL/image.py"}}, + CrossRepoResolveFiles: []string{"consumer/app.py"}, + }) + plans, files, exact := frontier.snapshot(1, false) + require.Contains(t, files, "consumer/app.py") + require.Equal(t, !full, exact) + for _, plan := range plans { + require.NotContains(t, plan.Files, "consumer/app.py") + } + } +} diff --git a/docs/multi-repo.md b/docs/multi-repo.md index 259f5d57d..a6179f78c 100644 --- a/docs/multi-repo.md +++ b/docs/multi-repo.md @@ -126,6 +126,32 @@ the participating repositories' allow-lists. This keeps a repository from silently losing framework edges requested by a sibling while allowing a workspace where every repository opts out to avoid the scans entirely. +### Tracked module imports + +- Go imports can link to a representative source file in a tracked module. + The longest matching root `go.mod` path selects the package. Ambiguous providers + stay unlinked. Local `replace` paths must point to indexed source files. +- Rust `use` imports can link to a tracked library declared in `Cargo.toml`. + Dependency aliases, workspace dependencies, explicit workspace members and + `[lib]` names and paths are supported. Undeclared crates, ambiguous providers + and reserved names such as `std`, `core` and `crate` stay unlinked. +- Python imports can link to a tracked distribution declared in `pyproject.toml` + or `requirements*.txt`. Root and `src/` layouts support modules and regular + packages. A local top-level package blocks a foreign link. Ambiguous providers, + undeclared distributions and Python stdlib imports stay unlinked. + +These import-file links do not make every symbol in the provider reachable. +Calls still need an imported path or type. Versions and build configurations +are not verified. + +### Python package re-exports + +Calls through a declared, tracked Python distribution can follow one level of +relative `from .module import Name` or `from .module import Name as Alias` in its +package `__init__.py`. The bridge requires an exact imported binding and a unique +function or class in the provider. It does not follow wildcard imports, +conditional or function-local imports, or multi-level re-export chains. + ## Daemon tuning (optional) The background daemon writes JSON lines to `daemon.log` and rotates it while @@ -494,9 +520,14 @@ Scoped tool responses carry a `scope_applied` meta field plus a one-line widen h - Entry lookup respects `exports` keys, wildcard subpaths and `null` blocks. It probes the selected `import`/`default` target first, then a `types` condition when that file is not indexed. Without `exports`, it tries `types`, `main` and directory indexes. Unsupported-condition subpaths never guess a directory target. - Bare calls use the caller file's named import binding, including its local alias, not another same-named function anywhere in the provider. A per-binding import links to a symbol marked public by the extractor; otherwise it stays at `external::::` with its `npm_import` marker and creates no call binding. Linked imports retain their specifier, survive provider eviction as external imports, and are rechecked by whole-graph resolution. - **Limits:** Build-output-only entries, such as `main` and `types` pointing into excluded `dist/`, stay external unless an indexed target is declared. Gortex does not infer `src/` from `dist/`. The importing repository's own packages, `npm:` aliases and workspace paths that escape the repository are not linked by this pass. Sibling dependencies declared with `file:`, `link:` or `portal:` are not supported. Default-import bare calls have no exact binding evidence yet and stay unresolved. After an upgrade, JavaScript and TypeScript files are re-extracted once (extractor version 2), so existing imports acquire the specifier and alias metadata; their links appear after that pass. Namespace member calls, CommonJS `require` calls and package re-exports are not linked by this npm import pass. Export targets without `./` are accepted leniently, as in workspace resolution, even though Node rejects them. Call binding is file-scoped: a parameter or local callable with the imported name can bind to the import. + - Go source imports can link to a representative indexed package file. The `go.mod` dependency contract remains separate. Linked imports retain their package identity across provider saves. - Go package-selector calls such as `library.Hello()` use tracked roots' `go.mod` module paths, not checkout names. An import of `example.org/library/sub` can bind to `/sub/file.go::Hello` even when the provider prefix is `b`. - Precise own-module, vendor and staging paths take priority. Otherwise the longest matching module path wins. A primary/non-worktree checkout wins a tie with dedicated worktree copies; ties between independent primary checkouts are refused. Cross-workspace calls still require the caller's `cross_workspace_deps` declaration. - A cross-repo module binding carries confidence `0.6` and origin `ast_inferred`. It is a source hint, not proof that the tracked checkout matches the required version. The edge retains its symbol-bearing import identity so provider edits and full reindexes can rebind it without editing the caller. Module-map changes re-check existing bindings. - - **Limits:** Only root `go.mod` identities are used; nested modules and `go.work` selection are not resolved. Required versions and module/version replacements are not verified. A local `replace` whose destination is untracked prevents module-path binding instead of selecting another checkout with the same module name. Tracked local replacements remain source hints, not full Go build-list resolution. Calls on values typed from another module, such as `client.Do()` for a `library.Client`, can remain `unresolved::*.Do`; receiver-type linking is not implemented here. + - **Limits:** Only each tracked root's `go.mod` identity is used. Separately tracked nested modules can win by longest path; nested manifests within one tracked root and `go.work` selection are not resolved. Required versions and module/version replacements are not verified. A local `replace` whose destination is untracked prevents module-path binding instead of selecting another checkout with the same module name. Tracked local replacements remain source hints, not full Go build-list resolution. Calls on values typed from another module, such as `client.Do()` for a `library.Client`, stay unresolved unless another pass has precise evidence. Importing the module alone does not enable name-only receiver binding. + - Rust `use` imports link to the declared provider's library root or module file. Dependency, dev-dependency, build-dependency and target tables are read. Calls bind only to a unique imported item or receiver in that file. + - **Cargo limits:** This is not Cargo build resolution. Versions, features and target conditions are not evaluated. Absolute or escaping library paths, conflicting aliases and multiple eligible providers are refused. Re-exports and untyped receivers are not inferred. Module-qualified calls such as `use lib::util; util::action()` are not linked by this bridge. Only direct members of a single group bind, such as `use lib::util::{a, b}`. Nested members inside braces are not linked. Grouped and ungrouped member calls require an indexed file for the exact parent module; an import that falls back to an ancestor file does not authorize those calls. Existing links are not always revisited after declaration or provider-set changes; reindex the provider to refresh them. + - Python distribution imports link only after a consumer declaration and a unique eligible provider match. Qualified calls use the linked module file. Typed methods require a receiver name bound by `from X import Name [as Alias]` and a matching provider method in that file. Plain `import pkg.mod` and wildcard imports never link typed methods. A same-named type defined in the caller file refuses the link. Package calls can follow the one-level re-exports described above. + - **Python limits:** Distribution versions and arbitrary source layouts are not verified. Providers must expose indexed root or `src/` modules or regular packages. Namespace-only providers and dynamic imports are not linked. Local top-level names take priority, including namespace directories. A working link is not reconsidered when a second provider appears, its dependency declaration is dropped, or a new local top-level package shadows it. Reindex the consumer to refresh it. - **Impact analysis** — `explain_change_impact`, `verify_change`, and `get_test_targets` follow cross-repo edges automatically, grouping results by repository. - **Shared repos** — the same repo can appear in multiple projects with different reference tags. It's indexed once and shared across projects. diff --git a/internal/indexer/added_file_declaration_gate_test.go b/internal/indexer/added_file_declaration_gate_test.go new file mode 100644 index 000000000..cda7ddd8e --- /dev/null +++ b/internal/indexer/added_file_declaration_gate_test.go @@ -0,0 +1,156 @@ +package indexer + +import ( + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/parser/languages" + "path/filepath" + "strings" + "testing" +) + +func TestPythonAddedFileDeclarationGate(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'library'\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\nname = 'app'\n") + g := &pythonLookupScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: "external::pkg.extra", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + for range 2 { + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + } + require.Zero(t, g.importScans, "no declaration means no import scan") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['library']\n") + require.Equal(t, []string{"consumer/app.py"}, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + require.Equal(t, 1, g.importScans) + require.Zero(t, g.nodeReads, "admission must not build the provider lookup") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\nname = 'app'\n") + g.importScans = 0 + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/next.py"})) + require.Zero(t, g.importScans) + // Existing but unreadable snapshot evidence must open admission. + prior := readDiskFile + readDiskFile = func(file string) ([]byte, bool) { return nil, false } + t.Cleanup(func() { readDiskFile = prior }) + require.Equal(t, []string{"consumer/app.py"}, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + require.Equal(t, 1, g.importScans) +} + +func TestCargoAddedFileDeclarationGate(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'library'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n") + g := &cargoWarmScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "external::library/extra", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + for range 2 { + require.Empty(t, mi.cargoAddedFileConsumers("provider")) + } + require.Zero(t, g.lightScans) + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\nlibrary = '1'\n") + require.Equal(t, []string{"consumer/src/lib.rs"}, mi.cargoAddedFileConsumers("provider")) + require.Equal(t, 1, g.lightScans) + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n") + g.lightScans = 0 + require.Empty(t, mi.cargoAddedFileConsumers("provider")) + require.Zero(t, g.lightScans) + prior := readDiskFile + readDiskFile = func(file string) ([]byte, bool) { return nil, false } + t.Cleanup(func() { readDiskFile = prior }) + require.Equal(t, []string{"consumer/src/lib.rs"}, mi.cargoAddedFileConsumers("provider")) + require.Equal(t, 1, g.lightScans) +} + +func TestPythonAddedTestPackageDoesNotScan(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'library'\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['library']\n") + g := &pythonLookupScanStore{Store: graph.New()} + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/tests/__init__.py", "provider/src/test/helpers.py"})) + require.Zero(t, g.importScans) +} + +func TestAddedFileAfterDeclarationMatchesCold(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, lang := range []string{"python", "cargo"} { + t.Run(backend+"/"+lang, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + manifest, caller, added := "pyproject.toml", "app.py", "pkg/extra.py" + providerManifest, consumerManifest := "[project]\nname = 'library'\n", "[project]\nname = 'app'\n" + declaration := "dependencies = ['library']\n" + providerSource, consumerSource := "", "from pkg.extra import f\ndef run(): return f()\n" + if lang == "cargo" { + manifest, caller, added = "Cargo.toml", "src/lib.rs", "src/extra.rs" + providerManifest, consumerManifest = "[package]\nname = 'library'\n", "[package]\nname = 'app'\n" + declaration = "[dependencies]\nlibrary = '1'\n" + providerSource, consumerSource = "pub mod extra;\n", "use library::extra::f;\npub fn run() { f(); }\n" + writeWorkspaceFile(t, provider, "src/lib.rs", providerSource) + } else { + writeWorkspaceFile(t, provider, "pkg/__init__.py", providerSource) + } + writeWorkspaceFile(t, provider, manifest, providerManifest) + writeWorkspaceFile(t, consumer, manifest, consumerManifest) + writeWorkspaceFile(t, consumer, caller, consumerSource) + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + mi.registry.Register(languages.NewRustExtractor()) + _, err := mi.IndexAll() + require.NoError(t, err) + for _, target := range revisionCallTargets(g, "consumer/"+caller, graph.EdgeImports) { + require.False(t, strings.HasPrefix(target, "provider/")) + } + // The declaration changes without recreating the MultiIndexer or resolver registry. + writeWorkspaceFile(t, consumer, manifest, consumerManifest+declaration) + if lang == "cargo" { + providerSource = "pub fn f() {}\n" + } else { + providerSource = "def f(): pass\n" + } + writeWorkspaceFile(t, provider, added, providerSource) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, added)}) + require.NoError(t, err) + targets := revisionCallTargets(g, "consumer/"+caller, graph.EdgeImports) + require.Contains(t, targets, "provider/"+added) + cold, coldGraph := newPythonBridgeIdentityIndexer(t, backend, entries) + cold.registry.Register(languages.NewRustExtractor()) + _, err = cold.IndexAll() + require.NoError(t, err) + require.ElementsMatch(t, revisionCallTargets(coldGraph, "consumer/"+caller, graph.EdgeImports), targets) + require.ElementsMatch(t, revisionCallTargets(coldGraph, "consumer/"+caller+"::run", graph.EdgeCalls), revisionCallTargets(g, "consumer/"+caller+"::run", graph.EdgeCalls)) + // Removal closes admission for the next addition. Full/retrack recovery is separate. + writeWorkspaceFile(t, consumer, manifest, consumerManifest) + if lang == "cargo" { + require.Empty(t, mi.cargoAddedFileConsumers("provider")) + } else { + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/next.py"})) + } + require.NoError(t, mi.GetIndexer("provider").IndexFile(filepath.Join(provider, added))) + }) + } + } +} + +func TestCargoRootOwnWorkspaceDependencyIsNotBridge(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + own := "[package]\nname='app'\n[workspace]\nmembers=['member']\n[dependencies]\nmember={path='member'}\n" + writeWorkspaceFile(t, first, "Cargo.toml", own) + writeWorkspaceFile(t, first, "member/Cargo.toml", "[package]\nname='member'\n") + writeWorkspaceFile(t, second, "Cargo.toml", "[package]\nname='other'\n") + g := &cargoWarmScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "first/src/lib.rs", FilePath: "first/src/lib.rs", To: "external::other/extra", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"first": {RootPath: first}, "second": {RootPath: second}}} + require.False(t, mi.cargoManifestSnapshot().cargoDeclared) + require.Empty(t, mi.cargoAddedFileConsumers("first")) + require.Zero(t, g.lightScans, "a local workspace dependency is not a tracked bridge") + writeWorkspaceFile(t, first, "Cargo.toml", own+"other='1'\n") + require.True(t, mi.cargoManifestSnapshot().cargoDeclared) + require.Equal(t, []string{"first/src/lib.rs"}, mi.cargoAddedFileConsumers("second")) + require.Equal(t, 1, g.lightScans) + writeWorkspaceFile(t, first, "Cargo.toml", own) + require.False(t, mi.cargoManifestSnapshot().cargoDeclared) + g.lightScans = 0 + require.Empty(t, mi.cargoAddedFileConsumers("first")) + require.Zero(t, g.lightScans) +} diff --git a/internal/indexer/affected_by_bound_test.go b/internal/indexer/affected_by_bound_test.go index 812e5518c..062a1f5c8 100644 --- a/internal/indexer/affected_by_bound_test.go +++ b/internal/indexer/affected_by_bound_test.go @@ -387,7 +387,7 @@ func TestDeletionShapedStagesCarryNoInEdges(t *testing.T) { stages := []*incrementalBatchStage{{graphPath: restubDefPath, priorNodes: prior}} view := loadIncrementalPriorView(st, stages) - carried, _ := restubIncomingRefsFromView(st, stages, view) + carried, _, _, _ := restubIncomingRefsFromView(st, stages, view) require.Empty(t, carried, "a deletion stage must park every in-edge; a carried edge has no re-state path here") assert.Equal(t, 1, st.restubWrites("Foo"), diff --git a/internal/indexer/bridge_import_restub.go b/internal/indexer/bridge_import_restub.go new file mode 100644 index 000000000..3f41b8942 --- /dev/null +++ b/internal/indexer/bridge_import_restub.go @@ -0,0 +1,38 @@ +package indexer + +import "github.com/zzet/gortex/internal/graph" + +// Repository parking can only change foreign imports/calls. Read their compact +// keys first, then hydrate those sites after the identity cursor has closed. +func foreignBridgeIncoming(g graph.Store, repo string, ids []string) map[string][]*graph.Edge { + identities := graph.InEdgeIdentitiesByNodeIDs(g, ids) + selected := map[graph.EdgeIdentity]bool{} + seen := map[graph.EdgeSite]bool{} + var sites []graph.EdgeSite + for _, rows := range identities { + for _, row := range rows { + if graph.RepoPrefixOfID(row.From) == repo || row.Kind != graph.EdgeImports && row.Kind != graph.EdgeCalls { + continue + } + selected[row] = true + site := graph.EdgeSite{From: row.From, Line: row.Line, Kind: row.Kind} + if !seen[site] { + seen[site] = true + sites = append(sites, site) + } + } + } + incoming := map[string][]*graph.Edge{} + for start := 0; start < len(sites); start += 512 { + page := sites[start:min(start+512, len(sites))] + rows := g.GetEdgeCandidates(nil, page) + for _, site := range page { + for _, edge := range rows.Site(site.From, site.Line, site.Kind) { + if selected[graph.EdgeIdentityFor(edge)] { + incoming[edge.To] = append(incoming[edge.To], edge) + } + } + } + } + return incoming +} diff --git a/internal/indexer/bridge_incoming_work_test.go b/internal/indexer/bridge_incoming_work_test.go new file mode 100644 index 000000000..870285b18 --- /dev/null +++ b/internal/indexer/bridge_incoming_work_test.go @@ -0,0 +1,65 @@ +package indexer + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +type bridgeIncomingCountStore struct { + graph.Store + fullReads, lightReads, hydrated int +} + +func (s *bridgeIncomingCountStore) GetInEdgesByNodeIDs(ids []string) map[string][]*graph.Edge { + s.fullReads++ + return s.Store.GetInEdgesByNodeIDs(ids) +} +func (s *bridgeIncomingCountStore) GetInEdgeIdentitiesByNodeIDs(ids []string) map[string][]graph.EdgeIdentity { + s.lightReads++ + return graph.InEdgeIdentitiesByNodeIDs(s.Store, ids) +} +func (s *bridgeIncomingCountStore) GetEdgeCandidates(endpoints []graph.EdgeEndpoint, sites []graph.EdgeSite) graph.EdgeCandidateSet { + s.hydrated += len(sites) + return s.Store.GetEdgeCandidates(endpoints, sites) +} + +func TestBridgeRepoRestubHydratesOnlyForeignSites(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, lang := range []string{"python", "cargo"} { + t.Run(backend+"/"+lang, func(t *testing.T) { + g := &bridgeIncomingCountStore{Store: cargoWarmStore(t, backend)} + file, importKey, callKey := "provider/mod.py", resolver.PythonImportStubMeta, resolver.PythonCallStubMeta + restub := restubTrackedPythonRepoIncoming + if lang == "cargo" { + file, importKey, callKey, restub = "provider/mod.rs", resolver.CargoImportStubMeta, resolver.CargoCallStubMeta, restubTrackedCargoRepoIncoming + } + target := file + "::f" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "provider"}) + g.AddNode(&graph.Node{ID: target, FilePath: file, Kind: graph.KindFunction, RepoPrefix: "provider"}) + for i := range 40 { + g.AddEdge(&graph.Edge{From: fmt.Sprintf("provider/local%d::run", i), To: target, FilePath: "provider/local", Kind: graph.EdgeCalls, CrossRepo: true, Meta: map[string]any{callKey: "unresolved::f"}}) + g.AddEdge(&graph.Edge{From: fmt.Sprintf("foreign/other%d", i), To: file, FilePath: "foreign/other", Kind: graph.EdgeContains}) + } + g.AddEdge(&graph.Edge{From: "consumer/app", To: file, FilePath: "consumer/app", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{importKey: "external::pkg.mod"}}) + g.AddEdge(&graph.Edge{From: "consumer/app::run", To: target, FilePath: "consumer/app", Kind: graph.EdgeCalls, Line: 2, CrossRepo: true, ConfidenceLabel: "high", Meta: map[string]any{callKey: "unresolved::f"}}) + g.AddEdge(&graph.Edge{From: "other/app", To: file, FilePath: "other/app", Kind: graph.EdgeImports, Line: 3, CrossRepo: true}) + require.Equal(t, []string{"consumer/app"}, restub(g, "provider")) + require.Zero(t, g.fullReads, "repository parking must not fetch full inbound adjacency") + require.Positive(t, g.lightReads) + require.Equal(t, 3, g.hydrated, "only foreign import/call sites need metadata") + require.Equal(t, file, g.GetOutEdges("other/app")[0].To) + require.Equal(t, target, g.GetOutEdges("provider/local0::run")[0].To) + call := g.GetOutEdges("consumer/app::run")[0] + require.Equal(t, "unresolved::f", call.To) + require.False(t, call.CrossRepo) + require.Empty(t, call.ConfidenceLabel) + require.True(t, graph.HasRestubProvenance(call)) + require.Empty(t, restub(g, "provider")) + }) + } + } +} diff --git a/internal/indexer/bridge_manifest_snapshot.go b/internal/indexer/bridge_manifest_snapshot.go new file mode 100644 index 000000000..445be546a --- /dev/null +++ b/internal/indexer/bridge_manifest_snapshot.go @@ -0,0 +1,281 @@ +package indexer + +import ( + "maps" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/zzet/gortex/internal/resolver" +) + +var readBridgeManifestDir = os.ReadDir + +// A file-set revision changes only when accepted files are added, removed, or +// replaced as a full inventory. Ordinary symbol/edge edits keep local tops valid. +type bridgeManifestInventory struct { + indexer *Indexer + revision uint64 + workspace string +} + +type bridgeManifestStamp struct { + size, modified int64 + kind os.FileMode + exists, valid bool +} + +func statBridgeManifest(file string) bridgeManifestStamp { + info, err := os.Stat(file) + if os.IsNotExist(err) { + return bridgeManifestStamp{valid: true} + } + if err != nil { + return bridgeManifestStamp{} + } + return bridgeManifestStamp{size: info.Size(), modified: info.ModTime().UnixNano(), kind: info.Mode().Type(), exists: true, valid: true} +} + +type bridgeManifestSnapshot struct { + revision uint64 + roots map[string]string + inventory map[string]bridgeManifestInventory + trees map[string]manifestTree + stampedAt time.Time + reusable bool + // Cargo discovers nearest manifest boundaries on demand after publication. + mu sync.Mutex + files map[string]bridgeManifestStamp + unknown bool + python *resolver.PythonPackageLookup + pythonBuilt bool + pythonEntries map[string]*pythonManifestEntry + cargo resolver.CargoCrateLookup + cargoDeclared bool +} + +// newBridgeManifestSnapshot captures config revision, repo roots and file-set +// revisions before manifest reads. Parsed views survive resolver passes only +// while these inputs and every watched size/mtime/mode stamp remain unchanged. +// A positive stamp must be strictly older than stampedAt - manifestRacyWindow; +// unknown evidence, source-backed trees and missing inventory owners forbid reuse. +func (mi *MultiIndexer) newBridgeManifestSnapshot() *bridgeManifestSnapshot { + s := &bridgeManifestSnapshot{ + roots: map[string]string{}, inventory: map[string]bridgeManifestInventory{}, + trees: map[string]manifestTree{}, files: map[string]bridgeManifestStamp{}, + stampedAt: time.Now(), reusable: true, + } + if mi.configMgr != nil { + s.revision = mi.configMgr.Revision() + } + for repo, meta := range mi.AllMetadata() { + if meta == nil || meta.RootPath == "" { + continue + } + s.roots[repo] = meta.RootPath + idx := mi.GetIndexer(repo) + if idx == nil { + // A hand-built registry cannot certify graph file inventory changes. + s.reusable = false + } else { + s.inventory[repo] = bridgeManifestInventory{idx, idx.fileSetRevision.Load(), idx.WorkspaceID()} + } + tree := mi.manifestTreeFor(repo, meta.RootPath) + if tree.sourced() { + s.reusable = false + } + s.trees[repo] = &bridgeManifestTree{manifestTree: tree, snapshot: s} + } + return s +} + +func (s *bridgeManifestSnapshot) matches(next *bridgeManifestSnapshot) bool { + if s == nil || s.revision != next.revision || !maps.Equal(s.roots, next.roots) || !maps.Equal(s.inventory, next.inventory) || !next.reusable { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if !s.reusable || s.unknown { + return false + } + for file, stamp := range s.files { + // A recently written file can change again without changing size/mtime on + // a coarse filesystem clock. Never certify such an observation as stable. + if !stamp.valid || stamp.exists && !time.Unix(0, stamp.modified).Before(s.stampedAt.Add(-manifestRacyWindow)) || statBridgeManifest(file) != stamp { + return false + } + } + return true +} + +func (s *bridgeManifestSnapshot) watch(file string) bridgeManifestStamp { + s.mu.Lock() + defer s.mu.Unlock() + stamp, seen := s.files[file] + if !seen { + stamp = statBridgeManifest(file) + s.files[file] = stamp + if !stamp.valid { + s.unknown = true + } + } + return stamp +} + +func rejectBridgeManifest(tree manifestTree) { + if watched, ok := tree.(*bridgeManifestTree); ok { + watched.snapshot.mu.Lock() + watched.snapshot.unknown = true + watched.snapshot.mu.Unlock() + } +} + +// Watch the same files/glob directory prefixes the native builders use. This +// records stat evidence only, never file bytes or directory content hashes. +type bridgeManifestTree struct { + manifestTree + snapshot *bridgeManifestSnapshot +} + +func (t *bridgeManifestTree) watch(rel string) bridgeManifestStamp { + if t.sourced() { + return bridgeManifestStamp{} + } + return t.snapshot.watch(filepath.Join(t.root(), filepath.FromSlash(rel))) +} +func (t *bridgeManifestTree) readFile(rel string) ([]byte, bool) { + stamp := t.watch(rel) + src, ok := t.manifestTree.readFile(rel) + if !ok && stamp.exists { + rejectBridgeManifest(t) + } + return src, ok +} +func (t *bridgeManifestTree) isFile(rel string) bool { + t.watch(rel) + return t.manifestTree.isFile(rel) +} +func (t *bridgeManifestTree) watchGlob(pattern string) { + if t.sourced() { + return + } + watchDir := func(rel string) { + stamp := t.watch(rel) + if stamp.exists && stamp.kind.IsDir() { + if _, err := readBridgeManifestDir(filepath.Join(t.root(), filepath.FromSlash(rel))); err != nil { + rejectBridgeManifest(t) + } + } + } + watchDir(".") + parts := strings.Split(filepath.ToSlash(pattern), "/") + for depth := 1; depth <= len(parts); depth++ { + prefix := strings.Join(parts[:depth], "/") + if !strings.ContainsAny(prefix, `*?[\`) { + watchDir(prefix) + } + for _, dir := range t.manifestTree.matchDirs(prefix) { + watchDir(dir) + } + } +} +func (t *bridgeManifestTree) matchFiles(pattern string) []string { + t.watchGlob(pattern) + return t.manifestTree.matchFiles(pattern) +} +func (t *bridgeManifestTree) matchDirs(pattern string) []string { + t.watchGlob(pattern) + return t.manifestTree.matchDirs(pattern) +} + +func (mi *MultiIndexer) pythonManifestSnapshot() *bridgeManifestSnapshot { + next := mi.newBridgeManifestSnapshot() + mi.bridgeManifestMu.Lock() + defer mi.bridgeManifestMu.Unlock() + if mi.pythonPackageCache.matches(next) { + return mi.pythonPackageCache + } + next.pythonEntries = readPythonPackageEntries(next.trees) + mi.pythonPackageCache = next + return next +} + +// pythonPackageLookup builds providers on first use of the reusable manifest +// snapshot. Providers require indexed root/src modules or regular packages; +// consumer-local indexed namespaces block foreign bindings. +func (mi *MultiIndexer) pythonPackageLookup() *resolver.PythonPackageLookup { + snapshot := mi.pythonManifestSnapshot() + mi.bridgeManifestMu.Lock() + defer mi.bridgeManifestMu.Unlock() + if !snapshot.pythonBuilt { + snapshot.python = mi.buildPythonPackageLookup(snapshot.trees, snapshot.pythonEntries) + snapshot.pythonBuilt = true + } + return snapshot.python +} + +func (mi *MultiIndexer) cargoManifestSnapshot() *bridgeManifestSnapshot { + next := mi.newBridgeManifestSnapshot() + mi.bridgeManifestMu.Lock() + defer mi.bridgeManifestMu.Unlock() + if mi.cargoPackageCache.matches(next) { + return mi.cargoPackageCache + } + next.cargo, next.cargoDeclared = mi.buildCargoCrateLookup(next.trees) + mi.cargoPackageCache = next + return next +} + +func (mi *MultiIndexer) cargoCrateLookup() resolver.CargoCrateLookup { + // Direct recovery callers need to know whether a bridge is possible before + // scanning imports. Keep their cheap root admission and defer manifest reads. + roots := 0 + for repo, meta := range mi.AllMetadata() { + if meta != nil && meta.RootPath != "" && mi.manifestTreeFor(repo, meta.RootPath).isFile("Cargo.toml") { + roots++ + } + } + if roots < 2 { + return nil + } + return mi.lazyCargoCrateLookup() +} + +// lazyCargoCrateLookup defers manifest parsing until the first lookup. +// cargoCrateLookup performs cheap root-count admission before installing it. +func (mi *MultiIndexer) lazyCargoCrateLookup() resolver.CargoCrateLookup { + var once sync.Once + var lookup resolver.CargoCrateLookup + return func(file, name string) []resolver.CargoCrate { + once.Do(func() { lookup = mi.cargoManifestSnapshot().cargo }) + if lookup == nil { + return nil + } + return lookup(file, name) + } +} + +func (s *bridgeManifestSnapshot) hasUnknownManifest() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.unknown +} + +func (s *bridgeManifestSnapshot) declaresPythonBridge() bool { + if s.hasUnknownManifest() { + return true + } + for provider, pkg := range s.pythonEntries { + if pkg.name == "" { + continue + } + for caller, consumer := range s.pythonEntries { + if caller != provider && consumer.deps[pkg.name] { + return true + } + } + } + return false +} diff --git a/internal/indexer/bridge_manifest_snapshot_test.go b/internal/indexer/bridge_manifest_snapshot_test.go new file mode 100644 index 000000000..6487e112e --- /dev/null +++ b/internal/indexer/bridge_manifest_snapshot_test.go @@ -0,0 +1,257 @@ +package indexer + +import ( + "io/fs" + "iter" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "go.uber.org/zap" +) + +type bridgeSnapshotCountStore struct { + graph.Store + files, nodes int +} + +func (s *bridgeSnapshotCountStore) GetNode(id string) *graph.Node { + s.nodes++ + return s.Store.GetNode(id) +} +func (s *bridgeSnapshotCountStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { + s.files++ + return graph.FileNodeIdentitiesSeq(s.Store, repos) +} + +func backdateBridgeFixture(t *testing.T, root string) { + t.Helper() + old := time.Now().Add(-time.Hour) + require.NoError(t, filepath.WalkDir(root, func(file string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + return os.Chtimes(file, old, old) + })) +} + +func TestPythonManifestSnapshotReusesParsedLookup(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + writeWorkspaceFile(t, consumer, "app.py", "import PIL\n") + g := &bridgeSnapshotCountStore{Store: graph.New()} + for repo, files := range map[string][]string{"provider": {"src/PIL/__init__.py"}, "consumer": {"app.py"}} { + for _, file := range files { + g.AddNode(&graph.Node{ID: repo + "/" + file, FilePath: repo + "/" + file, Kind: graph.KindFile, RepoPrefix: repo}) + } + } + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{}, indexers: map[string]*Indexer{}} + for repo, root := range map[string]string{"provider": provider, "consumer": consumer} { + idx := &Indexer{graph: g, repoPrefix: repo, logger: zap.NewNop()} + idx.SetRootPath(root) + initial := map[string]int64{} + if repo == "consumer" { + initial["app.py"] = 1 + } + idx.SetFileMtimes(initial) + mi.indexers[repo] = idx + mi.repos[repo] = &RepoMetadata{RootPath: root} + backdateBridgeFixture(t, root) + } + priorRead := readDiskFile + reads := 0 + readDiskFile = func(file string) ([]byte, bool) { reads++; return priorRead(file) } + t.Cleanup(func() { readDiskFile = priorRead }) + first := mi.pythonPackageLookup() + require.NotNil(t, first) + require.Len(t, first.Providers("consumer", "PIL"), 1) + reads, g.files, g.nodes = 0, 0, 0 + second := mi.pythonPackageLookup() + require.NotNil(t, second) + require.Len(t, second.Providers("consumer", "PIL"), 1) + require.Zero(t, reads, "unchanged pass must reuse parsed manifests") + require.Zero(t, g.nodes, "unchanged provider layout must not be probed") + require.Zero(t, g.files, "unchanged consumer inventory must not be scanned again") + // A structural edit that preserves the file set keeps the snapshot. + mi.indexers["consumer"].recordFileMtimeValue("app.py", 1) + second = mi.pythonPackageLookup() + require.Len(t, second.Providers("consumer", "PIL"), 1) + // A new local namespace blocks the foreign package on the next pass. + g.AddNode(&graph.Node{ID: "consumer/src/PIL/local.py", FilePath: "consumer/src/PIL/local.py", Kind: graph.KindFile, RepoPrefix: "consumer"}) + mi.indexers["consumer"].recordFileMtimeValue("src/PIL/local.py", 1) + require.Empty(t, mi.pythonPackageLookup().Providers("consumer", "PIL")) +} + +func TestCargoManifestSnapshotReusesParsedLookup(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'pillow'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\npillow = '1'\n") + mi := &MultiIndexer{graph: graph.New(), repos: map[string]*RepoMetadata{}, indexers: map[string]*Indexer{}} + for repo, root := range map[string]string{"provider": provider, "consumer": consumer} { + idx := &Indexer{} + idx.SetRootPath(root) + mi.indexers[repo] = idx + mi.repos[repo] = &RepoMetadata{RootPath: root} + backdateBridgeFixture(t, root) + } + priorRead := readDiskFile + reads := 0 + readDiskFile = func(file string) ([]byte, bool) { reads++; return priorRead(file) } + t.Cleanup(func() { readDiskFile = priorRead }) + first := mi.lazyCargoCrateLookup() + require.NotNil(t, first) + require.Zero(t, reads) + require.Len(t, first("consumer/src/lib.rs", "pillow"), 1) + reads = 0 + second := mi.cargoCrateLookup() + require.NotNil(t, second) + require.Len(t, second("consumer/src/lib.rs", "pillow"), 1) + require.Zero(t, reads, "unchanged Cargo lookup must reuse parsed manifests") + // A same-size dependency rename must invalidate the snapshot. + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\nmissing = '1'\n") + require.Empty(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow")) + // Restoring the manifest restores the dependency without a restart. + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\npillow = '1'\n") + require.Len(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow"), 1) +} + +func TestBridgeManifestSnapshotRacyBoundary(t *testing.T) { + stampTime := time.Unix(1700000000, 0) + for _, tc := range []struct { + name string + offset time.Duration + reusable bool + }{ + {"old", -time.Second, true}, {"boundary", 0, false}, {"recent", time.Second, false}, + } { + t.Run(tc.name, func(t *testing.T) { + file := filepath.Join(t.TempDir(), "pyproject.toml") + require.NoError(t, os.WriteFile(file, []byte("aaaa"), 0600)) + modified := stampTime.Add(-manifestRacyWindow).Add(tc.offset) + require.NoError(t, os.Chtimes(file, modified, modified)) + s := &bridgeManifestSnapshot{stampedAt: stampTime, reusable: true, files: map[string]bridgeManifestStamp{}} + s.watch(file) + // Same size/mtime can hide a second write on a coarse clock. + require.NoError(t, os.WriteFile(file, []byte("bbbb"), 0600)) + require.NoError(t, os.Chtimes(file, modified, modified)) + require.Equal(t, tc.reusable, s.matches(s)) + s.unknown = true + require.False(t, s.matches(s), "a failed read must not be reused") + }) + } +} + +func TestCargoManifestSnapshotWorkspaceAndBoundaryChanges(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[workspace]\nmembers = ['crates/*']\n") + require.NoError(t, os.MkdirAll(filepath.Join(provider, "crates/empty"), 0700)) + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\npillow = '1'\n") + mi := &MultiIndexer{repos: map[string]*RepoMetadata{}, indexers: map[string]*Indexer{}} + for repo, root := range map[string]string{"provider": provider, "consumer": consumer} { + idx := &Indexer{} + idx.SetRootPath(root) + mi.indexers[repo] = idx + mi.repos[repo] = &RepoMetadata{RootPath: root} + backdateBridgeFixture(t, root) + } + require.Empty(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow")) + writeWorkspaceFile(t, provider, "crates/empty/Cargo.toml", "[package]\nname = 'pillow'\n") + require.Len(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow"), 1) + backdateBridgeFixture(t, provider) + backdateBridgeFixture(t, consumer) + require.Len(t, mi.cargoCrateLookup()("consumer/tools/child/src/lib.rs", "pillow"), 1) + // A newly introduced nonmember boundary blocks inheritance from the root. + writeWorkspaceFile(t, consumer, "tools/child/Cargo.toml", "[package]\nname = 'child'\n") + require.Empty(t, mi.cargoCrateLookup()("consumer/tools/child/src/lib.rs", "pillow")) + require.NoError(t, os.Remove(filepath.Join(consumer, "tools/child/Cargo.toml"))) + require.Len(t, mi.cargoCrateLookup()("consumer/tools/child/src/lib.rs", "pillow"), 1) +} + +func TestBridgeManifestSnapshotInventoryAndRoots(t *testing.T) { + idx := &Indexer{graph: graph.New(), repoPrefix: "repo", logger: zap.NewNop()} + idx.SetRootPath(t.TempDir()) + mi := &MultiIndexer{repos: map[string]*RepoMetadata{"repo": {RootPath: idx.RootPath()}}, indexers: map[string]*Indexer{"repo": idx}} + idx.SetFileMtimes(map[string]int64{"a.py": 1}) + s := mi.newBridgeManifestSnapshot() + idx.recordFileMtimeValue("a.py", 2) + require.True(t, s.matches(mi.newBridgeManifestSnapshot()), "ordinary edits preserve file inventory") + idx.recordFileMtimeValue("b.py", 2) + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "additions change inventory") + s = mi.newBridgeManifestSnapshot() + idx.invalidateColdFileMtimes([]string{"b.py"}) + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "removals change inventory") + s = mi.newBridgeManifestSnapshot() + idx.SetFileMtimes(map[string]int64{"a.py": 2}) + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "full rebuild invalidates provider identities") + s = mi.newBridgeManifestSnapshot() + mi.indexers["repo"] = &Indexer{} + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "retrack replaces the owner") + s = mi.newBridgeManifestSnapshot() + mi.repos["repo"] = &RepoMetadata{RootPath: t.TempDir()} + require.False(t, s.matches(mi.newBridgeManifestSnapshot()), "a new root is not the old snapshot") +} + +func TestBridgeManifestSnapshotSourceAuthority(t *testing.T) { + provider, consumer, committed := t.TempDir(), t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "PIL/__init__.py", "") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\n[dependencies]\npillow = '1'\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + writeWorkspaceFile(t, committed, "Cargo.toml", "[package]\nname = 'app'\n") + writeWorkspaceFile(t, committed, "pyproject.toml", "[project]\nname = 'app'\n") + g := graph.New() + g.AddNode(&graph.Node{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", RepoPrefix: "provider", Kind: graph.KindFile}) + idx := &Indexer{} + idx.SetRootPath(consumer) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}, indexers: map[string]*Indexer{"consumer": idx}} + require.Len(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow"), 1) + require.NotNil(t, mi.pythonPackageLookup()) + idx.SetContentSource(snapshotOf(t, committed)) + require.Empty(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow")) + require.Nil(t, mi.pythonPackageLookup(), "dirty dependency must not escape the installed source") + idx.SetContentSource(nil) + require.Len(t, mi.cargoCrateLookup()("consumer/src/lib.rs", "pillow"), 1) + require.NotNil(t, mi.pythonPackageLookup()) +} + +func TestPythonManifestSnapshotFailedGlobIsUnknown(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname='library'\n") + writeWorkspaceFile(t, consumer, "requirements-private.txt", "library\n") + g := &pythonLookupScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeImports, To: "external::pkg.extra"}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + priorRead := readDiskFile + readDiskFile = func(file string) ([]byte, bool) { + if filepath.Base(file) == "requirements-private.txt" { + return nil, false + } + return priorRead(file) + } + t.Cleanup(func() { readDiskFile = priorRead }) + // stat can succeed even though enumeration fails; mirror Glob's empty result. + priorDir := readBridgeManifestDir + readBridgeManifestDir = func(dir string) ([]os.DirEntry, error) { + if dir == consumer { + return nil, fs.ErrPermission + } + return priorDir(dir) + } + t.Cleanup(func() { readBridgeManifestDir = priorDir }) + snapshot := mi.newBridgeManifestSnapshot() + snapshot.trees["consumer"] = &bridgeManifestTree{manifestTree: failingGlobManifestTree{manifestTree: newDiskManifestTree(consumer)}, snapshot: snapshot} + snapshot.pythonEntries = readPythonPackageEntries(snapshot.trees) + require.True(t, snapshot.hasUnknownManifest(), "failed glob cannot certify an absent declaration") + require.True(t, snapshot.declaresPythonBridge()) +} + +type failingGlobManifestTree struct{ manifestTree } + +func (t failingGlobManifestTree) matchFiles(string) []string { return nil } +func (t failingGlobManifestTree) matchDirs(string) []string { return nil } diff --git a/internal/indexer/cargo_crate.go b/internal/indexer/cargo_crate.go new file mode 100644 index 000000000..e38b5e9e7 --- /dev/null +++ b/internal/indexer/cargo_crate.go @@ -0,0 +1,196 @@ +package indexer + +import ( + "path" + "strings" + "sync" + + "github.com/pelletier/go-toml/v2" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/modules" + "github.com/zzet/gortex/internal/resolver" +) + +func cargoTable(m map[string]any, key string) map[string]any { + table, _ := m[key].(map[string]any) + return table +} + +func readCargoManifest(tree manifestTree, dir string) map[string]any { + src, ok := tree.readFile(path.Join(dir, "Cargo.toml")) + if !ok { + return nil + } + var manifest map[string]any + if toml.Unmarshal(src, &manifest) != nil { + rejectBridgeManifest(tree) + return nil + } + return manifest +} + +// buildCargoCrateLookup parses root/member manifests for one reusable snapshot. +// bridgeManifestSnapshot owns cross-pass invalidation. Only explicit workspace +// member globs are expanded; Cargo is never executed. +func (mi *MultiIndexer) buildCargoCrateLookup(trees map[string]manifestTree) (resolver.CargoCrateLookup, bool) { + type crate struct { + dir, name, root string + deps map[string]string + imports map[string][]resolver.CargoCrate + } + roots := map[string]manifestTree{} + for prefix, tree := range trees { + if tree.isFile("Cargo.toml") { + roots[prefix] = tree + } + } + if len(roots) < 2 { + return nil, false + } + owners := map[string]*crate{} + packages := map[string][]*crate{} + ownerCache := map[string]*crate{} + var ownerMu sync.Mutex + + for prefix, tree := range roots { + manifest := readCargoManifest(tree, ".") + src, _ := tree.readFile("Cargo.toml") + members, exclude := modules.ParseCargoWorkspace(src) + dirs := []string{"."} + skip := map[string]bool{".": true} + for _, pattern := range exclude { + for _, dir := range tree.matchDirs(pattern) { + skip[dir] = true + } + } + for _, pattern := range members { + for _, dir := range tree.matchDirs(pattern) { + if !path.IsAbs(dir) && !hasDotDotSegment(dir) && !skip[dir] && tree.isFile(path.Join(dir, "Cargo.toml")) { + dirs = append(dirs, dir) + skip[dir] = true + } + } + } + inherited := cargoTable(cargoTable(manifest, "workspace"), "dependencies") + for _, dir := range dirs { + m := manifest + if dir != "." { + m = readCargoManifest(tree, dir) + } + if m == nil { + continue + } + pkg, _ := cargoTable(m, "package")["name"].(string) + if pkg == "" { + continue + } + lib := cargoTable(m, "lib") + crateName, _ := lib["name"].(string) + if crateName == "" { + crateName = strings.ReplaceAll(pkg, "-", "_") + } + libPath, _ := lib["path"].(string) + if libPath == "" { + libPath = "src/lib.rs" + } + if path.IsAbs(libPath) || hasDotDotSegment(libPath) { + continue + } + deps := map[string]string{} + tables := []map[string]any{m} + for _, target := range cargoTable(m, "target") { + if table, ok := target.(map[string]any); ok { + tables = append(tables, table) + } + } + for _, table := range tables { + for _, key := range []string{"dependencies", "dev-dependencies", "build-dependencies"} { + for alias, value := range cargoTable(table, key) { + dep, isTable := value.(map[string]any) + if isTable && dep["workspace"] == true { + value = inherited[alias] + dep, isTable = value.(map[string]any) + } + _, isVersion := value.(string) + if !isTable && !isVersion { + continue + } + target := alias + if packageName, ok := dep["package"].(string); ok { + target = packageName + } + // Conflicting target-specific aliases cannot identify one package. + if old, exists := deps[alias]; exists && old != target { + target = "" + } + deps[alias] = target + } + } + } + c := &crate{dir: path.Join(prefix, dir), name: crateName, root: path.Join(prefix, dir, libPath), deps: deps} + if owners[c.dir] == nil { + owners[c.dir] = c + packages[pkg] = append(packages[pkg], c) + } + } + } + declared := false + for _, owner := range owners { + owner.imports = map[string][]resolver.CargoCrate{} + for alias, pkg := range owner.deps { + for _, provider := range packages[pkg] { + if graph.RepoPrefixOfID(provider.root) != graph.RepoPrefixOfID(owner.root) { + declared = true + } + name := provider.name + if alias != pkg { + name = strings.ReplaceAll(alias, "-", "_") + } + owner.imports[name] = append(owner.imports[name], resolver.CargoCrate{Root: provider.root, Name: provider.name}) + } + } + } + return func(file, name string) []resolver.CargoCrate { + // Cache the nearest manifest boundary, including nonmembers and excludes. + // A nested crate must never borrow an ancestor package's dependencies. + ownerMu.Lock() + defer ownerMu.Unlock() + var prefix string + for dir := path.Dir(file); dir != "."; dir = path.Dir(dir) { + if roots[dir] != nil { + prefix = dir + break + } + } + if prefix == "" { + return nil + } + var owner *crate + var visited []string + for dir := path.Dir(file); ; dir = path.Dir(dir) { + if cached, ok := ownerCache[dir]; ok { + owner = cached + break + } + visited = append(visited, dir) + if c := owners[dir]; c != nil { + owner = c + break + } + rel := strings.TrimPrefix(dir, prefix+"/") + if dir == prefix { + rel = "." + } + if roots[prefix].isFile(path.Join(rel, "Cargo.toml")) || dir == prefix { + break // Unknown, malformed, or unreadable manifest: no ownership evidence. + } + } + for _, dir := range visited { + ownerCache[dir] = owner + } + if owner == nil { + return nil + } + return owner.imports[name] + }, declared +} diff --git a/internal/indexer/cargo_crate_test.go b/internal/indexer/cargo_crate_test.go new file mode 100644 index 000000000..fbf561423 --- /dev/null +++ b/internal/indexer/cargo_crate_test.go @@ -0,0 +1,185 @@ +package indexer + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" + "go.uber.org/zap" +) + +func TestMultiIndexer_CargoCrateImport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"item", "module", "directory", "custom lib", "rename", "dev", "build", "target", "workspace", "undeclared", "sibling declaration", "ambiguous", "disallowed workspace", "same named repo", "ambiguous module", "missing inherited", "reserved", "escaped lib", "nested nonmember", "excluded member", "wrong lib name", "renamed policy allowed", "renamed policy refused", "incremental frontier", "incremental file"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + pkgDir, entry, spec := "", "src/lib.rs", "other_crate::Item" + providerManifest := "[package]\nname = \"other-crate\"\nversion = \"0.1.0\"\n" + dep := "[dependencies]\nother-crate = \"1\"\n" + switch mode { + case "module", "directory", "ambiguous module": + spec = "other_crate::a::b::Item" + entry = "src/a/b.rs" + if mode == "directory" { + entry = "src/a/b/mod.rs" + } + case "wrong lib name": + providerManifest += "[lib]\nname = \"engine\"\n" + case "custom lib": + providerManifest += "[lib]\nname = \"engine\"\npath = \"code/root.rs\"\n" + entry, spec = "code/root.rs", "engine::Item" + case "escaped lib": + providerManifest += "[lib]\npath = \"../src/lib.rs\"\n" + case "reserved": + dep, spec = "[dependencies]\ncore = { package = \"other-crate\", version = \"1\" }\n", "core::Item" + case "missing inherited": + dep = "[dependencies]\nother-crate = { workspace = true }\n" + case "rename", "renamed policy allowed", "renamed policy refused": + dep, spec = "[dependencies]\nalias = { package = \"other-crate\", version = \"1\" }\n", "alias::Item" + case "dev", "build": + dep = "[" + mode + "-dependencies]\nother-crate = \"1\"\n" + case "target": + dep = "[target.'cfg(unix)'.dependencies]\nother-crate = \"1\"\n" + case "workspace": + pkgDir = "crates/engine" + writeWorkspaceFile(t, provider, "Cargo.toml", "[workspace]\nmembers = [\"crates/*\"]\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[workspace]\nmembers = [\"apps/*\"]\n[workspace.dependencies]\nalias = { package = \"other-crate\", version = \"1\" }\n") + dep, spec = "[dependencies]\nalias = { workspace = true }\n", "alias::Item" + case "undeclared", "same named repo", "sibling declaration": + dep = "" + } + writeWorkspaceFile(t, provider, filepath.Join(pkgDir, "Cargo.toml"), providerManifest) + writeWorkspaceFile(t, provider, filepath.Join(pkgDir, "src/lib.rs"), "pub struct Item;\n") + writeWorkspaceFile(t, provider, filepath.Join(pkgDir, entry), "pub struct Item;\n") + if mode == "module" || mode == "directory" || mode == "ambiguous module" { + writeWorkspaceFile(t, provider, "src/lib.rs", "pub mod a;\n") + writeWorkspaceFile(t, provider, "src/a.rs", "pub mod b;\n") + } + if mode == "ambiguous module" { + writeWorkspaceFile(t, provider, "src/a/b/mod.rs", "pub struct Item;\n") + } + consumerDir := "" + if mode == "nested nonmember" || mode == "excluded member" { + consumerDir = "tools/child" + rootManifest := "[package]\nname = \"root\"\n" + dep + if mode == "excluded member" { + rootManifest += "[workspace]\nmembers = [\"tools/*\"]\nexclude = [\"tools/child\"]\n" + } + writeWorkspaceFile(t, consumer, "Cargo.toml", rootManifest) + dep = "" + } + if mode == "workspace" || mode == "sibling declaration" { + consumerDir = "apps/app" + } + if mode == "sibling declaration" { + writeWorkspaceFile(t, consumer, "Cargo.toml", "[workspace]\nmembers = [\"apps/*\"]\n") + writeWorkspaceFile(t, consumer, "apps/sibling/Cargo.toml", "[package]\nname = \"sibling\"\n[dependencies]\nother-crate = \"1\"\n") + } + writeWorkspaceFile(t, consumer, filepath.Join(consumerDir, "Cargo.toml"), "[package]\nname = \"app\"\nversion = \"0.1.0\"\n"+dep) + caller := filepath.Join(consumerDir, "src/lib.rs") + writeWorkspaceFile(t, consumer, caller, "use "+spec+";\n") + providerWS, providerName := "shared", "provider" + if mode == "disallowed workspace" || strings.HasPrefix(mode, "renamed policy") { + providerWS = "private" + } + if mode == "same named repo" { + providerName = "other_crate" + } + entries := []config.RepoEntry{{Path: provider, Name: providerName, Workspace: providerWS}, {Path: consumer, Name: "consumer", Workspace: "shared"}} + if mode == "ambiguous" { + duplicate := t.TempDir() + writeWorkspaceFile(t, duplicate, "Cargo.toml", providerManifest) + writeWorkspaceFile(t, duplicate, "src/lib.rs", "pub struct Item;\n") + entries = append(entries, config.RepoEntry{Path: duplicate, Name: "duplicate", Workspace: "shared"}) + } + if strings.HasPrefix(mode, "renamed policy") { + module := "other_crate" + if mode == "renamed policy refused" { + module = "alias" + } + writeWorkspaceFile(t, consumer, ".gortex.yaml", "cross_workspace_deps:\n - workspace: private\n modules: ["+module+"]\n mode: read-only\n") + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + reg := newTestRegistry() + reg.Register(languages.NewRustExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + _, err = mi.IndexAll() + require.NoError(t, err) + want := filepath.ToSlash(filepath.Join(providerName, pkgDir, entry)) + require.NotNil(t, g.GetNode(want)) + resolves := mode != "nested nonmember" && mode != "excluded member" && mode != "wrong lib name" && mode != "renamed policy refused" && mode != "undeclared" && mode != "same named repo" && mode != "sibling declaration" && mode != "ambiguous" && mode != "disallowed workspace" && mode != "ambiguous module" && mode != "missing inherited" && mode != "reserved" && mode != "escaped lib" + if strings.HasPrefix(mode, "incremental") { + var initial []string + for e := range g.EdgesByKind(graph.EdgeImports) { + if e.FilePath == "consumer/src/lib.rs" { + initial = append(initial, e.To) + } + } + require.Equal(t, []string{want}, initial, "initial binding") + writeWorkspaceFile(t, consumer, caller, "use "+spec+";\npub fn edited() {}\n") + require.NoError(t, mi.GetIndexer("consumer").IndexFileNoResolve(caller)) + if mode == "incremental frontier" { + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/src/lib.rs"}, nil)) + } else { + mi.runMasterResolveFiles([]string{"consumer/src/lib.rs"}, true) + mi.newCrossRepoResolver().ResolveForFile("consumer", "src/lib.rs") + } + } + var imports int + for e := range g.EdgesByKind(graph.EdgeImports) { + if e.FilePath != filepath.ToSlash(filepath.Join("consumer", caller)) { + continue + } + imports++ + assert.Equal(t, resolves, e.CrossRepo) + if resolves { + assert.Equal(t, want, e.To) + } else { + assert.Equal(t, "external::"+strings.ReplaceAll(spec, "::", "/"), e.To) + } + } + assert.Equal(t, 1, imports) + }) + } + } +} + +func TestCargoCrateLookupNoManifest(t *testing.T) { + for _, cargoRoots := range []int{0, 1} { + t.Run(fmt.Sprintf("cargo_roots=%d", cargoRoots), func(t *testing.T) { + // Given two repositories, with fewer than two Cargo roots. + first, second := t.TempDir(), t.TempDir() + if cargoRoots == 1 { + writeWorkspaceFile(t, first, "Cargo.toml", "[package]\nname='library'\n") + } + g := &cargoWarmScanStore{Store: graph.New()} + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"first": {RootPath: first}, "second": {RootPath: second}}} + // When cross-repo Cargo recovery is requested. + assert.Nil(t, mi.cargoCrateLookup(), "fewer than two Cargo roots cannot form a tracked bridge") + require.Empty(t, mi.cargoConsumerFilesForRepo("first", false)) + // Then it performs no import scan. + assert.Zero(t, g.lightScans, "no cross-repo Cargo provider is possible") + }) + } +} diff --git a/internal/indexer/cargo_import_boundary_test.go b/internal/indexer/cargo_import_boundary_test.go new file mode 100644 index 000000000..63fe1e730 --- /dev/null +++ b/internal/indexer/cargo_import_boundary_test.go @@ -0,0 +1,56 @@ +package indexer + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +func TestCargoCrateLookupUsesInstalledManifestTree(t *testing.T) { + for _, workspace := range []bool{false, true} { + t.Run(map[bool]string{false: "root", true: "workspace"}[workspace], func(t *testing.T) { + provider, consumer, providerSnapshot, consumerSnapshot := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'wrong-checkout'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'wrong-app'\n") + providerDir, consumerDir := "", "" + dependency := "[dependencies]\nalias = { package = 'other-crate', version = '1' }\n" + if workspace { + providerDir, consumerDir = "crates/engine/", "apps/app/" + writeWorkspaceFile(t, providerSnapshot, "Cargo.toml", "[workspace]\nmembers = ['crates/*']\n") + writeWorkspaceFile(t, consumerSnapshot, "Cargo.toml", "[workspace]\nmembers = ['apps/*']\nexclude = ['apps/excluded']\n[workspace.dependencies]\nalias = { package = 'other-crate', version = '1' }\n") + writeWorkspaceFile(t, consumerSnapshot, "apps/excluded/Cargo.toml", "[package]\nname = 'excluded'\n") + dependency = "[dependencies]\nalias = { workspace = true }\n" + } + writeWorkspaceFile(t, providerSnapshot, providerDir+"Cargo.toml", "[package]\nname = 'other-crate'\n") + writeWorkspaceFile(t, consumerSnapshot, consumerDir+"Cargo.toml", "[package]\nname = 'app'\n"+dependency) + g := graph.New() + providerIdx, consumerIdx := sideChannelIndexer(t, g, provider), sideChannelIndexer(t, g, consumer) + providerIdx.SetContentSource(snapshotOf(t, providerSnapshot)) + consumerIdx.SetContentSource(snapshotOf(t, consumerSnapshot)) + mi := &MultiIndexer{graph: g, + repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}, + indexers: map[string]*Indexer{"provider": providerIdx, "consumer": consumerIdx}} + lookup := mi.cargoCrateLookup() + require.NotNil(t, lookup) + require.Equal(t, []resolver.CargoCrate{{Root: "provider/" + providerDir + "src/lib.rs", Name: "other_crate"}}, lookup("consumer/"+consumerDir+"src/lib.rs", "alias")) + if workspace { + require.Empty(t, lookup("consumer/apps/excluded/src/lib.rs", "alias")) + } + }) + } +} + +func TestCargoRestubIgnoresMetadataOnlyTargets(t *testing.T) { + g := graph.New() + target := "provider/src/meta.rs" + edge := &graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: target, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{resolver.CargoImportStubMeta: "external::other_crate/meta"}} + g.AddNode(&graph.Node{ID: target, Kind: graph.KindFile, RepoPrefix: "provider"}) + g.AddEdge(edge) + incoming := g.GetInEdgesByNodeIDs([]string{target}) + require.Empty(t, restubTrackedCargoImportsFromView(g, map[string]struct{}{"provider/src/changed.rs": {}}, incoming, false)) + require.Equal(t, target, edge.To) + require.True(t, edge.CrossRepo) +} diff --git a/internal/indexer/cargo_import_fallback_test.go b/internal/indexer/cargo_import_fallback_test.go new file mode 100644 index 000000000..51b00fc4e --- /dev/null +++ b/internal/indexer/cargo_import_fallback_test.go @@ -0,0 +1,76 @@ +package indexer + +import ( + "context" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +// cargoFallbackRefreshTransform changes extra.rs's bytes on exactly its second +// application: the probe read sees call 1, takePreparedRefresh sees call 2 +// (mismatch, so the stage falls back), and the fallback indexFile sees call 3. +type cargoFallbackRefreshTransform struct{ calls *atomic.Int32 } + +func (cargoFallbackRefreshTransform) name() string { return "cargo-fallback-refresh" } +func (cargoFallbackRefreshTransform) matches(p string) bool { return strings.HasSuffix(p, "extra.rs") } +func (cargoFallbackRefreshTransform) asLanguage() string { return "" } +func (f cargoFallbackRefreshTransform) apply(p string, src []byte) ([]byte, error) { + if f.calls.Add(1) == 2 { + return append(append([]byte(nil), src...), []byte("\n// concurrent write\n")...), nil + } + return src, nil +} + +func TestTrackedCargoNewModuleThroughFallback(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") + writeWorkspaceFile(t, provider, "src/lib.rs", cargoLifecycleProvider) + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") + writeWorkspaceFile(t, consumer, "src/lib.rs", cargoLifecycleConsumer+"\nuse other_crate::extra::later;\npub fn after() { later(); }\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewRustExtractor()) + g := cargoWarmStore(t, backend) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + _, err = mi.IndexAll() + require.NoError(t, err) + + idx := mi.GetIndexer("provider") + require.NotNil(t, idx) + var calls atomic.Int32 + idx.transforms.transforms = append(idx.transforms.transforms, cargoFallbackRefreshTransform{calls: &calls}) + writeWorkspaceFile(t, provider, "src/extra.rs", "pub fn later() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/extra.rs")}) + require.NoError(t, err) + require.GreaterOrEqual(t, calls.Load(), int32(3), "fallback path must run") + require.NotNil(t, g.GetNode("provider/src/extra.rs")) + + cold := cargoWarmStore(t, backend) + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, cargoLifecycleBindings(cold, "fallback"), cargoLifecycleBindings(g, "fallback")) + + }) + } +} diff --git a/internal/indexer/cargo_import_lifecycle_test.go b/internal/indexer/cargo_import_lifecycle_test.go new file mode 100644 index 000000000..653b0fcb3 --- /dev/null +++ b/internal/indexer/cargo_import_lifecycle_test.go @@ -0,0 +1,387 @@ +package indexer + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedCargoImportLifecycle(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "provider_gains_empty_module", "no_new_file_noop", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "rename_and_back", "untrack_only", "no_change", "track", "deferred_track"} { + if operation == "deferred_track" && providerFirst { + continue // This case starts with an already tracked consumer. + } + order := "consumer_first" + if providerFirst { + order = "provider_first" + } + t.Run(backend+"/"+order+"/"+operation, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") + providerSource := cargoLifecycleProvider + if operation == "provider_gains_new_module" || operation == "provider_gains_empty_module" { + providerSource += "pub mod newmod;\n" + } + writeWorkspaceFile(t, provider, "src/lib.rs", providerSource) + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + if operation == "no_new_file_noop" { + writeWorkspaceFile(t, provider, "src/spare.rs", "pub fn unused() {}\n") + } + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") + consumerSource := cargoLifecycleConsumer + switch operation { + case "provider_gains_new_module": + consumerSource += "use other_crate::newmod::later;\npub fn after() { later(); }\n" + case "provider_gains_empty_module": + consumerSource += "use other_crate::newmod::*;\n" + } + writeWorkspaceFile(t, consumer, "src/lib.rs", consumerSource) + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + if providerFirst { + gc.Repos = []config.RepoEntry{entries[1], entries[0]} + } + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + var g graph.Store = graph.New() + if backend == "sqlite" { + disk, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, disk.Close()) }) + g = disk + } + reg := newTestRegistry() + reg.Register(languages.NewRustExtractor()) + core, logs := observer.New(zap.InfoLevel) + logger := zap.New(core) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + switch operation { + case "deferred_track": + _, err = mi.TrackRepoCtx(context.Background(), entries[0]) + require.NoError(t, err) + mi.BeginParallelBatch() + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + require.NoError(t, err) + for _, edge := range g.GetOutEdges("consumer/src/lib.rs") { + if edge.Kind == graph.EdgeImports { + require.False(t, edge.CrossRepo, "deferred track must not resolve shared imports early") + } + } + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs"}, nil)) + mi.EndBatch() + case "track": + for _, entry := range gc.Repos { + _, err = mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + default: + _, err = mi.IndexAll() + require.NoError(t, err) + } + assertImport := func() { + t.Helper() + var imports []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/src/lib.rs") { + if edge.Kind == graph.EdgeImports && edge.Line <= 3 { + imports = append(imports, edge) + } + } + require.Len(t, imports, 3) + for _, edge := range imports { + require.True(t, edge.CrossRepo) + } + calls := make(map[string]string) + for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { + if edge.Kind == graph.EdgeCalls { + require.True(t, edge.CrossRepo) + stub, _ := edge.Meta["cargo_call_stub"].(string) + calls[edge.To] = stub + } + } + require.Equal(t, map[string]string{ + "provider/src/lib.rs::Foo.new": "unresolved::new", + "provider/src/lib.rs::Foo.go": "unresolved::*.go", + "provider/src/lib.rs::action": "unresolved::action", + "provider/src/util.rs::helper": "unresolved::helper", + }, calls) + } + assertImport() + initial := cargoLifecycleBindings(g, "") + initialMarkers := map[string]string{} + for _, node := range g.GetFileNodes("consumer/src/lib.rs") { + for _, edge := range g.GetOutEdges(node.ID) { + key := "cargo_call_stub" + if edge.Kind == graph.EdgeImports { + key = "cargo_import_stub" + } + if marker, ok := edge.Meta[key].(string); ok { + initialMarkers[fmt.Sprint(edge.Kind, edge.Line)] = marker + } + } + } + logs.TakeAll() + switch operation { + case "consumer_save": + writeWorkspaceFile(t, consumer, "src/lib.rs", cargoLifecycleConsumer+"pub fn edited() {}\n") + _, err = mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "src/lib.rs")}) + case "provider_save": + writeWorkspaceFile(t, provider, "src/lib.rs", cargoLifecycleProvider+"pub fn added() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/lib.rs")}) + case "provider_save_util": + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\npub fn added() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) + case "provider_save_lib_contract": + writeWorkspaceFile(t, provider, "src/lib.rs", strings.Replace(cargoLifecycleProvider, "pub fn action() {}", "pub fn action(x: u8) {}", 1)) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/lib.rs")}) + case "provider_delete_util_batched": + require.NoError(t, os.Remove(filepath.Join(provider, "src/util.rs"))) + writeWorkspaceFile(t, provider, "src/lib.rs", strings.Replace(cargoLifecycleProvider, "pub mod util;\n", "", 1)) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs"), filepath.Join(provider, "src/lib.rs")}) + case "no_new_file_noop": + observed := &cargoImportReindexObserver{Store: mi.graph} + mi.graph = observed + writeWorkspaceFile(t, provider, "src/spare.rs", "pub fn unused() {}\npub fn added() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/spare.rs")}) + require.NoError(t, err) + require.Zero(t, observed.importReindexes, "an ordinary provider edit must not reindex working imports") + require.Zero(t, observed.importReads, "bound imports must not be admitted for an ordinary edit") + case "provider_gains_empty_module": + writeWorkspaceFile(t, provider, "src/newmod.rs", "") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/newmod.rs")}) + case "provider_gains_new_module": + writeWorkspaceFile(t, provider, "src/newmod.rs", "pub fn later() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/newmod.rs")}) + case "delete_restore_util", "consumer_save_then_restore": + if operation == "consumer_save_then_restore" { + writeWorkspaceFile(t, consumer, "src/lib.rs", cargoLifecycleConsumer+"pub fn edited() {}\n") + _, err = mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "src/lib.rs")}) + require.NoError(t, err) + assertImport() + } + require.NoError(t, os.Remove(filepath.Join(provider, "src/util.rs"))) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) + require.NoError(t, err) + var parked []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 8 { + parked = append(parked, edge) + } + } + require.Len(t, parked, 1) + require.Equal(t, "unresolved::helper", parked[0].To) + require.False(t, parked[0].CrossRepo) + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) + case "forced_evict_util": + require.NoError(t, os.Remove(filepath.Join(provider, "src/util.rs"))) + mi.GetIndexer("provider").EvictFile(filepath.Join(provider, "src/util.rs")) + case "consumer_full": + _, err = mi.IndexRepo("consumer") + case "provider_full": + var result *IndexResult + result, err = mi.IndexRepo("provider") + require.NoError(t, err) + require.Contains(t, result.CrossRepoResolveFiles, "consumer/src/lib.rs") + case "warm_full_retrack": + prior := map[string]map[string]int64{"consumer": mi.FileMtimes("consumer"), "provider": mi.FileMtimes("provider")} + restarted := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, restarted.Close(context.Background())) }) + restarted.BeginBatch() + var output struct { + Files []string `json:"cross_repo_resolve_files"` + } + for _, entry := range gc.Repos { + force := "0" + if entry.Name == "provider" { + force = "1" + } + t.Setenv("GORTEX_WARMUP_FULL_RETRACK", force) + result, restoreErr := restarted.ReconcileRepoCtx(context.Background(), entry, prior[entry.Name]) + require.NoError(t, restoreErr) + if entry.Name == "provider" { + require.True(t, result.FullRetrack) + } + data, marshalErr := json.Marshal(result) + require.NoError(t, marshalErr) + var current struct { + Files []string `json:"cross_repo_resolve_files"` + } + require.NoError(t, json.Unmarshal(data, ¤t)) + output.Files = appendUniqueSorted(output.Files, current.Files...) + if entry.Name == "provider" { + require.NotContains(t, result.DerivedInvalidation.Files, "consumer/src/lib.rs") + } + } + if backend == "sqlite" { + require.Contains(t, output.Files, "consumer/src/lib.rs") + } else { + require.Empty(t, output.Files, "memory replacement preserved the existing import") + } + err = restarted.RunPreEnrichResolveFiles(context.Background(), output.Files, nil) + restarted.EndBatch() + case "no_change": + result, noChangeErr := mi.IncrementalReindexRepo("provider", nil) + require.NoError(t, noChangeErr) + require.Zero(t, result.StaleFileCount) + case "rename_and_back": + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper_renamed() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) + require.NoError(t, err) + for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 8 { + require.False(t, edge.CrossRepo) + require.Equal(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)], edge.Meta["cargo_call_stub"]) + } + } + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/util.rs")}) + case "untrack_only": + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") + case "untrack_retrack": + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") + require.NoError(t, err) + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + } + require.NoError(t, err) + if operation == "rename_and_back" { + require.Equal(t, initial, cargoLifecycleBindings(g, "")) + } + if operation == "untrack_only" { + for _, node := range g.GetFileNodes("consumer/src/lib.rs") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind != graph.EdgeImports && edge.Kind != graph.EdgeCalls { + continue + } + require.False(t, edge.CrossRepo) + require.False(t, strings.HasPrefix(edge.To, "provider/")) + key := "cargo_call_stub" + if edge.Kind == graph.EdgeImports { + key = "cargo_import_stub" + } + require.NotEmpty(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)]) + require.Equal(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)], edge.Meta[key]) + } + } + } + if operation == "provider_delete_util_batched" || operation == "forced_evict_util" { + var parked []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/src/lib.rs::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 8 { + parked = append(parked, edge) + } + } + require.Len(t, parked, 1) + require.Equal(t, "unresolved::helper", parked[0].To) + require.Equal(t, parked[0].To, parked[0].Meta["cargo_call_stub"]) + require.False(t, parked[0].CrossRepo) + } + var cold graph.Store = graph.New() + if backend == "sqlite" { + db, openErr := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, openErr) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + cold = db + } + coldConfig := cm + if operation == "untrack_only" { + coldConfig = newTestConfigManager(t) + coldConfig.Global().Repos = []config.RepoEntry{entries[0]} + } + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), coldConfig, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, cargoLifecycleBindings(cold, operation), cargoLifecycleBindings(g, operation), "after %s", operation) + require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) + require.Empty(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveAll").All()) + }) + } + } + } +} + +const cargoLifecycleProvider = "pub mod util;\npub struct Foo;\nimpl Foo {\n pub fn new() -> Foo { Foo }\n pub fn go(&self) {}\n}\npub fn action() {}\n" +const cargoLifecycleConsumer = "use other_crate::Foo;\nuse other_crate::action;\nuse other_crate::util::helper;\npub fn run() {\n let f = Foo::new();\n f.go();\n action();\n helper();\n}\n" + +func cargoLifecycleBindings(g graph.Store, operation string) []string { + var bindings []string + for _, node := range g.GetFileNodes("consumer/src/lib.rs") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind != graph.EdgeImports && edge.Kind != graph.EdgeCalls { + continue + } + copy := *edge + parked := operation == "untrack_only" || ((operation == "provider_delete_util_batched" || operation == "forced_evict_util") && ((edge.Kind == graph.EdgeImports && edge.Line == 3) || (edge.Kind == graph.EdgeCalls && edge.Line == 8))) + if parked && !edge.CrossRepo { + copy.Meta = maps.Clone(edge.Meta) + key := "cargo_call_stub" + if edge.Kind == graph.EdgeImports { + key = "cargo_import_stub" + } + delete(copy.Meta, key) // Historical parked identity is asserted separately. + } + bindings = append(bindings, builderRenderEdge(©)) + } + } + slices.Sort(bindings) + return bindings +} + +// Observe only the shared resolver; the per-repo indexer retains its native store. +type cargoImportReindexObserver struct { + graph.Store + importReindexes int + importReads int +} + +func (s *cargoImportReindexObserver) ReindexEdges(batch []graph.EdgeReindex) { + for _, change := range batch { + if change.Edge != nil && change.Edge.Kind == graph.EdgeImports && change.Edge.FilePath == "consumer/src/lib.rs" { + s.importReindexes++ + } + } + s.Store.ReindexEdges(batch) +} + +func (s *cargoImportReindexObserver) GetEdgeCandidates(endpoints []graph.EdgeEndpoint, sites []graph.EdgeSite) graph.EdgeCandidateSet { + for _, site := range sites { + if site.Kind == graph.EdgeImports && site.From == "consumer/src/lib.rs" { + s.importReads++ + } + } + return s.Store.GetEdgeCandidates(endpoints, sites) +} + +func TestCargoAddedFileWithoutConsumersDoesNotRunGlobalResolve(t *testing.T) { + core, logs := observer.New(zap.InfoLevel) + g := graph.New() + g.AddNode(&graph.Node{ID: "consumer/src/lib.rs::run", Kind: graph.KindFunction, Name: "run", FilePath: "consumer/src/lib.rs", RepoPrefix: "consumer", Language: "rust"}) + g.AddNode(&graph.Node{ID: "provider/src/new.rs::f", Kind: graph.KindFunction, Name: "f", FilePath: "provider/src/new.rs", RepoPrefix: "provider", Language: "rust"}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs::run", To: "unresolved::f", Kind: graph.EdgeCalls, FilePath: "consumer/src/lib.rs"}) + mi := &MultiIndexer{graph: g, logger: zap.New(core)} + mi.resolveIncrementalRepoMutation("provider", &IndexResult{}, &graph.MutationReceipt{Complete: true}, + &reparsePendingEnrichmentBatch{cargoAddedFile: true}) + require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) +} diff --git a/internal/indexer/cargo_import_restub.go b/internal/indexer/cargo_import_restub.go new file mode 100644 index 000000000..c54677d52 --- /dev/null +++ b/internal/indexer/cargo_import_restub.go @@ -0,0 +1,194 @@ +package indexer + +import ( + "context" + "sort" + "strings" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +// Preserve importer-owned facts before their provider's files disappear. +func restubTrackedCargoRepoIncoming(g graph.Store, repo string) []string { + if repo == "" { + return nil + } + var files, ids []string + flush := func() { + incoming := foreignBridgeIncoming(g, repo, ids) + files = appendUniqueSorted(files, restubTrackedCargoImportsFromView(g, nil, incoming, true)...) + ids = ids[:0] + } + for node := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFile, graph.KindFunction, graph.KindMethod) { + file, _, _ := strings.Cut(node.ID, "::") + if !strings.HasSuffix(file, ".rs") { + continue + } + ids = append(ids, node.ID) + if len(ids) == 512 { + flush() + } + } + if len(ids) > 0 { + flush() + } + return files +} + +func restubTrackedCargoImportsFromView(g graph.Store, evicted map[string]struct{}, incoming map[string][]*graph.Edge, parkCalls bool) []string { + consumers := make(map[string]struct{}) + var batch []graph.EdgeReindex + for id, edges := range incoming { + if _, changing := evicted[id]; !parkCalls && !changing { + continue + } + for _, edge := range edges { + if edge == nil || !edge.CrossRepo { + continue + } + key := resolver.CargoImportStubMeta + if edge.Kind == graph.EdgeCalls { + key = resolver.CargoCallStubMeta + } else if edge.Kind != graph.EdgeImports { + continue + } + stub, _ := edge.Meta[key].(string) + if _, gone := evicted[edge.From]; stub == "" || gone || edge.To == stub { + continue + } + consumers[edge.FilePath] = struct{}{} + // Save paths keep the existing carry/restub decision and its dataflow sites. + if edge.Kind == graph.EdgeCalls && !parkCalls { + continue + } + old := edge.To + if edge.Kind == graph.EdgeCalls { + graph.StashRestubProvenance(edge) + edge.ConfidenceLabel = "" + } + if edge.Kind == graph.EdgeImports { + delete(edge.Meta, resolver.CargoImportModuleMeta) + } + edge.To, edge.CrossRepo = stub, false + batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: old}) + } + } + if len(batch) > 0 { + g.ReindexEdges(batch) + } + return sortedStringKeys(consumers) +} + +// Retry parked imports; new Rust files can also retarget marked provider bindings. +func (mi *MultiIndexer) cargoConsumerFilesForRepo(repo string, retargetBound bool) []string { + return mi.cargoConsumerFilesForRepos([]string{repo}, retargetBound) +} + +func (mi *MultiIndexer) cargoConsumerFilesForRepos(repos []string, retargetBound bool) []string { + lookup := mi.cargoCrateLookup() + if lookup == nil || len(repos) == 0 { + return nil + } + return mi.cargoConsumerFilesForLookup(repos, retargetBound, lookup) +} + +func (mi *MultiIndexer) cargoAddedFileConsumers(repo string) []string { + snapshot := mi.cargoManifestSnapshot() + if snapshot.hasUnknownManifest() { + // Missing evidence must widen recovery, never discard a possible importer. + files := map[string]struct{}{} + for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { + if strings.HasSuffix(edge.FilePath, ".rs") { + files[edge.FilePath] = struct{}{} + } + } + return sortedStringKeys(files) + } + if !snapshot.cargoDeclared { + return nil + } + return mi.cargoConsumerFilesForLookup([]string{repo}, true, snapshot.cargo) +} + +func (mi *MultiIndexer) cargoConsumerFilesForLookup(repos []string, retargetBound bool, lookup resolver.CargoCrateLookup) []string { + wanted := make(map[string]bool, len(repos)) + for _, repo := range repos { + wanted[repo] = true + } + files := make(map[string]struct{}) + var sites []graph.EdgeSite + for edge := range graph.EdgesLightSeq(mi.graph, graph.EdgeImports) { + if !strings.HasSuffix(edge.FilePath, ".rs") { + continue + } + spec, external := strings.CutPrefix(edge.To, "external::") + if !external { + spec, external = strings.CutPrefix(edge.To, "unresolved::import::") + } + if !external { + if retargetBound && edge.CrossRepo && wanted[graph.RepoPrefixOfID(edge.To)] { + sites = append(sites, graph.EdgeSite{From: edge.From, Line: edge.Line, Kind: graph.EdgeImports}) + } + continue + } + name, _, _ := strings.Cut(spec, "/") + for _, crate := range lookup(edge.FilePath, name) { + if wanted[graph.RepoPrefixOfID(crate.Root)] || wanted[graph.RepoPrefixOfID(edge.From)] { + files[edge.FilePath] = struct{}{} + break + } + } + } + // Close the light cursor before hydrating only marked-provider candidates. + for start := 0; start < len(sites); start += 512 { + page := sites[start:min(start+512, len(sites))] + full := mi.graph.GetEdgeCandidates(nil, page) + for _, site := range page { + for _, edge := range full.Site(site.From, site.Line, site.Kind) { + stub, _ := edge.Meta[resolver.CargoImportStubMeta].(string) + if stub != "" && edge.CrossRepo && wanted[graph.RepoPrefixOfID(edge.To)] && strings.HasSuffix(edge.FilePath, ".rs") { + files[edge.FilePath] = struct{}{} + } + } + } + } + return sortedStringKeys(files) +} + +// Consume warm receipts only after all repositories are registered. +func (mi *MultiIndexer) runWarmCargoResolve(ctx context.Context, files []string, full bool) error { + mi.mu.RLock() + pending := map[string]*IndexResult{} + var repos []string + for repo, meta := range mi.repos { + if meta != nil && meta.cargoRetarget != nil { + pending[repo] = meta.cargoRetarget + repos = append(repos, repo) + } + } + mi.mu.RUnlock() + sort.Strings(repos) + var err error + if full { + err = mi.runCrossRepoResolveContext(ctx, false, repos...) + } else { + if len(repos) > 0 { + files = appendUniqueSorted(files, mi.cargoConsumerFilesForRepos(repos, true)...) + } + if err = context.Cause(ctx); err == nil { + mi.runCrossRepoResolveFiles(files, repos...) + err = context.Cause(ctx) + } + } + if err == nil { + mi.mu.Lock() + for repo, receipt := range pending { + if meta := mi.repos[repo]; meta != nil && meta.cargoRetarget == receipt { + meta.cargoRetarget = nil + } + } + mi.mu.Unlock() + } + return err +} diff --git a/internal/indexer/cargo_import_restub_test.go b/internal/indexer/cargo_import_restub_test.go new file mode 100644 index 000000000..5403e4fc9 --- /dev/null +++ b/internal/indexer/cargo_import_restub_test.go @@ -0,0 +1,99 @@ +package indexer + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +func TestCargoRestubLeavesUnmarkedImportsUnchanged(t *testing.T) { + for _, file := range []string{"provider/src/lib.rs", "provider/main.go"} { + t.Run(file, func(t *testing.T) { + g := graph.New() + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "provider"}) + edge := &graph.Edge{From: "consumer/source", FilePath: "consumer/source", To: file, Kind: graph.EdgeImports, CrossRepo: true} + if file == "provider/main.go" { + edge.Meta = map[string]any{resolver.CargoImportStubMeta: "external::other_crate/action"} + } + g.AddEdge(edge) + require.Empty(t, restubTrackedCargoRepoIncoming(g, "provider")) + require.Equal(t, file, edge.To) + require.True(t, edge.CrossRepo) + }) + } +} + +func TestCargoRestubKeepsSameRepoCallsOnNativePath(t *testing.T) { + for _, mode := range []string{"point", "batch"} { + t.Run(mode, func(t *testing.T) { + g := graph.New() + file, definition, caller := "repo/a.rs", "repo/a.rs::foo", "repo/b.rs::run" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "repo"}) + g.AddNode(&graph.Node{ID: definition, FilePath: file, Name: "foo", Kind: graph.KindFunction, RepoPrefix: "repo"}) + g.AddNode(&graph.Node{ID: caller, FilePath: "repo/b.rs", Name: "run", Kind: graph.KindFunction, RepoPrefix: "repo"}) + edge := &graph.Edge{From: caller, FilePath: "repo/b.rs", Kind: graph.EdgeCalls, To: definition, + Meta: map[string]any{resolver.CargoCallStubMeta: "unresolved::*.foo"}} + g.AddEdge(edge) + if mode == "point" { + idx := &Indexer{graph: g} + idx.restubIncomingRefs(file) + } else { + stages := []*incrementalBatchStage{{graphPath: file, priorNodes: g.GetFileNodes(file)}} + view := loadIncrementalPriorView(g, stages) + restubIncomingRefsFromView(g, stages, view) + } + require.Equal(t, "unresolved::foo", edge.To) + require.False(t, edge.CrossRepo) + }) + } +} + +func TestCargoConsumerFrontierIncludesUnresolvedImportStubs(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname='library'\nversion='1.0.0'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname='app'\nversion='1.0.0'\n[dependencies]\nlibrary='1'\n") + g := graph.New() + g.AddNode(&graph.Node{ID: "provider/src/lib.rs", FilePath: "provider/src/lib.rs", Kind: graph.KindFile, RepoPrefix: "provider", Language: "rust"}) + for _, tc := range []struct{ file, target string }{ + {"consumer/a.rs", "external::library/action"}, + {"consumer/b.rs", "unresolved::import::library/action"}, + {"consumer/c.rs", "unresolved::import::unknown/action"}, + } { + g.AddEdge(&graph.Edge{From: tc.file, FilePath: tc.file, To: tc.target, Kind: graph.EdgeImports}) + } + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + require.Equal(t, []string{"consumer/a.rs", "consumer/b.rs"}, mi.cargoConsumerFilesForRepo("provider", false)) +} + +func TestCargoRestubClearsMatchedModuleEvidence(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"untrack", "file_eviction"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + // Given a tracked import with its previously matched module path. + g := cargoWarmStore(t, backend) + file, target, stub := "consumer/src/lib.rs", "provider/src/util.rs", "external::other_crate/util" + for _, path := range []string{file, target} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{resolver.CargoImportStubMeta: stub, resolver.CargoImportModuleMeta: "other_crate::util"}}) + // When untracking or file eviction restubs the import. + var consumers []string + if mode == "untrack" { + consumers = restubTrackedCargoRepoIncoming(g, "provider") + } else { + consumers = restubTrackedCargoImportsFromView(g, map[string]struct{}{target: {}}, g.GetInEdgesByNodeIDs([]string{target}), false) + } + // Then its persisted state has the stub, not matched-module evidence. + require.Equal(t, []string{file}, consumers) + imports := g.GetOutEdges(file) + require.Len(t, imports, 1) + require.Equal(t, stub, imports[0].To) + require.False(t, imports[0].CrossRepo) + require.NotContains(t, imports[0].Meta, resolver.CargoImportModuleMeta) + require.Equal(t, stub, imports[0].Meta[resolver.CargoImportStubMeta]) + }) + } + } +} diff --git a/internal/indexer/cargo_import_warm_recovery_test.go b/internal/indexer/cargo_import_warm_recovery_test.go new file mode 100644 index 000000000..9c88f5c83 --- /dev/null +++ b/internal/indexer/cargo_import_warm_recovery_test.go @@ -0,0 +1,212 @@ +package indexer + +import ( + "context" + "fmt" + "iter" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedCargoWarmFileRecovery(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, deferred := range []bool{false, true} { + for _, full := range []bool{false, true} { + for _, restore := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/provider_first=%v/deferred=%v/restore=%v/full=%v", backend, providerFirst, deferred, restore, full), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") + writeWorkspaceFile(t, provider, "src/lib.rs", cargoLifecycleProvider) + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") + consumerSource := cargoLifecycleConsumer + if !restore { + consumerSource += "\nuse other_crate::extra::*;\n" + } + writeWorkspaceFile(t, consumer, "src/lib.rs", consumerSource) + for i := 0; i < 6; i++ { + writeWorkspaceFile(t, provider, fmt.Sprintf("src/pad%d.rs", i), "pub fn pad() {}\n") + writeWorkspaceFile(t, consumer, fmt.Sprintf("src/pad%d.rs", i), "pub fn pad() {}\n") + } + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + if providerFirst { + entries[0], entries[1] = entries[1], entries[0] + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewRustExtractor()) + newMI := func(g graph.Store) *MultiIndexer { + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + return mi + } + g := cargoWarmStore(t, backend) + mi := newMI(g) + _, err = mi.IndexAll() + require.NoError(t, err) + if restore { + file := filepath.Join(provider, "src/util.rs") + require.NoError(t, os.Remove(file)) + _, err = mi.IncrementalReindexRepo("provider", []string{file}) + require.NoError(t, err) + } + prior := map[string]map[string]int64{"consumer": mi.FileMtimes("consumer"), "provider": mi.FileMtimes("provider")} + require.NoError(t, mi.Close(context.Background())) + if restore { + writeWorkspaceFile(t, provider, "src/util.rs", "pub fn helper() {}\n") + } else { + writeWorkspaceFile(t, provider, "src/extra.rs", "") + } + restarted := newMI(g) + if deferred { + restarted.BeginParallelBatch() + } else { + restarted.BeginBatch() + } + var files []string + for _, entry := range entries { + result, err := restarted.ReconcileRepoCtx(context.Background(), entry, prior[entry.Name]) + require.NoError(t, err) + require.False(t, result.FullRetrack, "must exercise scoped warm reconcile") + if entry.Name == "provider" { + require.True(t, result.CargoRetarget) + } + files = appendUniqueSorted(files, result.DerivedInvalidation.Files...) + files = appendUniqueSorted(files, result.CrossRepoResolveFiles...) + } + if full { + require.NoError(t, restarted.RunPreEnrichResolve(context.Background(), nil, nil)) + } else { + require.NoError(t, restarted.RunPreEnrichResolveFiles(context.Background(), files, nil)) + } + restarted.EndBatch() + cold := cargoWarmStore(t, backend) + _, err = newMI(cold).IndexAll() + require.NoError(t, err) + require.Equal(t, cargoLifecycleBindings(cold, "warm"), cargoLifecycleBindings(g, "warm")) + }) + } + } + } + } + } +} + +func cargoWarmStore(t *testing.T, backend string) graph.Store { + t.Helper() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db + } + return graph.New() +} + +func TestCargoWarmRetargetReceiptCancellation(t *testing.T) { + a, b := &IndexResult{CargoRetarget: true}, &IndexResult{CargoRetarget: true} + mi := &MultiIndexer{graph: graph.New(), repos: map[string]*RepoMetadata{ + "a": {cargoRetarget: a}, "b": {cargoRetarget: b}, + }} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, mi.runWarmCargoResolve(ctx, nil, false), context.Canceled) + require.Same(t, a, mi.repos["a"].cargoRetarget) + require.Same(t, b, mi.repos["b"].cargoRetarget) + require.NoError(t, mi.runWarmCargoResolve(context.Background(), nil, false)) + require.Nil(t, mi.repos["a"].cargoRetarget) + require.Nil(t, mi.repos["b"].cargoRetarget) + require.True(t, a.CargoRetarget, "consumption must not mutate a returned result") +} + +func TestCargoAddedFileFrontierOnlyMarkedProviderImports(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname = 'other-crate'\nversion = '0.1.0'\n") + writeWorkspaceFile(t, provider, "src/lib.rs", "") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname = 'app'\nversion = '0.1.0'\n[dependencies]\nother-crate = '1'\n") + g := graph.New() + for _, row := range []struct { + file, target string + marker any + }{ + {"consumer/marked.rs", "provider/src/lib.rs", "external::other_crate/util"}, + {"consumer/unmarked.rs", "provider/src/lib.rs", nil}, + {"consumer/invalid.rs", "provider/src/lib.rs", 1}, + {"consumer/other.rs", "other/src/lib.rs", "external::other_crate/util"}, + {"consumer/nonrust.py", "provider/src/lib.rs", "external::other_crate/util"}, + } { + g.AddEdge(&graph.Edge{From: row.file, FilePath: row.file, To: row.target, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{"cargo_import_stub": row.marker}}) + } + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{ + "provider": {RootPath: provider}, "consumer": {RootPath: consumer}, + }} + require.Equal(t, []string{"consumer/marked.rs"}, mi.cargoConsumerFilesForRepo("provider", true)) + require.Empty(t, mi.cargoConsumerFilesForRepo("provider", false)) +} + +type cargoWarmScanStore struct { + graph.Store + fullScans, lightScans int +} + +func (s *cargoWarmScanStore) EdgesByKind(kind graph.EdgeKind) iter.Seq[*graph.Edge] { + if kind == graph.EdgeImports { + s.fullScans++ + } + return s.Store.EdgesByKind(kind) +} +func (s *cargoWarmScanStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + if len(kinds) == 1 && kinds[0] == graph.EdgeImports { + s.lightScans++ + } + return graph.EdgesLightSeq(s.Store, kinds...) +} +func TestCargoWarmRecoveryScansImportsOnce(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + for _, root := range []string{provider, consumer} { + writeWorkspaceFile(t, root, "Cargo.toml", "[package]\nname='library'\nversion='1.0.0'\n") + } + g := &cargoWarmScanStore{Store: graph.New()} + g.AddEdge(&graph.Edge{From: "unrelated/lib.rs", FilePath: "unrelated/lib.rs", To: "external::unknown/f", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{ + "provider": {RootPath: provider, cargoRetarget: &IndexResult{CargoRetarget: true}}, + "consumer": {RootPath: consumer, cargoRetarget: &IndexResult{CargoRetarget: true}}, + }} + require.NoError(t, mi.runWarmCargoResolve(t.Context(), nil, false)) + require.Zero(t, g.fullScans, "warm recovery must not read every import's metadata") + require.Equal(t, 1, g.lightScans, "all receipt repos must share one scan") +} + +func TestCargoLookupDefersManifestReadsUntilUse(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + mi := &MultiIndexer{graph: graph.New(), repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + for _, root := range []string{provider, consumer} { + writeWorkspaceFile(t, root, "Cargo.toml", "") + } + lookup := mi.cargoCrateLookup() + require.NotNil(t, lookup, "construction may stat roots but must defer reading Cargo manifests") + writeWorkspaceFile(t, provider, "Cargo.toml", "[package]\nname='library'\nversion='1.0.0'\n") + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname='app'\nversion='1.0.0'\n[dependencies]\nlibrary='1'\n") + providers := lookup("consumer/src/lib.rs", "library") + require.Len(t, providers, 1) + require.Equal(t, "provider/src/lib.rs", providers[0].Root) + writeWorkspaceFile(t, consumer, "Cargo.toml", "[package]\nname='app'\nversion='1.0.0'\n") + require.Equal(t, providers, lookup("consumer/src/lib.rs", "library"), "one pass must retain one manifest snapshot") +} diff --git a/internal/indexer/cargo_resolver_admission_test.go b/internal/indexer/cargo_resolver_admission_test.go new file mode 100644 index 000000000..311d9cef7 --- /dev/null +++ b/internal/indexer/cargo_resolver_admission_test.go @@ -0,0 +1,62 @@ +package indexer + +import ( + "iter" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "go.uber.org/zap" +) + +type cargoAdmissionStore struct { + graph.Store + scans int +} + +func (s *cargoAdmissionStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + return func(yield func(*graph.Edge) bool) { + if len(kinds) == 1 && kinds[0] == graph.EdgeImports { + s.scans++ + } + graph.EdgesLightSeq(s.Store, kinds...)(yield) + } +} + +func TestCargoResolverSingleRootAdmission(t *testing.T) { + for _, withCall := range []bool{false, true} { + name := "imports_only" + if withCall { + name = "cross_repo_call" + } + t.Run(name, func(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, first, "Cargo.toml", "[package]\nname='app'\n") + g := &cargoAdmissionStore{Store: graph.New()} + file, target := "first/src/lib.rs", "second/src/lib.rs" + for _, n := range []*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, Language: "rust", RepoPrefix: "first", WorkspaceID: "shared"}, + {ID: target, FilePath: target, Kind: graph.KindFile, Language: "rust", RepoPrefix: "second", WorkspaceID: "shared"}, + {ID: target + "::Run", FilePath: target, Name: "Run", Kind: graph.KindFunction, Language: "rust", RepoPrefix: "second", WorkspaceID: "shared"}, + } { + g.AddNode(n) + } + g.AddEdge(&graph.Edge{From: file, To: target, FilePath: file, Kind: graph.EdgeImports, Line: 1, CrossRepo: true}) + if withCall { + g.AddEdge(&graph.Edge{From: file, To: "unresolved::Run", FilePath: file, Kind: graph.EdgeCalls, Line: 2}) + } + mi := &MultiIndexer{graph: g, logger: zap.NewNop(), repos: map[string]*RepoMetadata{"first": {RootPath: first}, "second": {RootPath: second}}} + mi.newCrossRepoResolver().ResolveAll() + wantScans := 2 // Existing npm and Python import discovery on the minimal Store wrapper. + if withCall { + wantScans++ + } // Reachability for the pending call. + require.Equal(t, wantScans, g.scans, "Cargo must not consume another import scan below two roots") + edges := g.GetOutEdges(file) + require.Contains(t, edges, &graph.Edge{From: file, To: target, FilePath: file, Kind: graph.EdgeImports, Line: 1, CrossRepo: true}) + if withCall { + require.Contains(t, edges, &graph.Edge{From: file, To: target + "::Run", FilePath: file, Kind: graph.EdgeCalls, Line: 2, CrossRepo: true}) + } + }) + } +} diff --git a/internal/indexer/cold_manifest_census.go b/internal/indexer/cold_manifest_census.go index 21b27553e..98d74984d 100644 --- a/internal/indexer/cold_manifest_census.go +++ b/internal/indexer/cold_manifest_census.go @@ -165,6 +165,9 @@ func (idx *Indexer) invalidateColdFileMtimes(paths []string) { idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() for _, path := range paths { + if _, exists := idx.fileMtimes[path]; exists { + idx.fileSetRevision.Add(1) + } delete(idx.fileMtimes, path) } idx.mtimeMu.Unlock() @@ -195,6 +198,9 @@ func (idx *Indexer) publishColdFileMtimes(mtimes map[string]int64, authoritative idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() for path, mtime := range mtimes { + if _, exists := idx.fileMtimes[path]; !exists { + idx.fileSetRevision.Add(1) + } idx.fileMtimes[path] = mtime } idx.mtimeMu.Unlock() diff --git a/internal/indexer/edit_delta_capability_test.go b/internal/indexer/edit_delta_capability_test.go index 390e19b6d..8b271b886 100644 --- a/internal/indexer/edit_delta_capability_test.go +++ b/internal/indexer/edit_delta_capability_test.go @@ -38,6 +38,7 @@ var editDeltaAbsentCapabilities = map[string]string{ "anonymous{ReindexEdgesApplied}": "batch-drop receipt for persistent stores; DeltaWriter applies its reindexes locally and unconditionally through ReindexEdges", "anonymous{AddBatchChecked}": "checked durable publication: DeltaWriter stages non-failing writes in its local work graph through AddBatch; its owner checks final layer publication, and forwarding this capability would bypass the delta", "anonymous{WriteMark}": "process/store write telemetry, not a graph capability: the per-save indexer and resolver receive the underlying store handle's sampler explicitly; a MultiIndexer without this optional sampler falls back to process-wide counters", + "anonymous{ScanTestEdgeProjections}": "paging accelerator: trackedPythonImportEdges falls back to the composed EdgesLightSeq/EdgesByKind import scan with the same eligibility filter, preserving delta masks", "graph.ContractExtractionEdgeReader": "physical checked extraction adjacency; outgoingMetadataContext dispatches the composed DeltaWriter through its selected view rather than forwarding base-only rows", "graph.OverlayLayerContractExtractionEdgeReader": "physical ancestry-layer projection, not the composed delta; outgoingMetadataContext applies each selected layer and its masks explicitly", diff --git a/internal/indexer/edit_delta_meta_key_class_test.go b/internal/indexer/edit_delta_meta_key_class_test.go index da703ae7b..951837819 100644 --- a/internal/indexer/edit_delta_meta_key_class_test.go +++ b/internal/indexer/edit_delta_meta_key_class_test.go @@ -6,9 +6,13 @@ import ( "sort" "testing" + "github.com/stretchr/testify/require" "go.uber.org/zap" "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" ) // metaKeysRestatedOutsideTheFingerprint names every node or edge metadata key @@ -45,6 +49,16 @@ var metaKeysRestatedOutsideTheFingerprint = map[string]string{ // External module stubs: the module scanner / external-call attribution. "ecosystem": "modules scanner / resolver external_call_attribution (module nodes)", "import_path": "modules scanner / resolver external_call_attribution (module nodes)", + + // Python imports are restated by whole resolves, file saves, and tracked binding. + "python_import": "resolver attributeNonGoModuleImports / attributePythonModuleImportsForFiles / resolveTrackedPythonImports (exact dotted import spelling)", + "cargo_import_stub": "resolver bindCargoImport / restubTrackedCargoImportsFromView (exact pre-bind import target)", + "cargo_import_module": "resolver bindCargoImport / resolveCargoImports (matched module; cleared by Cargo restub)", + "cargo_call_stub": "resolver CrossRepoResolver.resolveEdge / reuseBindingMetaKeys carry (fresh call spelling)", + "python_import_stub": "resolver resolveTrackedPythonImports / restubTrackedPythonImportsFromView (exact pre-bind target)", + "python_call_stub": "resolver Resolver.resolveEdge / CrossRepoResolver.resolveEdge / reuseBindingMetaKeys carry", + "python_reexport_file": "resolver resolvePythonPackageReExport / resolutionMetaAdded prior-binding carry (cleared by Python restub)", + "go_import_fallback": "resolver resolveGoModuleImports (original import target retained in copied Meta)", // Test classification after extraction (test_edges.go). "is_test": "test pass (test_edges.go)", "is_test_file": "test pass (test_edges.go)", @@ -141,3 +155,97 @@ func TestEveryMetadataKeyIsFingerprintedOrRestated(t *testing.T) { }) } } + +func TestPythonImportSpellingEditColdEquivalent(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + openStore := func() graph.Store { + if backend == "memory" { + return graph.New() + } + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db + } + newIndexer := func(g graph.Store) *Indexer { + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + idx := New(g, reg, config.Default().Index, zap.NewNop()) + idx.SetRepoPrefix("consumer") + t.Cleanup(idx.Close) + return idx + } + imports := func(g graph.Store) []*graph.Edge { + var out []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports { + out = append(out, edge) + } + } + require.Len(t, out, 1) + return out + } + // Given an attributed import with a stable local alias and module target. + root := t.TempDir() + file := filepath.Join(root, "app.py") + writeFile(t, file, "import pkg.mod as current\ndef run(): return current.open()\n") + warm := openStore() + idx := newIndexer(warm) + _, err := idx.Index(root) + require.NoError(t, err) + before := imports(warm)[0] + require.Equal(t, "pkg.mod", before.Meta["python_import"]) + require.Equal(t, "module::pypi:pkg", before.To) + // When only the dotted module spelling changes, keeping the alias and top. + writeFile(t, file, "import pkg.other as current\ndef run(): return current.open()\n") + require.NoError(t, idx.IndexFile(file)) + cold := openStore() + _, err = newIndexer(cold).Index(root) + require.NoError(t, err) + // Then the incremental edge, including python_import, equals a cold index. + require.Equal(t, "pkg.other", imports(warm)[0].Meta["python_import"]) + require.Equal(t, "module::pypi:pkg", imports(warm)[0].To) + require.Equal(t, imports(cold), imports(warm)) + }) + } +} + +func TestTrackedBridgeMetadataKeysHaveRestaters(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + trees := map[string]map[string]string{ + provider: {"go.mod": "module example.org/library\n", "lib.go": "package library\nfunc Hello() {}\n", "Cargo.toml": "[package]\nname='bridge-crate'\nversion='0.1.0'\n", "src/lib.rs": "pub fn helper() {}\n", "pyproject.toml": "[project]\nname='image-dist'\n", "pkg/__init__.py": "from .impl import Factory\n", "pkg/impl.py": "class Factory: pass\n"}, + consumer: {"go.mod": "module example.org/caller\nrequire example.org/library v1.0.0\n", "main.go": "package caller\nimport library \"example.org/library\"\nfunc Run() { library.Hello() }\n", "Cargo.toml": "[package]\nname='consumer'\nversion='0.1.0'\n[dependencies]\nbridge-crate='1'\n", "src/lib.rs": "use bridge_crate::helper;\npub fn run() { helper(); }\n", "pyproject.toml": "[project]\ndependencies=['image-dist']\n", "app.py": "from pkg import Factory\ndef run(): return Factory()\n"}, + } + for root, tree := range trees { + for file, source := range tree { + writeWorkspaceFile(t, root, file, source) + } + } + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + mi.registry.Register(languages.NewRustExtractor()) + _, err := mi.IndexAll() + require.NoError(t, err) + check := func() { + present := map[string]bool{} + for _, edge := range g.AllEdges() { + for key := range edge.Meta { + present[key] = true + } + } + for _, key := range []string{"cargo_import_stub", "cargo_import_module", "cargo_call_stub", "python_import_stub", "python_call_stub", "python_reexport_file", "go_import_fallback"} { + require.True(t, present[key], "fixture must exercise %s", key) + require.NotEmpty(t, metaKeysRestatedOutsideTheFingerprint[key], "missing bridge restater for %s", key) + } + } + check() + result, err := mi.IncrementalReindexRepo("consumer", nil) + require.NoError(t, err) + require.Zero(t, result.StaleFileCount) + check() + }) + } +} diff --git a/internal/indexer/edit_delta_prior_bindings.go b/internal/indexer/edit_delta_prior_bindings.go index 7cb4e605d..90eb64b35 100644 --- a/internal/indexer/edit_delta_prior_bindings.go +++ b/internal/indexer/edit_delta_prior_bindings.go @@ -1,6 +1,7 @@ package indexer import ( + "slices" "strings" "github.com/zzet/gortex/internal/graph" @@ -168,11 +169,17 @@ func stagePriorBindings(g graph.Store, stages []*incrementalBatchStage, view inc if ambiguous || chosen == nil { continue } + meta := resolutionMetaAdded(chosen.edge.Meta, edge.Meta) + for _, key := range reuseBindingMetaKeys { + if _, kept := meta[key]; kept { + meta[key] = edge.To + } + } out[resolver.PriorBindingKey(edge.From, edge.Kind, edge.Line, edge.To)] = resolver.PriorBinding{ To: chosen.target, Origin: chosen.edge.Origin, Confidence: chosen.edge.Confidence, - Meta: resolutionMetaAdded(chosen.edge.Meta, edge.Meta), + Meta: meta, } } } @@ -259,13 +266,18 @@ func pruneBuiltinReuse(reuse map[reuseKey]*reuseVal) { // edge (semantic.ConfirmEdge). var reuseSemanticMetaKeys = []string{"confirmed_from_origin", "semantic_source"} +// reuseBindingMetaKeys are importer-owned binding stubs a reused bind keeps +// even without confirmation reuse; applyReuseSemanticMeta rewrites each to the +// fresh call spelling. +var reuseBindingMetaKeys = []string{"cargo_call_stub", "python_call_stub"} + // reuseCarriedMetaKeys are importer-owned binding markers a reused bind keeps // verbatim, even when its confirmation provenance is dropped. go_import lets a // parked cross-module Go call relink when its provider returns. var reuseCarriedMetaKeys = []string{"go_import"} // reuseSemanticMeta is the confirmation provenance and binding markers of a -// captured bind, nil when it carries neither. A confirmed bind's origin is +// captured bind, nil when it carries none. A confirmed bind's origin is // LSP-grade, and a later confirmation of an edge that already carries an // LSP-grade origin leaves it as it is, so a reuse that restored the origin // without this provenance would leave the edge different from the one a whole @@ -275,7 +287,7 @@ func reuseSemanticMeta(edge *graph.Edge) map[string]any { return nil } var out map[string]any - for _, keys := range [][]string{reuseSemanticMetaKeys, reuseCarriedMetaKeys} { + for _, keys := range [][]string{reuseSemanticMetaKeys, reuseBindingMetaKeys, reuseCarriedMetaKeys} { for _, key := range keys { if value, ok := edge.Meta[key]; ok { if out == nil { @@ -288,9 +300,9 @@ func reuseSemanticMeta(edge *graph.Edge) map[string]any { return out } -// applyReuseSemanticMeta restores a reused bind's confirmation provenance and -// carried binding markers. -func applyReuseSemanticMeta(edge *graph.Edge, meta map[string]any) { +// applyReuseSemanticMeta restores provenance, carried markers and the fresh +// call spelling of binding stubs. +func applyReuseSemanticMeta(edge *graph.Edge, meta map[string]any, originalTarget string) { if len(meta) == 0 { return } @@ -298,6 +310,9 @@ func applyReuseSemanticMeta(edge *graph.Edge, meta map[string]any) { edge.Meta = make(map[string]any, len(meta)) } for key, value := range meta { + if slices.Contains(reuseBindingMetaKeys, key) { + value = originalTarget // A name-level reuse can cover different call spellings. + } edge.Meta[key] = value } } diff --git a/internal/indexer/edit_delta_prior_bindings_test.go b/internal/indexer/edit_delta_prior_bindings_test.go index ad7137319..71754b1d7 100644 --- a/internal/indexer/edit_delta_prior_bindings_test.go +++ b/internal/indexer/edit_delta_prior_bindings_test.go @@ -13,6 +13,8 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/resolver" ) // A save that shifts every later line of the file keeps the file's unchanged @@ -81,3 +83,24 @@ func TestPriorBindingsCarryUnchangedReferencesAcrossAReparse(t *testing.T) { require.Equal(t, rows(fresh), rows(g), "the carried rows must equal a whole index of the edited tree") require.Contains(t, strings.Join(rows(g), "\n"), "-calls-> "+builderRepoPrefix+"/p/a.go::Later", "the reference the edit made resolvable is bound") } + +func TestPriorBindingsKeepFreshBridgeCallSpelling(t *testing.T) { + for _, key := range []string{"cargo_call_stub", "python_call_stub"} { + t.Run(key, func(t *testing.T) { + g := graph.New() + caller := &graph.Node{ID: "app.py::run", Name: "run", FilePath: "app.py", Kind: graph.KindFunction, StartLine: 1} + provider := &graph.Node{ID: "provider.py::f", Name: "f", FilePath: "provider.py", Kind: graph.KindFunction} + prior := &graph.Edge{From: caller.ID, To: provider.ID, Kind: graph.EdgeCalls, FilePath: "app.py", Line: 2, CrossRepo: true, Meta: map[string]any{key: "unresolved::old.f", "keep": "metadata"}} + g.AddBatch([]*graph.Node{caller, provider}, []*graph.Edge{prior}) + fresh := &graph.Edge{From: caller.ID, To: "unresolved::new.f", Kind: graph.EdgeCalls, FilePath: "app.py", Line: 2} + stage := &incrementalBatchStage{graphPath: "app.py", priorNodes: []*graph.Node{caller}, result: &parser.ExtractionResult{Nodes: []*graph.Node{caller}, Edges: []*graph.Edge{fresh}}, reuse: map[reuseKey]*reuseVal{}} + stages := []*incrementalBatchStage{stage} + carried := stagePriorBindings(g, stages, loadIncrementalPriorView(g, stages)) + binding, ok := carried[resolver.PriorBindingKey(fresh.From, fresh.Kind, fresh.Line, fresh.To)] + require.True(t, ok, "the prior binding must carry") + require.Equal(t, fresh.To, binding.Meta[key]) + require.Equal(t, "metadata", binding.Meta["keep"]) + require.Equal(t, "unresolved::old.f", prior.Meta[key], "carry must not change prior metadata") + }) + } +} diff --git a/internal/indexer/extractor_version.go b/internal/indexer/extractor_version.go index fb33b938e..1feb74117 100644 --- a/internal/indexer/extractor_version.go +++ b/internal/indexer/extractor_version.go @@ -49,6 +49,7 @@ var extractorVersions = map[string]int{ "julia": 3, // callee and macro decoding from CST children (chained/parametric callees, Base.@time, Base.:(==)), macro/operator exports, member_of for module consts and nested modules, @doc and public metadata (was: bespoke tree-sitter extractor replaced the regex extractor) "javascript": 2, // renamed per-binding imports persist import_alias Meta; older rows keyed a renamed import by its original name "typescript": 2, // renamed per-binding imports persist import_alias Meta; older rows keyed a renamed import by its original name + "python": 2, // package relative from-imports retain exact re-export bindings; from-imports retain python_bindings } // extractorSaltExtLang maps a lower-case file extension to the language diff --git a/internal/indexer/extractor_version_test.go b/internal/indexer/extractor_version_test.go index e409f6e91..966d98db8 100644 --- a/internal/indexer/extractor_version_test.go +++ b/internal/indexer/extractor_version_test.go @@ -227,8 +227,14 @@ func TestStaleLangsDetection(t *testing.T) { t.Errorf("persisted policy epoch = %d, want %d", got, postExtractionPolicyVersion) } + if got := ExtractorVersionStaleLangs(snapshotJSON(t, nil, "python")); !reflect.DeepEqual(got, []string{"python"}) { + t.Errorf("snapshot without Python re-export evidence = %v, want [python]", got) + } policySalt := "_post_extraction_policy@2" - for _, path := range []string{"model.py", "Model.java", "model.rb", "schema.ex"} { + if got, want := merkleSaltFor("model.py"), policySalt+"|python@2"; got != want { + t.Errorf("Python extractor salt = %q, want %q", got, want) + } + for _, path := range []string{"Model.java", "model.rb", "schema.ex"} { if got := merkleSaltFor(path); got != policySalt { t.Errorf("global policy salt for %s = %q, want %q", path, got, policySalt) } diff --git a/internal/indexer/file_index_failures_test.go b/internal/indexer/file_index_failures_test.go index 007be7c7a..2169d308d 100644 --- a/internal/indexer/file_index_failures_test.go +++ b/internal/indexer/file_index_failures_test.go @@ -311,7 +311,7 @@ func TestIncrementalRetryLogsTerminalReadError(t *testing.T) { require.Len(t, warnings, 1) require.Contains(t, warnings[0].ContextMap()["error"], "no space left on device") var plan DerivedInvalidationPlan - idx.evictDeletedFilesBatched([]string{"source.go"}, &plan) + idx.evictDeletedFilesBatched([]string{"source.go"}, &plan, nil) require.Empty(t, LoadFileIndexFailures(g, ""), "explicit deletion clears even a never-indexed failure") } diff --git a/internal/indexer/file_meta.go b/internal/indexer/file_meta.go index dd646109a..d49c0079a 100644 --- a/internal/indexer/file_meta.go +++ b/internal/indexer/file_meta.go @@ -197,6 +197,10 @@ const reparsePendingEnrichmentBatchSize = 256 type reparsePendingEnrichmentBatch struct { // npmConsumerFiles are unchanged importers parked by this mutation. npmConsumerFiles []string + cargoAddedFile bool + cargoConsumerFiles []string + pythonConsumerFiles []string + pythonAddedFiles []string byFile map[string]bool deferResolverCatchup bool deferredAffectedFiles map[string]struct{} diff --git a/internal/indexer/file_mtime_writers_test.go b/internal/indexer/file_mtime_writers_test.go index 22085f115..554f8af91 100644 --- a/internal/indexer/file_mtime_writers_test.go +++ b/internal/indexer/file_mtime_writers_test.go @@ -43,7 +43,7 @@ func TestBatchedFileMtimeEvictionDetachesPublishedSnapshot(t *testing.T) { idx.fileMtimes = map[string]int64{"a.go": 1, "b.go": 2} published := idx.publishFileMtimes() - idx.evictDeletedFilesBatched([]string{"a.go"}, &DerivedInvalidationPlan{}) + idx.evictDeletedFilesBatched([]string{"a.go"}, &DerivedInvalidationPlan{}, nil) assert.Equal(t, int64(1), published["a.go"]) assert.NotContains(t, idx.fileMtimes, "a.go") diff --git a/internal/indexer/go_module_import_lifecycle_test.go b/internal/indexer/go_module_import_lifecycle_test.go new file mode 100644 index 000000000..c36ee98ec --- /dev/null +++ b/internal/indexer/go_module_import_lifecycle_test.go @@ -0,0 +1,241 @@ +package indexer + +import ( + "context" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/resolver" + "github.com/zzet/gortex/internal/search" +) + +// Given a tracked provider and an unchanged Go consumer, every provider +// transition must leave the same imports as an independent cold installation. +func TestGoModuleImportLifecycleColdParity(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, reverse := range []bool{false, true} { + order := "consumer_first" + if reverse { + order = "provider_first" + } + for _, mode := range []string{"point", "watcher", "full", "reconcile", "untrack", "rename", "denied"} { + t.Run(backend+"/"+order+"/"+mode, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + writeFile(t, filepath.Join(repos[0].Path, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\n") + writeFile(t, filepath.Join(repos[0].Path, "main.go"), "package caller\nimport library \"example.org/library\"\nimport _ \"example.net/untracked\"\nfunc Caller() { library.Hello() }\n") + if mode == "denied" { + repos[1].Workspace = "other" + } + entries := []config.RepoEntry{repos[0], repos[1]} + if reverse { + entries[0], entries[1] = entries[1], entries[0] + } + track := func(idx *MultiIndexer, entries []config.RepoEntry) { + t.Helper() + for _, entry := range entries { + _, err := idx.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + } + track(mi, entries) + type importRow struct { + To string + CrossRepo bool + Confidence float64 + Origin string + } + imports := func(store graph.Store) []importRow { + t.Helper() + var rows []importRow + for _, e := range store.GetOutEdges("a/main.go") { + if e.Kind != graph.EdgeImports { + continue + } + rows = append(rows, importRow{e.To, e.CrossRepo, e.Confidence, e.Origin}) + if e.CrossRepo { + assert.Equal(t, "example.org/library", e.Meta["go_import"], "retain package identity for eviction and map changes") + } + } + sort.Slice(rows, func(i, j int) bool { return rows[i].To < rows[j].To }) + require.Len(t, rows, 2, "both the tracked and unrelated imports survive") + return rows + } + compareCold := func(active []config.RepoEntry) { + t.Helper() + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + track(cold, active) + cold.RunGlobalResolve() + assert.Equal(t, imports(coldGraph), imports(g), "warm imports must match a fresh cold graph") + } + initial := imports(g) + assert.Contains(t, initial, importRow{To: "external::example.net/untracked"}) + if mode == "denied" { + assert.Contains(t, initial, importRow{To: "dep::example.org/library"}) + } else { + assert.Contains(t, initial, importRow{To: "b/lib.go", CrossRepo: true}) + } + compareCold(entries) + lib := filepath.Join(repos[1].Path, "lib.go") + writeFile(t, lib, "package library\nfunc Hello() { println(1) }\n") + var err error + switch mode { + case "point", "denied": + err = mi.GetIndexer("b").IndexFile(lib) + case "watcher": + _, err = mi.incrementalPointRepoRaw("b", lib) + case "full": + _, err = mi.IndexRepo("b") + case "reconcile": + t.Setenv("GORTEX_WARMUP_FULL_RETRACK", "1") + restored := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), mi.configMgr, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, restored.Close(context.Background())) }) + var result *IndexResult + result, err = restored.ReconcileRepoCtx(t.Context(), repos[1], mi.FileMtimes("b")) + require.NoError(t, err) + require.True(t, result.FullRetrack) + case "untrack": + _, _, err = mi.UntrackRepoChecked(t.Context(), "b") + require.NoError(t, err) + mi.RunGlobalResolve() + compareCold([]config.RepoEntry{repos[0]}) + _, err = mi.TrackRepoCtx(t.Context(), repos[1]) + case "rename": + manifest := filepath.Join(repos[1].Path, "go.mod") + writeFile(t, manifest, "module example.org/renamed\n") + err = mi.GetIndexer("b").IndexFile(manifest) + } + require.NoError(t, err) + compareCold(entries) + mi.RunGlobalResolve() + compareCold(entries) + }) + } + } + } +} + +func TestGoModuleImportParkBeforeEviction(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"repo", "file", "deleted_batch"} { + t.Run(backend+"/"+route, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "park.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + g.AddBatch([]*graph.Node{ + {ID: "a/main.go", Name: "main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, + {ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{{From: "a/main.go", To: "b/lib.go", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2, CrossRepo: true, Confidence: 0.6, ConfidenceLabel: "EXTRACTED", Origin: graph.OriginASTInferred, Meta: map[string]any{"go_import": "example.org/lib"}}}) + switch route { + case "repo": + restubGoModuleRepoIncoming(g, "b") + case "deleted_batch": + provider := g.GetNode("b/lib.go") + carried, _, _, _ := restubIncomingRefsFromView(g, + []*incrementalBatchStage{{graphPath: "b/lib.go", priorNodes: []*graph.Node{provider}}}, + incrementalPriorView{inByNode: g.GetInEdgesByNodeIDs([]string{provider.ID})}) + assert.Empty(t, carried, "a deleted file cannot carry a resolved target") + default: + idx := newTestIndexer(g) + idx.SetRepoPrefix("b") + t.Cleanup(idx.Close) + idx.restubIncomingRefs("b/lib.go") + } + g.EvictFile("b/lib.go") + rows := g.GetOutEdges("a/main.go") + require.Len(t, rows, 1, "the consumer-owned row must survive provider eviction") + assert.Equal(t, "unresolved::import::example.org/lib", rows[0].To) + assert.False(t, rows[0].CrossRepo) + assert.Empty(t, rows[0].ConfidenceLabel) + assert.Empty(t, rows[0].Origin) + assert.Zero(t, rows[0].Confidence) + assert.Equal(t, "example.org/lib", rows[0].Meta["go_import"]) + g.AddNode(&graph.Node{ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + cr := resolver.NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.org/lib"}) + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/lib.go"}) + assert.False(t, admission.Refused) + rows = g.GetOutEdges("a/main.go") + require.Len(t, rows, 1) + assert.Equal(t, "b/lib.go", rows[0].To, "the exact incoming frontier must relink without a retained import statement") + assert.True(t, rows[0].CrossRepo) + }) + } + } +} + +// The optional asset capability makes the otherwise code-only Go language +// exercise the content-admission return as well as the ordinary size cap. +type goImportAdmissionExtractor struct{ parser.Extractor } + +func (goImportAdmissionExtractor) AssetClass() parser.AssetClass { return parser.AssetData } + +func TestGoModuleLocalImportSurvivesAdmissionSkip(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"size", "content", "normal"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + root := t.TempDir() + writeWorkspaceFile(t, root, "main.go", "package p\n") + writeWorkspaceFile(t, root, "lib.go", "package p\nfunc F() {}\n") + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + idx := newTestIndexer(g) + _, err := idx.Index(root) + require.NoError(t, err) + g.AddEdge(&graph.Edge{From: "main.go", FilePath: "main.go", To: "lib.go", Kind: graph.EdgeImports, Line: 1, Origin: graph.OriginASTResolved, Confidence: 1, Meta: map[string]any{"go_import": "example.org/lib", "go_import_fallback": "external::example.org/lib"}}) + if mode == "size" { + idx.config.MaxFileSize = 1 + } + if mode == "content" { + ext, ok := idx.registry.GetByLanguage("go") + require.True(t, ok) + idx.registry.Register(goImportAdmissionExtractor{ext}) + idx.config.Content.IndexData = false + } + for pass := 0; pass < 2; pass++ { + require.NoError(t, idx.IndexFileNoResolve(filepath.Join(root, "lib.go"))) + var imports []*graph.Edge + for _, e := range g.GetOutEdges("main.go") { + if e.Kind == graph.EdgeImports { + imports = append(imports, e) + } + } + require.Len(t, imports, 1) + if mode == "normal" { + require.Equal(t, "lib.go", imports[0].To) + } else { + require.Equal(t, "unresolved::import::example.org/lib", imports[0].To) + require.False(t, imports[0].CrossRepo) + require.Empty(t, imports[0].Origin) + require.Zero(t, imports[0].Confidence) + } + } + }) + } + } +} diff --git a/internal/indexer/go_module_import_recovery_test.go b/internal/indexer/go_module_import_recovery_test.go new file mode 100644 index 000000000..dc40ea466 --- /dev/null +++ b/internal/indexer/go_module_import_recovery_test.go @@ -0,0 +1,409 @@ +package indexer + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/search" +) + +func TestGoModuleImportReplaceRecoveryColdParity(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, reverse := range []bool{false, true} { + for _, scen := range []string{"inreplace", "samedir"} { + t.Run(fmt.Sprintf("%s/%s/reverse=%v", backend, scen, reverse), func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a, b := repos[0].Path, repos[1].Path + gomod := "module example.org/caller\nrequire example.org/library v1.0.0\n" + if scen == "inreplace" { + gomod += "replace example.org/library => ./third_party/library\n" + require.NoError(t, os.MkdirAll(filepath.Join(a, "third_party", "library"), 0o755)) + writeFile(t, filepath.Join(a, "third_party", "library", "lib.go"), "package library\nfunc Hello() {}\n") + } else { + require.NoError(t, os.MkdirAll(filepath.Join(a, "pkg", "library"), 0o755)) + writeFile(t, filepath.Join(a, "pkg", "library", "l.go"), "package library\nfunc Hello() {}\n") + } + writeFile(t, filepath.Join(a, "go.mod"), gomod) + writeFile(t, filepath.Join(a, "main.go"), "package caller\nimport library \"example.org/library\"\nfunc Caller() { library.Hello() }\n") + entries := []config.RepoEntry{repos[0], repos[1]} + if reverse { + entries[0], entries[1] = entries[1], entries[0] + } + for _, e := range entries { + _, err := mi.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + compareCold := func() { + t.Helper() + var cg graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + defer func() { require.NoError(t, db.Close()) }() + cg = db + } + cold := NewMultiIndexer(cg, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + defer func() { require.NoError(t, cold.Close(context.Background())) }() + for _, e := range entries { + _, err := cold.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + cold.RunGlobalResolve() + require.Len(t, goImportRecoveryRows(cg, "a/main.go"), 1) + assert.Equal(t, goImportRecoveryRows(cg, "a/main.go"), goImportRecoveryRows(g, "a/main.go")) + } + compareCold() + lib := filepath.Join(b, "lib.go") + for i := 0; i < 3; i++ { + writeFile(t, lib, "package library\nfunc Hello() { println("+string(rune('1'+i))+") }\n") + _, err := mi.IncrementalReindexRepo("b", []string{lib}) + require.NoError(t, err) + compareCold() + } + for i := 0; i < 3; i++ { + mi.RunGlobalResolve() + compareCold() + } + assert.Len(t, goImportRecoveryRows(g, "a/main.go"), 1, "one import statement must keep one IMPORT edge") + }) + } + } + } +} + +func goImportRecoveryRows(g graph.Store, file string) []string { + var rows []string + for _, e := range g.GetOutEdges(file) { + if e.Kind != graph.EdgeImports { + continue + } + rows = append(rows, fmt.Sprintf("L%d %s cross=%v conf=%v origin=%s gi=%v fb=%v", e.Line, e.To, e.CrossRepo, e.Confidence, e.Origin, e.Meta["go_import"], e.Meta["go_import_fallback"])) + } + sort.Strings(rows) + return rows +} + +func TestGoModuleImportPopularProviderSave(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, reverse := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/reverse=%v", backend, reverse), func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a, b := repos[0].Path, repos[1].Path + writeFile(t, filepath.Join(a, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\n") + writeFile(t, filepath.Join(a, "main.go"), "package caller\nimport library \"example.org/library\"\nfunc Caller() { library.New() }\n") + writeFile(t, filepath.Join(a, "newpkg.go"), "package caller\nimport _ \"example.org/library/new\"\n") + entries := []config.RepoEntry{repos[0], repos[1]} + if reverse { + entries[0], entries[1] = entries[1], entries[0] + } + track := func(idx *MultiIndexer) { + t.Helper() + for _, entry := range entries { + _, err := idx.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + } + track(mi) + populate := func(store graph.Store) { + var nodes []*graph.Node + var edges []*graph.Edge + for i := 0; i < graph.MaxIncomingSourceCandidateRows+1; i++ { + f := fmt.Sprintf("a/filler%d.go", i) + nodes = append(nodes, &graph.Node{ID: f, Kind: graph.KindFile, FilePath: f, RepoPrefix: "a", WorkspaceID: "shared", Language: "go"}, &graph.Node{ID: f + "::import::example.org/library", Kind: graph.KindImport, FilePath: f, RepoPrefix: "a", WorkspaceID: "shared", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/library"}}) + edges = append(edges, &graph.Edge{From: f, To: "b/lib.go", Kind: graph.EdgeImports, FilePath: f, Line: 2, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/library", "go_import_fallback": "dep::example.org/library"}}) + } + store.AddBatch(nodes, edges) + } + populate(g) + // Reconcile the unchanged persisted graph into a new process, then seed on a full pass. + restored := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), mi.configMgr, zap.NewNop()) + defer func() { require.NoError(t, restored.Close(context.Background())) }() + for _, entry := range entries { + _, err := restored.ReconcileRepoCtx(t.Context(), entry, mi.FileMtimes(entry.Name)) + require.NoError(t, err) + } + restored.RunGlobalResolve() + // Filler consumers share the tracked caller's workspace, so the full + // pass must keep their imports bound and the save over budget. + require.Greater(t, len(g.GetInEdges("b/lib.go")), graph.MaxIncomingSourceCandidateRows) + f := filepath.Join(b, "new.go") + writeFile(t, f, "package library\nfunc New() {}\n") + require.NoError(t, restored.GetIndexer("b").IndexFile(f)) + assert.Equal(t, "b/new.go::New", goModuleLifecycleCall(t, g).To, "the actual provider save must bind the unchanged caller") + require.NoError(t, os.MkdirAll(filepath.Join(b, "new"), 0o755)) + pkg := filepath.Join(b, "new", "p.go") + writeFile(t, pkg, "package newpkg\n") + require.NoError(t, restored.GetIndexer("b").IndexFile(pkg)) + restored.RunGlobalResolve() + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + defer func() { require.NoError(t, db.Close()) }() + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + defer func() { require.NoError(t, cold.Close(context.Background())) }() + track(cold) + populate(coldGraph) + cold.RunGlobalResolve() + for _, file := range []string{"a/main.go", "a/newpkg.go"} { + assert.Equal(t, goImportRecoveryRows(coldGraph, file), goImportRecoveryRows(g, file)) + } + assert.Equal(t, goModuleLifecycleCall(t, coldGraph).To, goModuleLifecycleCall(t, g).To) + }) + } + } +} + +func TestGoModuleImportReplaceAddedLater(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a := repos[0].Path + manifest := filepath.Join(a, "go.mod") + writeFile(t, manifest, "module example.org/caller\nrequire example.org/library v1.0.0\n") + entries := []config.RepoEntry{repos[0], repos[1]} + for _, e := range entries { + _, err := mi.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) + require.NoError(t, os.MkdirAll(filepath.Join(a, "third_party", "library"), 0o755)) + vendored := filepath.Join(a, "third_party", "library", "lib.go") + writeFile(t, vendored, "package library\nfunc Hello() {}\n") + writeFile(t, manifest, "module example.org/caller\nrequire example.org/library v1.0.0\nreplace example.org/library => ./third_party/library\n") + _, err := mi.IncrementalReindexRepo("a", []string{manifest, vendored}) + require.NoError(t, err) + assert.Equal(t, "a/third_party/library/lib.go::Hello", goModuleLifecycleCall(t, g).To) + mi.RunGlobalResolve() + assert.Equal(t, "a/third_party/library/lib.go::Hello", goModuleLifecycleCall(t, g).To) + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + for _, e := range entries { + _, err := cold.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + cold.RunGlobalResolve() + require.Len(t, goImportRecoveryRows(coldGraph, "a/main.go"), 1) + assert.Equal(t, goModuleLifecycleCall(t, coldGraph).To, goModuleLifecycleCall(t, g).To) + // Retain retry metadata on the local row so removing replace can rebind it. + assert.Equal(t, goImportBindingRows(coldGraph, "a/main.go"), goImportBindingRows(g, "a/main.go")) + }) + } +} + +func TestGoModuleImportReplaceAddedThenRemoved(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a := repos[0].Path + manifest := filepath.Join(a, "go.mod") + plain := "module example.org/caller\nrequire example.org/library v1.0.0\n" + writeFile(t, manifest, plain) + entries := []config.RepoEntry{repos[0], repos[1]} + for _, e := range entries { + _, err := mi.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) + require.NoError(t, os.MkdirAll(filepath.Join(a, "third_party", "library"), 0o755)) + vendored := filepath.Join(a, "third_party", "library", "lib.go") + writeFile(t, vendored, "package library\nfunc Hello() {}\n") + writeFile(t, manifest, plain+"replace example.org/library => ./third_party/library\n") + _, err := mi.IncrementalReindexRepo("a", []string{manifest, vendored}) + require.NoError(t, err) + assert.Equal(t, "a/third_party/library/lib.go::Hello", goModuleLifecycleCall(t, g).To) + writeFile(t, manifest, plain) + _, err = mi.IncrementalReindexRepo("a", []string{manifest}) + require.NoError(t, err) + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) + mi.RunGlobalResolve() + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + for _, e := range entries { + _, err := cold.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + cold.RunGlobalResolve() + assert.Equal(t, goModuleLifecycleCall(t, coldGraph).To, goModuleLifecycleCall(t, g).To) + assert.Equal(t, goImportRecoveryRows(coldGraph, "a/main.go"), goImportRecoveryRows(g, "a/main.go")) + }) + } +} + +func goImportBindingRows(g graph.Store, file string) []string { + var rows []string + for _, e := range g.GetOutEdges(file) { + if e.Kind == graph.EdgeImports { + rows = append(rows, fmt.Sprintf("L%d %s cross=%v conf=%v origin=%s", e.Line, e.To, e.CrossRepo, e.Confidence, e.Origin)) + } + } + sort.Strings(rows) + return rows +} + +func TestGoModuleImportDuplicateAliases(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scope := range []string{"same_repo", "cross_repo"} { + t.Run(backend+"/"+scope, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + writeFile(t, filepath.Join(repos[0].Path, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\n") + provider, target, importPath := repos[1].Path, "b/sub/lib.go", "example.org/library/sub" + if scope == "same_repo" { + provider, target, importPath = repos[0].Path, "a/sub/lib.go", "example.org/caller/sub" + } + require.NoError(t, os.MkdirAll(filepath.Join(provider, "sub"), 0o755)) + writeFile(t, filepath.Join(provider, "sub", "lib.go"), "package sub\nfunc Hello() {}\n") + writeFile(t, filepath.Join(repos[0].Path, "main.go"), "package caller\nimport (\n first \""+importPath+"\"\n second \""+importPath+"\"\n)\nfunc Caller() {\n first.Hello()\n second.Hello()\n}\n") + for _, entry := range []config.RepoEntry{repos[1], repos[0]} { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + check := func() { + t.Helper() + var imports, calls []*graph.Edge + for _, edge := range g.GetOutEdges("a/main.go") { + if edge.Kind == graph.EdgeImports { + imports = append(imports, edge) + } + } + for _, edge := range g.GetOutEdges("a/main.go::Caller") { + if edge.Kind == graph.EdgeCalls { + calls = append(calls, edge) + } + } + require.Len(t, imports, 2, "line identity preserves both imports, as for same-repo aliases") + require.Len(t, calls, 2, "calls through both aliases must survive") + for _, edge := range imports { + assert.Equal(t, target, edge.To, "import line %d", edge.Line) + assert.Equal(t, scope == "cross_repo", edge.CrossRepo) + } + for _, edge := range calls { + assert.Equal(t, target+"::Hello", edge.To, "call line %d", edge.Line) + } + } + check() + mi.RunGlobalResolve() + check() + }) + } + } +} + +func TestGoModuleReplacementFileSaveColdParity(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"file", "batch", "fallback"} { + t.Run(backend+"/"+route, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + replacement := filepath.Join(repos[0].Path, "third_party", "library", "lib.go") + require.NoError(t, os.MkdirAll(filepath.Dir(replacement), 0o755)) + writeFile(t, filepath.Join(repos[0].Path, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\nreplace example.org/library => ./third_party/library\n") + writeFile(t, filepath.Join(repos[0].Path, "main.go"), "package caller\nimport library \"example.org/library\"\nfunc Caller() { library.Hello() }\n") + writeFile(t, replacement, "package library\nfunc Hello() {}\n") + for _, entry := range repos { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + // Reproduce a persisted same-repo binding emitted by the module bridge. + for _, imp := range g.GetOutEdges("a/main.go") { + if imp.Kind != graph.EdgeImports { + continue + } + imp.Meta = map[string]any{"go_import": "example.org/library", "go_import_fallback": "dep::example.org/library"} + g.ReindexEdges([]graph.EdgeReindex{{Edge: imp, OldTo: imp.To}}) + } + rows := func(store graph.Store) []string { + var result []string + for _, imp := range store.GetOutEdges("a/main.go") { + if imp.Kind == graph.EdgeImports { + result = append(result, fmt.Sprintf("L%d %s cross=%v", imp.Line, imp.To, imp.CrossRepo)) + } + } + sort.Strings(result) + return result + } + before := rows(g) + require.Len(t, before, 1) + require.Contains(t, before[0], "a/third_party/library/lib.go cross=false") + writeFile(t, replacement, "package library\nfunc Hello() { println(1) }\n") + switch route { + case "fallback": + require.NoError(t, mi.GetIndexer("a").indexFile(replacement, true, nil)) + case "file": + require.NoError(t, mi.GetIndexer("a").IndexFile(replacement)) + default: + _, err := mi.IncrementalReindexRepo("a", []string{replacement}) + require.NoError(t, err) + } + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "replacement-cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + for _, entry := range repos { + _, err := cold.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + require.Equal(t, before, rows(coldGraph)) + require.Equal(t, rows(coldGraph), rows(g), "provider save must match batch and cold imports") + }) + } + } +} + +func TestSingleFileRestubLeavesFileReferencesForEviction(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file, page := "app/+page.server.ts", "app/+page.svelte" + g.AddNode(&graph.Node{ID: file, Name: "+page.server.ts", FilePath: file, Kind: graph.KindFile}) + g.AddNode(&graph.Node{ID: page, Name: "+page.svelte", FilePath: page, Kind: graph.KindFile}) + // SvelteKit load synthesis references the server file, not a named symbol. + g.AddEdge(&graph.Edge{From: page, To: file, FilePath: page, Kind: graph.EdgeReferences}) + idx := &Indexer{graph: g} + idx.restubIncomingRefs(file) + refs := g.GetOutEdges(page) + require.Len(t, refs, 1) + require.Equal(t, file, refs[0].To, "file references must not become unresolvable name stubs") + g.EvictFile(file) + require.Empty(t, g.GetOutEdges(page)) + }) + } +} diff --git a/internal/indexer/go_module_import_test.go b/internal/indexer/go_module_import_test.go new file mode 100644 index 000000000..e57d4b1a1 --- /dev/null +++ b/internal/indexer/go_module_import_test.go @@ -0,0 +1,133 @@ +package indexer + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/search" +) + +func TestMultiIndexer_GoModuleImport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"track", "reverse", "all", "sub_track", "sub_reverse"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + consumerRoot, provider := t.TempDir(), t.TempDir() + consumer, consumerFile := consumerRoot, "main.go" + if mode == "sub_track" || mode == "sub_reverse" { + consumer = filepath.Join(consumerRoot, "svc") + consumerFile = "svc/main.go" + require.NoError(t, os.MkdirAll(consumer, 0o755)) + } + writeFile(t, filepath.Join(consumer, "go.mod"), "module example.com/consumer\nrequire example.com/lib v1.0.0\n") + writeFile(t, filepath.Join(consumer, "main.go"), `package main +import "example.com/lib" +func main() { lib.F() } +`) + writeFile(t, filepath.Join(provider, "go.mod"), "module example.com/lib\n") + writeFile(t, filepath.Join(provider, "lib.go"), "package lib\nfunc F() {}\n") + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + cm := newTestConfigManager(t) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + entries := []config.RepoEntry{ + {Path: consumerRoot, Name: "consumer", Workspace: "shared"}, + {Path: provider, Name: "provider", Workspace: "shared"}, + } + if mode == "all" { + cm.Global().Repos = entries + _, err := mi.IndexAll() + require.NoError(t, err) + } else { + if mode == "reverse" || mode == "sub_reverse" { + entries[0], entries[1] = entries[1], entries[0] + } + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + } + mi.RunGlobalResolve() + var imports []*graph.Edge + for _, e := range g.GetOutEdges("consumer/" + consumerFile) { + if e.Kind == graph.EdgeImports { + imports = append(imports, e) + } + } + require.Len(t, imports, 1) + assert.Equal(t, "provider/lib.go", imports[0].To) + assert.True(t, imports[0].CrossRepo) + contractID := "dep::example.com/lib" + if mode == "all" { + contractID = "dep::provider::lib" + } + if consumerFile == "main.go" { + require.NotNil(t, g.GetNode(contractID), "keep the go.mod dependency contract") + } + var calls []string + for _, e := range g.GetOutEdges("consumer/" + consumerFile + "::main") { + if e.Kind == graph.EdgeCalls { + calls = append(calls, e.To) + } + } + assert.Equal(t, []string{"provider/lib.go::F"}, calls) + for _, mode := range []string{"file", "repo"} { + writeFile(t, filepath.Join(consumer, "main.go"), "// edited\npackage main\nimport \"example.com/lib\"\nfunc main() { lib.F() }\n") + switch mode { + case "file": + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "main.go"))) + case "repo": + _, err := mi.IndexRepo("consumer") + require.NoError(t, err) + } + var targets []string + for _, e := range g.GetOutEdges("consumer/" + consumerFile) { + if e.Kind == graph.EdgeImports { + targets = append(targets, e.To) + assert.True(t, e.CrossRepo) + } + } + assert.Equal(t, []string{"provider/lib.go"}, targets, mode) + mi.RunGlobalResolve() + var afterGlobal []string + for _, e := range g.GetOutEdges("consumer/" + consumerFile) { + if e.Kind == graph.EdgeImports { + afterGlobal = append(afterGlobal, e.To) + } + } + assert.Equal(t, []string{"provider/lib.go"}, afterGlobal, mode+" global") + } + writeFile(t, filepath.Join(provider, "lib.go"), "package lib\nfunc F() { /* edited */ }\n") + require.NoError(t, mi.GetIndexer("provider").IndexFile(filepath.Join(provider, "lib.go"))) + for _, phase := range []string{"provider_file", "provider_repo"} { + if phase == "provider_repo" { + _, err := mi.IndexRepo("provider") + require.NoError(t, err) + } + var targets []string + for _, e := range g.GetOutEdges("consumer/" + consumerFile) { + if e.Kind == graph.EdgeImports { + targets = append(targets, e.To) + } + } + assert.Equal(t, []string{"provider/lib.go"}, targets, phase) + } + }) + } + } +} diff --git a/internal/indexer/go_module_import_tie_test.go b/internal/indexer/go_module_import_tie_test.go new file mode 100644 index 000000000..686ae65e5 --- /dev/null +++ b/internal/indexer/go_module_import_tie_test.go @@ -0,0 +1,53 @@ +package indexer + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/search" +) + +func TestGoModuleImportTieBrokenByRename(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + a := repos[0].Path + writeFile(t, filepath.Join(a, "go.mod"), "module example.org/caller\nrequire example.org/library v1.0.0\n") + entries := []config.RepoEntry{repos[0], repos[1], repos[2]} + for _, e := range entries { + _, err := mi.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + mi.RunGlobalResolve() + assert.False(t, goModuleLifecycleCall(t, g).CrossRepo, "ambiguous modules must not bind") + manifest := filepath.Join(repos[2].Path, "go.mod") + writeFile(t, manifest, "module example.org/forked\n") + require.NoError(t, mi.GetIndexer("fork").IndexFile(manifest)) + assert.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) + var coldGraph graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + coldGraph = db + } + cold := NewMultiIndexer(coldGraph, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, cold.Close(context.Background())) }) + for _, e := range entries { + _, err := cold.TrackRepoCtx(t.Context(), e) + require.NoError(t, err) + } + cold.RunGlobalResolve() + assert.Equal(t, goModuleLifecycleCall(t, coldGraph).To, goModuleLifecycleCall(t, g).To) + assert.Equal(t, goImportRecoveryRows(coldGraph, "a/main.go"), goImportRecoveryRows(g, "a/main.go")) + }) + } +} diff --git a/internal/indexer/go_module_lifecycle_test.go b/internal/indexer/go_module_lifecycle_test.go index 3fffefb44..d38f2b6c8 100644 --- a/internal/indexer/go_module_lifecycle_test.go +++ b/internal/indexer/go_module_lifecycle_test.go @@ -294,7 +294,7 @@ func TestNonGoRepoEvictionSkipsModuleRestub(t *testing.T) { g.AddNode(&graph.Node{ID: "js/main.js::Hello", RepoPrefix: "js", FilePath: "js/main.js", Kind: graph.KindFunction, Language: "javascript", Name: "Hello"}) mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) - _, _, err := mi.purgeRepositoryPayload("js", &RepoMetadata{RootPath: t.TempDir()}) + _, _, err := mi.purgeRepositoryPayload("js", &RepoMetadata{RootPath: t.TempDir()}, nil) require.NoError(t, err) assert.Zero(t, g.definitionScans, "non-Go eviction must not run the module incoming-edge scan") assert.Nil(t, g.GetNode("js/main.js::Hello")) diff --git a/internal/indexer/go_module_restub.go b/internal/indexer/go_module_restub.go index d0dfeed79..fb2b193d9 100644 --- a/internal/indexer/go_module_restub.go +++ b/internal/indexer/go_module_restub.go @@ -7,8 +7,13 @@ import ( ) func goModuleRestubTarget(e *graph.Edge, fallback string) string { - if spec, ok := e.Meta["go_import"].(string); ok && spec != "" && e.CrossRepo && strings.HasSuffix(e.FilePath, ".go") { - return graph.UnresolvedMarker + "extern::" + spec + if spec, ok := e.Meta["go_import"].(string); ok && spec != "" && strings.HasSuffix(e.FilePath, ".go") { + if e.Kind == graph.EdgeImports { + return graph.UnresolvedMarker + "import::" + spec + } + if e.CrossRepo { + return graph.UnresolvedMarker + "extern::" + spec + } } return fallback } @@ -25,7 +30,7 @@ func restubGoModuleRepoIncoming(g graph.Store, repo string) []graph.EdgeIdentity var batch []graph.EdgeReindex for _, incoming := range g.GetInEdgesByNodeIDs(ids) { for _, e := range incoming { - if !graph.IsResolvableRefEdge(e.Kind) || graph.RepoPrefixOfID(e.From) == repo { + if !graph.IsResolvableRefEdge(e.Kind) && e.Kind != graph.EdgeImports || graph.RepoPrefixOfID(e.From) == repo { continue } stub := goModuleRestubTarget(e, "") @@ -43,7 +48,7 @@ func restubGoModuleRepoIncoming(g graph.Store, repo string) []graph.EdgeIdentity g.ReindexEdges(batch) ids = ids[:0] } - for n := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFunction, graph.KindType, graph.KindInterface) { + for n := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { ids = append(ids, n.ID) if len(ids) == 512 { flush() diff --git a/internal/indexer/go_module_revision_test.go b/internal/indexer/go_module_revision_test.go index 5b9f1b012..91543183c 100644 --- a/internal/indexer/go_module_revision_test.go +++ b/internal/indexer/go_module_revision_test.go @@ -2,6 +2,7 @@ package indexer import ( "context" + "maps" "os" "path/filepath" "strings" @@ -138,6 +139,14 @@ func TestGoModuleUntrackSettlesColdEquivalent(t *testing.T) { cold := revisionColdGraph(t, backend, coldRepos) require.Equal(t, want, goModuleLifecycleCall(t, cold).To) assert.Equal(t, want, goModuleLifecycleCall(t, g).To) + // Import settlement must match cold immediately, without a global resolve. + wantImports := revisionCallTargets(cold, "a/main.go", graph.EdgeImports) + if mode != "tie" { + require.Equal(t, []string{"external::example.org/library"}, wantImports) + } else { + require.Equal(t, []string{"fork/lib.go"}, wantImports) + } + assert.ElementsMatch(t, wantImports, revisionCallTargets(g, "a/main.go", graph.EdgeImports)) }) } } @@ -244,10 +253,13 @@ func TestGoModuleUntrackRetainsOnlyParkedIdentities(t *testing.T) { require.NoError(t, err) } parked := restubGoModuleRepoIncoming(g, "b") - require.Len(t, parked, 1) + require.ElementsMatch(t, []graph.EdgeIdentity{ + {From: "a/main.go", To: "unresolved::import::example.org/library", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}, + {From: "a/main.go::Caller", To: "unresolved::extern::example.org/library::Hello", Kind: graph.EdgeCalls, FilePath: "a/main.go", Line: 3}, + }, parked, "both caller-owned references must survive and settle") assert.IsType(t, []graph.EdgeIdentity{}, parked, "teardown must not retain full metadata-bearing edge objects") g.EvictFile("b/lib.go") - require.NoError(t, mi.finalizeUntrackedGoModuleCalls(parked)) + require.NoError(t, mi.finalizeUntrackedGoModuleCalls("b", parked)) cold := revisionColdGraph(t, backend, repos[:1]) assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) }) @@ -316,13 +328,16 @@ func TestGoModuleParkedLookupRejectsMismatchedRows(t *testing.T) { require.NoError(t, err) } parked := restubGoModuleRepoIncoming(g, "b") - require.Len(t, parked, 1) + require.ElementsMatch(t, []graph.EdgeIdentity{ + {From: "a/main.go", To: "unresolved::import::example.org/library", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}, + {From: "a/main.go::Caller", To: "unresolved::extern::example.org/library::Hello", Kind: graph.EdgeCalls, FilePath: "a/main.go", Line: 3}, + }, parked, "both caller-owned references must survive and settle") require.NotEmpty(t, g.(graph.EdgeIdentityBatchFinder).FindEdgesByIdentities(parked), "parked row must exist before purge") g.EvictFile("b/lib.go") require.NotEmpty(t, g.(graph.EdgeIdentityBatchFinder).FindEdgesByIdentities(parked), "parked row must survive purge") wrapped := &goModuleMismatchedFinderStore{Store: g} mi.graph = wrapped - require.NoError(t, mi.finalizeUntrackedGoModuleCalls(parked)) + require.NoError(t, mi.finalizeUntrackedGoModuleCalls("b", parked)) assert.Zero(t, wrapped.writes, "identity mismatch must not schedule an edge mutation") assert.Zero(t, wrapped.lookups, "identity mismatch must be rejected before symbol lookup") }) @@ -366,7 +381,7 @@ func TestGoModuleParkedLookupWithoutFinder(t *testing.T) { parked := restubGoModuleRepoIncoming(g, "b") g.EvictFile("b/lib.go") mi.graph = &goModuleNoFinderStore{Store: g} - require.NoError(t, mi.finalizeUntrackedGoModuleCalls(parked)) + require.NoError(t, mi.finalizeUntrackedGoModuleCalls("b", parked)) cold := revisionColdGraph(t, backend, repos[:1]) assert.ElementsMatch(t, revisionCallTargets(cold, "a/main.go::Caller", graph.EdgeCalls), revisionCallTargets(g, "a/main.go::Caller", graph.EdgeCalls)) }) @@ -448,3 +463,41 @@ func TestGoModuleSameStampCallerReindexColdEquivalent(t *testing.T) { }) } } + +// Given A imports B and an unrelated C is tracked, when C is untracked, +// then A's resolved import and B's package projection remain unchanged. +func TestGoModuleUntrackUnrelatedRepoKeepsImport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, repos := goModuleLifecycleFixture(t, backend) + third := t.TempDir() + writeFile(t, filepath.Join(third, "go.mod"), "module example.org/unrelated\n") + writeFile(t, filepath.Join(third, "lib.go"), "package unrelated\nfunc Other() {}\n") + for _, entry := range append(repos[:2], config.RepoEntry{Path: third, Name: "c", Workspace: "shared"}) { + _, err := mi.TrackRepoCtx(t.Context(), entry) + require.NoError(t, err) + } + imports := g.GetOutEdges("a/main.go") + var before *graph.Edge + for _, edge := range imports { + if edge.Kind == graph.EdgeImports { + copy := *edge + copy.Meta = maps.Clone(edge.Meta) + before = © + } + } + require.NotNil(t, before) + require.Equal(t, "b/lib.go", before.To) + require.True(t, before.CrossRepo) + _, _, err := mi.UntrackRepoChecked(t.Context(), "c") + require.NoError(t, err) + for _, edge := range g.GetOutEdges("a/main.go") { + if edge.Kind == graph.EdgeImports { + require.Equal(t, *before, *edge) + } + } + require.Equal(t, []string{"b/lib.go"}, revisionCallTargets(g, "a/main.go", graph.EdgeImports)) + require.Equal(t, "b/lib.go::Hello", goModuleLifecycleCall(t, g).To) + }) + } +} diff --git a/internal/indexer/go_module_source_store_test.go b/internal/indexer/go_module_source_store_test.go index a805c76b6..877728a04 100644 --- a/internal/indexer/go_module_source_store_test.go +++ b/internal/indexer/go_module_source_store_test.go @@ -87,7 +87,7 @@ func TestGoModuleSourceStoreDropsTrustAfterConcurrentEviction(t *testing.T) { if reads == 1 { // The file is deleted and evicted during the read. require.NoError(t, os.Remove(main)) - idx.evictDeletedFilesBatched([]string{"main.go"}, &DerivedInvalidationPlan{}) + idx.evictDeletedFilesBatched([]string{"main.go"}, &DerivedInvalidationPlan{}, nil) } return src, err } diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 2779b774d..1c6c212af 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -158,7 +158,7 @@ func (idx *Indexer) reindexIncrementalFilesBatched( deleteTiming := idx.startReconcilePhase(idx.logger, idx.repoPrefix, "graph_delete", zap.Int("deleted_files", len(deletedFiles))) defer deleteTiming.abort() - idx.evictDeletedFilesBatched(deletedFiles, &invalidation) + idx.evictDeletedFilesBatched(deletedFiles, &invalidation, markerBatch) deleteTiming.complete(nil) passPlan, reparsed, failed, versionChanged := idx.reindexIncrementalStalePass( @@ -519,6 +519,12 @@ func (idx *Indexer) reindexIncrementalFallback( if err := idx.indexFile(fallback.filePath, resolve, markerBatch); err != nil { return err } + if markerBatch != nil && len(fallback.priorNodes) == 0 && strings.HasSuffix(fallback.graphPath, ".rs") { + markerBatch.cargoAddedFile = true + } + if markerBatch != nil && len(fallback.priorNodes) == 0 && strings.HasSuffix(fallback.graphPath, ".py") { + markerBatch.pythonAddedFiles = append(markerBatch.pythonAddedFiles, fallback.graphPath) + } freshNodes := idx.graph.GetFileNodesByPaths([]string{fallback.graphPath})[fallback.graphPath] freshDerived := storedDerivedFingerprints(freshNodes) if fallback.probeOK && fallback.probe.derived.complete() { @@ -996,7 +1002,7 @@ func applyResolvedOutEdgesFromView( edge.Origin = value.origin edge.Tier = value.tier applyReuseResolutionTag(edge, value.resolution) - applyReuseSemanticMeta(edge, value.semanticMeta) + applyReuseSemanticMeta(edge, value.semanticMeta, oldTo) retargeted = append(retargeted, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) reused++ } @@ -1081,6 +1087,9 @@ func (idx *Indexer) commitStructuralIncrementalBatch( var edges []*graph.Edge var priorPending []*graph.Edge for _, stage := range stages { + if markerBatch != nil && len(stage.priorNodes) == 0 && strings.HasSuffix(stage.graphPath, ".py") { + markerBatch.pythonAddedFiles = append(markerBatch.pythonAddedFiles, stage.graphPath) + } paths = append(paths, stage.graphPath) legacyIDs = append(legacyIDs, stage.legacyIDs...) if stage.legacyPath != "" { @@ -1110,9 +1119,16 @@ func (idx *Indexer) commitStructuralIncrementalBatch( markerBatch.recordDeferredPriorBindings(stagePriorBindings(idx.graph, stages, view)) } idx.applyLap("prior_bindings") - carried, npmConsumers := restubIncomingRefsFromView(idx.graph, stages, view) + carried, npmConsumers, cargoConsumers, pythonConsumers := restubIncomingRefsFromView(idx.graph, stages, view) if markerBatch != nil { + markerBatch.pythonConsumerFiles = appendUniqueSorted(markerBatch.pythonConsumerFiles, pythonConsumers...) markerBatch.npmConsumerFiles = appendUniqueSorted(markerBatch.npmConsumerFiles, npmConsumers...) + markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, cargoConsumers...) + for _, stage := range stages { + if strings.HasSuffix(stage.graphPath, ".rs") && len(stage.priorNodes) == 0 { + markerBatch.cargoAddedFile = true + } + } } idx.applyLap("restub") // The capability state of the files, read before the eviction below @@ -1370,8 +1386,10 @@ func restubIncomingRefsFromView( g graph.Store, stages []*incrementalBatchStage, view incrementalPriorView, -) ([]*graph.Edge, []string) { +) ([]*graph.Edge, []string, []string, []string) { evicted := structuralPriorIDs(stages) + cargoConsumers := restubTrackedCargoImportsFromView(g, evicted, view.inByNode, false) + pythonConsumers := restubTrackedPythonImportsFromView(g, evicted, view.inByNode, false) reparsed := reparsedPaths(stages) npmConsumers := restubTrackedNpmImportsFromView(g, evicted, view.inByNode) var reindexes []graph.EdgeReindex @@ -1393,9 +1411,24 @@ func restubIncomingRefsFromView( // path) survives its own reparse. The eviction deletes the // row and no pass of this save re-derives it — the importer // is not reparsed — so re-state it. - if _, survives := frontier.survivingIDs[node.ID]; survives || frontier.conservative { + if _, survives := frontier.survivingIDs[node.ID]; !survives && (!frontier.conservative || stage.result == nil) { + for _, edge := range view.inByNode[node.ID] { + if edge == nil || edge.Kind != graph.EdgeImports || edge.To != node.ID { + continue + } + if _, sourceEvicted := evicted[edge.From]; sourceEvicted { + continue + } + if stub := goModuleRestubTarget(edge, ""); stub != "" { + oldTo := edge.To + graph.StashRestubProvenance(edge) + edge.To, edge.CrossRepo, edge.ConfidenceLabel = stub, false, "" + reindexes = append(reindexes, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) + } + } + } else { for _, edge := range view.inByNode[node.ID] { - if edge == nil || edge.Kind != graph.EdgeImports { + if edge == nil || edge.Kind != graph.EdgeImports || edge.To != node.ID { continue } if _, sourceEvicted := evicted[edge.From]; sourceEvicted { @@ -1530,6 +1563,13 @@ func restubIncomingRefsFromView( if edge.To != stub { edge.CrossRepo, edge.ConfidenceLabel = false, "" } + if original, _ := edge.Meta[resolver.CargoCallStubMeta].(string); edge.Kind == graph.EdgeCalls && edge.CrossRepo && original != "" { + edge.To, edge.CrossRepo, edge.ConfidenceLabel = original, false, "" + } + if original, _ := edge.Meta[resolver.PythonCallStubMeta].(string); edge.Kind == graph.EdgeCalls && edge.CrossRepo && original != "" { + edge.To, edge.CrossRepo, edge.ConfidenceLabel = original, false, "" + delete(edge.Meta, resolver.PythonReExportFileMeta) + } reindexes = append(reindexes, graph.EdgeReindex{Edge: edge, OldTo: oldTo}) } } @@ -1537,7 +1577,7 @@ func restubIncomingRefsFromView( if len(reindexes) > 0 { g.ReindexEdges(reindexes) } - return canonicalizeLegacyCarried(stages, evicted, carried), npmConsumers + return canonicalizeLegacyCarried(stages, evicted, carried), npmConsumers, cargoConsumers, pythonConsumers } // dataflowSourceSite is a reference's site: the file that records it and its @@ -1844,6 +1884,9 @@ func (idx *Indexer) recordFileReadVersionsBatched(receipts []fileReadReceipt) (f idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() for path, mtime := range mtimes { + if _, exists := idx.fileMtimes[path]; !exists { + idx.fileSetRevision.Add(1) + } idx.fileMtimes[path] = mtime } idx.mtimeMu.Unlock() @@ -2410,7 +2453,7 @@ func (idx *Indexer) evictFileIncrementalRaw(relPath string) forcedFileEviction { // recover this file's records from a sibling's canonical scalar payload. idx.ensureIncrementalContractRegistry() var invalidation DerivedInvalidationPlan - nodesRemoved, edgesRemoved := idx.evictDeletedFilesBatched([]string{relPath}, &invalidation) + nodesRemoved, edgesRemoved := idx.evictDeletedFilesBatched([]string{relPath}, &invalidation, batch) graphPath := idx.prefixPath(relPath) idx.removeIncrementalContractsForFile(graphPath, &invalidation) invalidation.Files = appendUniqueSorted(invalidation.Files, dependencyFiles...) @@ -2460,7 +2503,7 @@ func (idx *Indexer) removeIncrementalContractsForFile(graphPath string, plan *De plan.ContractSymbolIDs = appendUniqueSorted(plan.ContractSymbolIDs, refresh.SymbolIDs...) } -func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInvalidationPlan) (nodesRemoved, edgesRemoved int) { +func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInvalidationPlan, markerBatch *reparsePendingEnrichmentBatch) (nodesRemoved, edgesRemoved int) { if len(deleted) == 0 { return 0, 0 } @@ -2524,7 +2567,12 @@ func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInva // carry therefore means a deletion stage stopped being conservative, // which would destroy those edges with no compile error to show for // it — so the assumption is checked, not just documented. - if carried, _ := restubIncomingRefsFromView(idx.graph, stages, view); len(carried) > 0 { + carried, _, cargoConsumers, pythonConsumers := restubIncomingRefsFromView(idx.graph, stages, view) + if markerBatch != nil { + markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, cargoConsumers...) + markerBatch.pythonConsumerFiles = appendUniqueSorted(markerBatch.pythonConsumerFiles, pythonConsumers...) + } + if len(carried) > 0 { idx.logger.Error("indexer: deletion restub carried in-edges with no re-state path", zap.String("repo", idx.repoPrefix), zap.Int("carried", len(carried)), @@ -2546,6 +2594,9 @@ func (idx *Indexer) evictDeletedFilesBatched(deleted []string, plan *DerivedInva idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() for _, relPath := range deleted { + if _, exists := idx.fileMtimes[relPath]; exists { + idx.fileSetRevision.Add(1) + } delete(idx.fileMtimes, relPath) } idx.mtimeMu.Unlock() diff --git a/internal/indexer/incremental_restub_gate_test.go b/internal/indexer/incremental_restub_gate_test.go index 7f25e99ba..af42f0003 100644 --- a/internal/indexer/incremental_restub_gate_test.go +++ b/internal/indexer/incremental_restub_gate_test.go @@ -118,7 +118,7 @@ func runRestubStage( stages := []*incrementalBatchStage{stage} view := loadIncrementalPriorView(st, stages) - carried, _ := restubIncomingRefsFromView(st, stages, view) + carried, _, _, _ := restubIncomingRefsFromView(st, stages, view) evictFilesBatched(st, []string{restubDefPath}) st.AddBatch(fresh, append(append([]*graph.Edge{}, freshEdges...), carried...)) diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index 86f241bbf..d3a9cb8e1 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -537,13 +537,25 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( batch.dropDeferredPriorBindings() } crossRepoFiles := appendUniqueSorted(nil, files...) + if result != nil && batch != nil { + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.pythonAddedFileConsumers(batch.pythonAddedFiles)...) + } + cargoRetargetRepo := "" if batch != nil { crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.npmConsumerFiles...) + crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.cargoConsumerFiles...) + if batch.cargoAddedFile { + crossRepoFiles = appendUniqueSorted(crossRepoFiles, mi.cargoAddedFileConsumers(repoPrefix)...) + cargoRetargetRepo = repoPrefix + } + crossRepoFiles = appendUniqueSorted(crossRepoFiles, batch.pythonConsumerFiles...) } if result != nil { + crossRepoFiles = appendUniqueSorted(crossRepoFiles, result.CrossRepoResolveFiles...) crossRepoFiles = appendUniqueSorted(crossRepoFiles, result.DerivedInvalidation.Files...) } idx := mi.GetIndexer(repoPrefix) + var masterResolvedFiles []string if needed && len(files) > 0 && idx != nil { resolvedFiles := idx.runIncrementalResolutionCatchup(files, batch, func(frontier []string) { if idx.incrementalResolveFilesHook != nil { @@ -552,12 +564,14 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } mi.runMasterResolveFilesWithEvidence(frontier, false, idx.resolver) }) + masterResolvedFiles = files crossRepoFiles = appendUniqueSorted(crossRepoFiles, resolvedFiles...) if receipt != nil && receipt.Complete { mi.runMasterResolveNames(vanishedReceiptNames(mi.graph, receipt)) } } else if needed && len(files) > 0 { mi.runMasterResolveFiles(files, false) + masterResolvedFiles = files if receipt != nil && receipt.Complete { mi.runMasterResolveNames(vanishedReceiptNames(mi.graph, receipt)) } @@ -588,12 +602,12 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( } crossRepoFiles = appendUniqueSorted(crossRepoFiles, semanticFiles...) } - if needed || len(semanticFiles) > 0 || batch != nil && len(batch.npmConsumerFiles) > 0 { + if needed || len(semanticFiles) > 0 || result != nil && len(result.CrossRepoResolveFiles) > 0 || batch != nil && (len(batch.npmConsumerFiles) > 0 || len(batch.pythonConsumerFiles) > 0 || len(crossRepoFiles) > 0 && (batch.cargoAddedFile || len(batch.cargoConsumerFiles) > 0)) { if idx != nil { idx.observeIncrementalCatchup("cross_repo", crossRepoFiles) } if len(crossRepoFiles) > 0 { - mi.runCrossRepoResolveFiles(crossRepoFiles) + mi.runCrossRepoResolveMutationFrontiers(crossRepoFiles, crossRepoFiles, crossRepoFiles, masterResolvedFiles, cargoRetargetRepo) } else { mi.runCrossRepoResolve(false) } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 64c006ea5..ac9223f50 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -134,6 +134,8 @@ type IndexResult struct { // CrossRepoResolveFiles contains parked consumers outside this repository's // derived frontier. Batch callers must include them in cross-repo resolution. CrossRepoResolveFiles []string `json:"cross_repo_resolve_files,omitempty"` + // CargoRetarget records a Rust file addition for the post-registration warm tail. + CargoRetarget bool `json:"cargo_retarget,omitempty"` // mutationErr carries a point-operation compatibility error through the // resolver/semantic/derived tail. It is intentionally unexported and never @@ -487,6 +489,7 @@ type Indexer struct { parseErrors []IndexError fileMtimes map[string]int64 fileMtimesShared bool + fileSetRevision atomic.Uint64 // Any failed sidecar write can leave the durable keyset incomplete even // while the immutable in-memory snapshot is current. Receipt paging must // fall back to that snapshot until a full authoritative replace repairs it. @@ -3442,6 +3445,11 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index var err error shadowReplacementStarted = true restubTrackedNpmRepoIncoming(diskTarget, idx.RepoPrefix()) + parked := restubTrackedCargoRepoIncoming(diskTarget, idx.RepoPrefix()) + parked = appendUniqueSorted(parked, restubTrackedPythonRepoIncoming(diskTarget, idx.RepoPrefix())...) + if result != nil { + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, parked...) + } evictedNodes, evictedEdges, err = sqlite.EvictRepoForShadowReplacement(ctx, idx.RepoPrefix()) if err != nil { retErr = fmt.Errorf("indexer: retire shadow replacement rows: %w", err) @@ -5150,8 +5158,16 @@ func (idx *Indexer) indexFile( priorNodes, legacyIDs, legacyPath := idx.filePriorWithLegacy(graphPath, idx.graph.GetFileNodes(graphPath)) var legacyCarried []*graph.Edge var oldFuncIDs []string + var carriedGoImports []*graph.Edge evictExisting := func(result *parser.ExtractionResult) { idx.invalidateGoModuleSourceStamp(relPath) + if strings.HasSuffix(graphPath, ".go") { + for _, e := range idx.graph.GetInEdges(graphPath) { + if e.Kind == graph.EdgeImports && !e.CrossRepo && e.Meta["go_import"] != nil && e.FilePath != graphPath { + carriedGoImports = append(carriedGoImports, e) + } + } + } var oldFTSNodeIDs []string for _, n := range priorNodes { if n == nil { @@ -5169,9 +5185,9 @@ func (idx *Indexer) indexFile( // batch replacement by deleting its prior documents explicitly; // canonical contracts follow the store's owner-retention decision. idx.deleteSymbolFTS(oldFTSNodeIDs) - var npmConsumers []string + var npmConsumers, cargoConsumers, pythonConsumers []string if len(legacyIDs) == 0 { - npmConsumers = idx.restubIncomingRefs(graphPath) + npmConsumers, cargoConsumers, pythonConsumers = idx.restubIncomingRefs(graphPath) } else { // Extraction is still unprefixed here. Build only the identity surface // needed by the same incoming frontier used by the structural batch. @@ -5186,10 +5202,12 @@ func (idx *Indexer) indexFile( } stage := &incrementalBatchStage{graphPath: graphPath, legacyPath: legacyPath, priorNodes: priorNodes, result: fresh} stages := []*incrementalBatchStage{stage} - legacyCarried, npmConsumers = restubIncomingRefsFromView(idx.graph, stages, loadIncrementalPriorView(idx.graph, stages)) + legacyCarried, npmConsumers, cargoConsumers, pythonConsumers = restubIncomingRefsFromView(idx.graph, stages, loadIncrementalPriorView(idx.graph, stages)) } if markerBatch != nil { markerBatch.npmConsumerFiles = appendUniqueSorted(markerBatch.npmConsumerFiles, npmConsumers...) + markerBatch.cargoConsumerFiles = appendUniqueSorted(markerBatch.cargoConsumerFiles, cargoConsumers...) + markerBatch.pythonConsumerFiles = appendUniqueSorted(markerBatch.pythonConsumerFiles, pythonConsumers...) } if len(legacyIDs) == 0 { idx.graph.EvictFile(graphPath) @@ -5198,6 +5216,15 @@ func (idx *Indexer) indexFile( } } + parkCarriedGoImports := func() []*graph.Edge { + for _, e := range carriedGoImports { + graph.StashRestubProvenance(e) + e.To = goModuleRestubTarget(e, "") + e.CrossRepo, e.ConfidenceLabel = false, "" + } + return carriedGoImports + } + // A delta probe already owns a shared admission lease together with its // transformed source and parse tree. Validate and consume that snapshot // before acquiring another lease; acquiring first can self-deadlock when @@ -5262,7 +5289,7 @@ func (idx *Indexer) indexFile( } } evictExisting(result) - idx.graph.AddBatch([]*graph.Node{n}, nil) + idx.graph.AddBatch([]*graph.Node{n}, parkCarriedGoImports()) if !idx.recordFileReadVersion(mtimeKey, absPath, readVersion) { return idx.fileIndexFailureError(absPath) } @@ -5286,7 +5313,7 @@ func (idx *Indexer) indexFile( } } evictExisting(result) - idx.graph.AddBatch([]*graph.Node{n}, nil) + idx.graph.AddBatch([]*graph.Node{n}, parkCarriedGoImports()) if !idx.recordFileReadVersion(mtimeKey, absPath, readVersion) { return idx.fileIndexFailureError(absPath) } @@ -5439,7 +5466,10 @@ func (idx *Indexer) indexFile( // the node. idx.replaceContentSections(graphPath, result.Nodes, false) - idx.graph.AddBatch(result.Nodes, append(result.Edges, legacyCarried...)) + if _, survives := newNodeIDs[graphPath]; !survives { + carriedGoImports = nil + } + idx.graph.AddBatch(result.Nodes, append(append(result.Edges, legacyCarried...), carriedGoImports...)) idx.persistFileMeta(relPath, src, result) idx.persistConstValues(result) // No subsequent stage reads source bytes or the parse tree. Release both @@ -5693,6 +5723,9 @@ func (idx *Indexer) recordFileReadVersion(relPath, absPath string, version fileR func (idx *Indexer) recordFileMtimeValue(relPath string, mtime int64) { idx.mtimeMu.Lock() idx.ensureFileMtimesWritableLocked() + if _, exists := idx.fileMtimes[relPath]; !exists { + idx.fileSetRevision.Add(1) + } idx.fileMtimes[relPath] = mtime idx.mtimeMu.Unlock() if w, ok := idx.graph.(graph.FileMtimeWriter); ok { @@ -5952,10 +5985,10 @@ func (idx *Indexer) reresolveFileScopedRaw(filePath string) error { // agnostic: GetInEdges + ReindexEdges are the same Store primitives the // resolver uses, so this behaves identically on the in-memory and disk // stores. -func (idx *Indexer) restubIncomingRefs(graphPath string) []string { +func (idx *Indexer) restubIncomingRefs(graphPath string) ([]string, []string, []string) { nodes := idx.graph.GetFileNodes(graphPath) if len(nodes) == 0 { - return nil + return nil, nil, nil } evicted := make(map[string]struct{}, len(nodes)) refIDs := make([]string, 0, len(nodes)) @@ -5969,26 +6002,34 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) []string { } } if len(refIDs) == 0 { - return nil + return nil, nil, nil } // Fetch the whole changed file's incoming adjacency in one bounded store // operation. On SQLite this is chunked under the parameter limit; more // importantly, it avoids one transaction/query per definition in the file. inEdges := idx.graph.GetInEdgesByNodeIDs(refIDs) npmConsumers := restubTrackedNpmImportsFromView(idx.graph, evicted, inEdges) + cargoConsumers := restubTrackedCargoImportsFromView(idx.graph, evicted, inEdges, false) + pythonConsumers := restubTrackedPythonImportsFromView(idx.graph, evicted, inEdges, false) var batch []graph.EdgeReindex for _, n := range nodes { - if n == nil || n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) { + if n == nil || n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) && n.Kind != graph.KindFile { continue } stub := graph.UnresolvedMarker + n.Name for _, e := range inEdges[n.ID] { - if e == nil || !graph.IsResolvableRefEdge(e.Kind) { + if n.Kind == graph.KindFile && (e == nil || e.Kind != graph.EdgeImports || e.Meta["go_import"] == nil) { + continue + } + if e == nil || !graph.IsResolvableRefEdge(e.Kind) && (e.Kind != graph.EdgeImports || e.Meta["go_import"] == nil) { continue } if _, fromEvicted := evicted[e.From]; fromEvicted { continue // intra-file edge: the source is evicted too } + if e.Kind == graph.EdgeImports && !e.CrossRepo && e.Meta["go_import"] != nil { + continue // The single-file save carries surviving local file imports. + } if graph.IsUnresolvedTarget(e.To) { continue // already a pending stub } @@ -6000,13 +6041,23 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) []string { // moved target leaves the stub honestly unresolved. graph.StashRestubProvenance(e) e.To = goModuleRestubTarget(e, stub) + if e.Kind == graph.EdgeImports { + e.CrossRepo, e.ConfidenceLabel = false, "" + } + if original, _ := e.Meta[resolver.CargoCallStubMeta].(string); e.Kind == graph.EdgeCalls && e.CrossRepo && original != "" { + e.To, e.CrossRepo, e.ConfidenceLabel = original, false, "" + } + if original, _ := e.Meta[resolver.PythonCallStubMeta].(string); e.Kind == graph.EdgeCalls && e.CrossRepo && original != "" { + e.To, e.CrossRepo, e.ConfidenceLabel = original, false, "" + delete(e.Meta, resolver.PythonReExportFileMeta) + } batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) } } if len(batch) > 0 { idx.graph.ReindexEdges(batch) } - return npmConsumers + return npmConsumers, cargoConsumers, pythonConsumers } // embeddingDimsOrDefault returns the embedder's reported vector width, @@ -6963,6 +7014,7 @@ func (idx *Indexer) pruneDeletedFileMtimes(deleted []string) { func (idx *Indexer) SetFileMtimes(mtimes map[string]int64) { idx.mtimeMu.Lock() defer idx.mtimeMu.Unlock() + idx.fileSetRevision.Add(1) idx.fileMtimes = make(map[string]int64, len(mtimes)) idx.fileMtimesShared = false for k, v := range mtimes { @@ -7534,6 +7586,9 @@ func (idx *Indexer) incrementalReindexPathsMode( capabilityPrior: idx.takeCapabilityPrior(), } result.CrossRepoResolveFiles = appendUniqueSorted(nil, markerBatch.npmConsumerFiles...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, markerBatch.cargoConsumerFiles...) + result.CargoRetarget = markerBatch.cargoAddedFile + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, markerBatch.pythonConsumerFiles...) for _, path := range failedFiles { result.Errors = append(result.Errors, IndexError{FilePath: path, Error: idx.fileIndexFailureError(path).Error()}) } diff --git a/internal/indexer/legacy_file_identity_test.go b/internal/indexer/legacy_file_identity_test.go index a5f494302..0389142b1 100644 --- a/internal/indexer/legacy_file_identity_test.go +++ b/internal/indexer/legacy_file_identity_test.go @@ -230,7 +230,7 @@ func TestLegacyIdentityRepairRebindsForeignCaller(t *testing.T) { prior, ids, native := mergeLegacyFilePrior(legacy, fresh, old) stage := &incrementalBatchStage{graphPath: path, legacyPath: native, legacyIDs: ids, priorNodes: prior, result: &parser.ExtractionResult{Nodes: fresh}} stages := []*incrementalBatchStage{stage} - carried, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) + carried, _, _, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) idx := New(store, parser.NewRegistry(), config.Default().Index, zap.NewNop()) idx.SetRepoPrefix("repo") idx.evictFileIdentities([]string{path}, ids, []string{native}) @@ -308,7 +308,7 @@ func TestLegacyIdentityRepairRespectsDataflowFrontier(t *testing.T) { Nodes: fresh, Edges: []*graph.Edge{{From: fresh[2].ID, To: fresh[1].ID, Kind: graph.EdgeParamOf, FilePath: path}}, }} stages := []*incrementalBatchStage{stage} - carried, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) + carried, _, _, _ := restubIncomingRefsFromView(store, stages, loadIncrementalPriorView(store, stages)) store.(graph.FileIdentityBatchEvicter).EvictFilesWithNodeIDs([]string{path}, ids) store.AddBatch(fresh, append(stage.result.Edges, carried...)) find := func(from, to string, kind graph.EdgeKind) bool { diff --git a/internal/indexer/module_work_count_test.go b/internal/indexer/module_work_count_test.go index 7a2ef74e6..0176b157a 100644 --- a/internal/indexer/module_work_count_test.go +++ b/internal/indexer/module_work_count_test.go @@ -128,9 +128,15 @@ func TestNonGoUntrackSkipsModuleSweep(t *testing.T) { } require.NoError(t, mi.SeedGoModuleDeps(t.Context())) g.scans = 0 + mi.goModuleMu.Lock() + mi.goModuleChecked = time.Time{} // any module snapshot read would refresh this stamp + mi.goModuleMu.Unlock() _, _, err = mi.UntrackRepoChecked(t.Context(), "a") require.NoError(t, err) assert.Zero(t, g.scans, "JS-only untrack must not rescan Go terminals") + mi.goModuleMu.Lock() + defer mi.goModuleMu.Unlock() + assert.True(t, mi.goModuleChecked.IsZero(), "untrack with nothing parked must not build a cross-repo resolver") } type goModuleUntrackScanStore struct{ *goModuleRetryScanStore } diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index f2507de14..82825c802 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -35,6 +35,7 @@ import ( // RepoMetadata holds per-repo indexing state. type RepoMetadata struct { + cargoRetarget *IndexResult // warm receipt, guarded by MultiIndexer.mu RepoPrefix string RootPath string Identity *RepoIdentity @@ -62,6 +63,8 @@ type RepoMetadata struct { // retries from repeating successful payload/vector phases and keeps the stable // mutation lane closed until every external side effect commits. type repositoryUntrackState struct { + bridgeConsumerFiles []string + retainAdmission bool mu sync.Mutex metadata *RepoMetadata @@ -119,6 +122,9 @@ type MultiIndexer struct { goModuleRetry resolver.GoModuleDepRetry npmPackageMu sync.Mutex npmPackageCache *npmPackageSnapshot + bridgeManifestMu sync.Mutex + pythonPackageCache *bridgeManifestSnapshot + cargoPackageCache *bridgeManifestSnapshot // pendingRepositoryUntracks is guarded by mu. Entries are installed before // a live repo is hidden and removed only after payload, vector, config, and @@ -957,8 +963,10 @@ func (mi *MultiIndexer) resolveDeferredMutations(receipt *graph.MutationReceipt, return deferredResolveSkipped, true } + var masterResolvedFiles []string if receipt.ResolutionRelevant { mi.runMasterResolveFiles(resolutionFiles, false) + masterResolvedFiles = resolutionFiles // Evicted definitions' pending references live outside the file // frontier (their name is no longer declared in any frontier // file); rebind them by the names the receipt recorded that no @@ -968,7 +976,7 @@ func (mi *MultiIndexer) resolveDeferredMutations(receipt *graph.MutationReceipt, // Resolve only files that can create or bind unresolved edges. Resolved // edge sources still materialise their cross_repo_* generation without // repeating the substantially larger name-resolution frontier. - mi.runCrossRepoResolveMutationFrontiers(resolutionFiles, receipt.ChangedFiles, receipt.DefinitionFiles) + mi.runCrossRepoResolveMutationFrontiers(resolutionFiles, receipt.ChangedFiles, receipt.DefinitionFiles, masterResolvedFiles) return deferredResolveExact, true } if !fallbackNeeded { @@ -991,11 +999,9 @@ func (mi *MultiIndexer) newMasterResolver(useLSP bool) *resolver.Resolver { } master := resolver.New(mi.graph) master.SetLogger(mi.logger) - // The master resolve is the only pass with whole-graph evidence, so it is - // the one allowed to durably flag terminally-unresolved edges (permanently - // external / stdlib / definition-less) that a later scoped warm resolve can - // skip. A scoped/file pass leaves the flag alone; only a full ResolveAll - // stamps and self-heals terminal state. + // Full resolution owns terminal stamps. Completed tracked-package catch-up + // may finish marked callers using the same global name-count proof through + // ReconcileBridgeCallerTerminals; ordinary file passes only carry stamps. master.SetStampTerminal(true) // The pre-enrichment queryability pass uses resolver-time LSP precision. // The post-enrichment catch-up disables it because semantic providers just @@ -1204,7 +1210,7 @@ func (mi *MultiIndexer) RunPreEnrichResolve(ctx context.Context, scope map[strin // Cross-repo references resolve here so a multi-repo workspace is fully // resolved, not just within each repo. Whole-graph so inbound references // from unchanged repos into the changed repos bind too. - return mi.runCrossRepoResolveContext(ctx, false) + return mi.runWarmCargoResolve(ctx, nil, true) } // RunPreEnrichResolveFiles preserves an exact warm-reconcile frontier through @@ -1234,10 +1240,9 @@ func (mi *MultiIndexer) RunPreEnrichResolveFiles(ctx context.Context, files []st return err } if resolutionAffected > 0 { - return mi.runCrossRepoResolveContext(ctx, false) + return mi.runWarmCargoResolve(ctx, nil, true) } - mi.runCrossRepoResolveFiles(files) - return context.Cause(ctx) + return mi.runWarmCargoResolve(ctx, files, false) } func (mi *MultiIndexer) runDeferredGoModAll() { @@ -2789,6 +2794,9 @@ func (mi *MultiIndexer) indexRepoRaw(repoPrefix string) (*IndexResult, error) { if err := mi.postPublishGoModules(context.Background(), idx, false); err != nil { mi.logger.Warn("post-publish Go module follow-up", zap.Error(err)) } + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(repoPrefix, false)...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.pythonConsumerFilesForRepo(repoPrefix)...) + mi.runCrossRepoResolveFiles(result.CrossRepoResolveFiles) mi.ReconcileContractEdges() @@ -3434,6 +3442,10 @@ func (mi *MultiIndexer) trackPreparedRepoSourceCtx( // repository's first registration, never a rewrite of an entry. mi.markRepoRegistered(prefix, absPath) mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix)) + if !batchMode.deferResolve { + mi.runCrossRepoResolveFiles(mi.cargoConsumerFilesForRepo(prefix, false)) + mi.runCrossRepoResolveFiles(mi.pythonConsumerFilesForRepo(prefix)) + } // Add to global config. entry.Path = absPath @@ -3699,6 +3711,9 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE IsWorktree: isWorktree, } mi.indexers[prefix] = idx + if result.CargoRetarget { + mi.repos[prefix].cargoRetarget = result + } mi.mu.Unlock() installed = true mi.markRepoRegistered(prefix, absPath) @@ -3732,6 +3747,9 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE // Parallel warmup deliberately defers both tails to its later workspace // resolve/global phases. A full retrack already ran its own resolve/derived // pipeline and only needs the existing cross-repository contract reconcile. + if batchMode.deferResolve && batch != nil { + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.pythonAddedFileConsumers(batch.pythonAddedFiles)...) + } if !result.FullRetrack && !batchMode.deferResolve { mi.resolveIncrementalRepoMutation(prefix, result, receipt, batch) if !batchMode.deferGlobalPasses { @@ -3741,8 +3759,11 @@ func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoE }) } } else if result.FullRetrack { + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.cargoConsumerFilesForRepo(prefix, false)...) + result.CrossRepoResolveFiles = appendUniqueSorted(result.CrossRepoResolveFiles, mi.pythonConsumerFilesForRepo(prefix)...) if !batchMode.deferResolve { mi.runCrossRepoResolveFiles(mi.npmConsumerFilesForRepo(prefix)) + mi.runCrossRepoResolveFiles(result.CrossRepoResolveFiles) } if !batchMode.deferGlobalPasses { if err := mi.postPublishGoModules(ctx, idx, false); err != nil { diff --git a/internal/indexer/npm_lifecycle_test.go b/internal/indexer/npm_lifecycle_test.go index 4dc83a4f6..c89b4e365 100644 --- a/internal/indexer/npm_lifecycle_test.go +++ b/internal/indexer/npm_lifecycle_test.go @@ -38,18 +38,19 @@ func TestTrackedNpmImportsSurviveEviction(t *testing.T) { switch mode { case "file": idx := New(g, newTestRegistry(), config.IndexConfig{}, zap.NewNop()) - assert.Equal(t, []string{"app/a.ts"}, idx.restubIncomingRefs("lib/index.ts")) + consumers, _, _ := idx.restubIncomingRefs("lib/index.ts") + assert.Equal(t, []string{"app/a.ts"}, consumers) g.EvictFile("lib/index.ts") case "batch": stages := []*incrementalBatchStage{{graphPath: "lib/index.ts", priorNodes: nodes}} - _, consumers := restubIncomingRefsFromView(g, stages, loadIncrementalPriorView(g, stages)) + _, consumers, _, _ := restubIncomingRefsFromView(g, stages, loadIncrementalPriorView(g, stages)) assert.Equal(t, []string{"app/a.ts"}, consumers) g.EvictFile("lib/index.ts") case "replace": evictRepoCurrentGeneration(g, "lib") case "untrack": mi := &MultiIndexer{graph: g} - _, _, err := mi.purgeRepositoryPayload("lib", nil) + _, _, err := mi.purgeRepositoryPayload("lib", nil, nil) require.NoError(t, err) } var targets []string diff --git a/internal/indexer/python_added_file_warm_gate_test.go b/internal/indexer/python_added_file_warm_gate_test.go new file mode 100644 index 000000000..b0e9e291d --- /dev/null +++ b/internal/indexer/python_added_file_warm_gate_test.go @@ -0,0 +1,60 @@ +package indexer + +import ( + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "testing" +) + +func TestPythonUnregisteredConfigDoesNotKeepWarming(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &pythonLookupScanStore{Store: cargoWarmStore(t, backend)} + g.AddNode(&graph.Node{ID: "provider/pkg/__init__.py", FilePath: "provider/pkg/__init__.py", RepoPrefix: "provider", Kind: graph.KindFile}) + cm := newTestConfigManager(t) + cm.Global().Repos = []config.RepoEntry{{Name: "provider", Path: t.TempDir()}, {Name: "missing", Path: t.TempDir() + "/absent"}, {Name: "failed", Path: t.TempDir()}} + mi := &MultiIndexer{graph: g, configMgr: cm, repos: map[string]*RepoMetadata{"provider": {RootPath: cm.Global().Repos[0].Path}}} + for range 2 { + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + } + require.Zero(t, g.importScans, "unregistered config is not a live warmup") + for _, parallel := range []bool{false, true} { + if parallel { + mi.BeginParallelBatch() + } else { + mi.BeginBatch() + } + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + require.Zero(t, g.importScans, "a one-repo graph has no pending foreign consumers") + mi.ResetBatch() + } + }) + } +} + +func TestPythonAddedFileWarmGateRequiresPendingGraphRepo(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, parallel := range []bool{false, true} { + t.Run(backend+map[bool]string{false: "/serial", true: "/parallel"}[parallel], func(t *testing.T) { + g := &pythonLookupScanStore{Store: cargoWarmStore(t, backend)} + for _, file := range []string{"provider/pkg/__init__.py", "consumer/app.py"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, RepoPrefix: graph.RepoPrefixOfID(file), Kind: graph.KindFile}) + } + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: "external::pkg.extra", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: t.TempDir()}}} + if parallel { + mi.BeginParallelBatch() + } else { + mi.BeginBatch() + } + require.Equal(t, []string{"consumer/app.py"}, mi.pythonAddedFileConsumers([]string{"provider/pkg/extra.py"})) + require.Equal(t, 1, g.importScans) + mi.ResetBatch() + g.importScans = 0 + require.Empty(t, mi.pythonAddedFileConsumers([]string{"provider/pkg/next.py"})) + require.Zero(t, g.importScans, "a completed warmup must not leave admission open") + }) + } + } +} diff --git a/internal/indexer/python_bridge_identity_test.go b/internal/indexer/python_bridge_identity_test.go new file mode 100644 index 000000000..df0d7396b --- /dev/null +++ b/internal/indexer/python_bridge_identity_test.go @@ -0,0 +1,98 @@ +package indexer + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func newPythonBridgeIdentityIndexer(t *testing.T, backend string, entries []config.RepoEntry) (*MultiIndexer, graph.Store) { + t.Helper() + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + cm := newTestConfigManager(t) + cm.Global().Repos = entries + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + return mi, g +} + +func TestTrackedPythonSameLineImportIdentities(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'dist'\n") + writeWorkspaceFile(t, provider, "pkg/__init__.py", "") + writeWorkspaceFile(t, provider, "pkg/a.py", "def f(): pass\n") + writeWorkspaceFile(t, provider, "pkg/b.py", "def g(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['dist']\n") + source := "import pkg.a, pkg.b\ndef run():\n pkg.b.g()\n" + writeWorkspaceFile(t, consumer, "app.py", source) + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + _, err := mi.IndexAll() + require.NoError(t, err) + check := func() { + require.ElementsMatch(t, []string{"provider/pkg/a.py", "provider/pkg/b.py"}, revisionCallTargets(g, "consumer/app.py", graph.EdgeImports)) + require.ElementsMatch(t, []string{"provider/pkg/b.py::g"}, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + } + // Given two dotted imports on one line, cold indexing keeps both. + check() + // No-change and edited-file saves must link those same two files. + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "app.py"))) + check() + writeWorkspaceFile(t, consumer, "app.py", source+"# comment\n") + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "app.py"))) + check() + cold, coldGraph := newPythonBridgeIdentityIndexer(t, backend, entries) + _, err = cold.IndexAll() + require.NoError(t, err) + require.ElementsMatch(t, revisionCallTargets(coldGraph, "consumer/app.py", graph.EdgeImports), revisionCallTargets(g, "consumer/app.py", graph.EdgeImports)) + }) + } +} + +func TestTrackedPythonSameLineCallsAfterConsumerFirst(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'dist'\n") + writeWorkspaceFile(t, provider, "pkg/__init__.py", "") + writeWorkspaceFile(t, provider, "pkg/image.py", "def f(): pass\ndef g(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['dist']\n") + writeWorkspaceFile(t, consumer, "app.py", "import pkg.image as image\ndef run():\n image.f(); image.g()\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, nil) + _, err := mi.TrackRepoCtx(context.Background(), entries[0]) + require.NoError(t, err) + require.True(t, mi.GetIndexer("consumer").externalCallSynthesisEnabled(), "exercise default synthesis policy") + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + require.NoError(t, err) + want := []string{"provider/pkg/image.py::f", "provider/pkg/image.py::g"} + require.ElementsMatch(t, want, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "pkg/image.py")}) + require.NoError(t, err) + require.ElementsMatch(t, want, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + cold, cg := newPythonBridgeIdentityIndexer(t, backend, entries) + _, err = cold.IndexAll() + require.NoError(t, err) + require.ElementsMatch(t, revisionCallTargets(cg, "consumer/app.py::run", graph.EdgeCalls), revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + }) + } +} diff --git a/internal/indexer/python_import_fallback_test.go b/internal/indexer/python_import_fallback_test.go new file mode 100644 index 000000000..3e5c650ee --- /dev/null +++ b/internal/indexer/python_import_fallback_test.go @@ -0,0 +1,76 @@ +package indexer + +import ( + "context" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +// pythonFallbackRefreshTransform changes extra.py's bytes on exactly its second +// application: the probe read sees call 1, takePreparedRefresh sees call 2 +// (mismatch, so the stage falls back), and the fallback indexFile sees call 3. +type pythonFallbackRefreshTransform struct{ calls *atomic.Int32 } + +func (pythonFallbackRefreshTransform) name() string { return "python-fallback-refresh" } +func (pythonFallbackRefreshTransform) matches(p string) bool { return strings.HasSuffix(p, "extra.py") } +func (pythonFallbackRefreshTransform) asLanguage() string { return "" } +func (f pythonFallbackRefreshTransform) apply(p string, src []byte) ([]byte, error) { + if f.calls.Add(1) == 2 { + return append(append([]byte(nil), src...), []byte("\n# concurrent write\n")...), nil + } + return src, nil +} + +func TestTrackedPythonNewModuleThroughFallback(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider) + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + writeWorkspaceFile(t, consumer, "app.py", pythonLifecycleConsumer+"\nfrom PIL.extra import later\ndef after(): return later()\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + g := pythonLocalStore(t, backend) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + _, err = mi.IndexAll() + require.NoError(t, err) + + idx := mi.GetIndexer("provider") + require.NotNil(t, idx) + var calls atomic.Int32 + idx.transforms.transforms = append(idx.transforms.transforms, pythonFallbackRefreshTransform{calls: &calls}) + writeWorkspaceFile(t, provider, "src/PIL/extra.py", "def later(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/extra.py")}) + require.NoError(t, err) + require.GreaterOrEqual(t, calls.Load(), int32(3), "fallback path must run") + require.NotNil(t, g.GetNode("provider/src/PIL/extra.py")) + + cold := pythonLocalStore(t, backend) + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, pythonLifecycleBindings(cold, ""), pythonLifecycleBindings(g, "")) + }) + } +} diff --git a/internal/indexer/python_import_lifecycle_test.go b/internal/indexer/python_import_lifecycle_test.go new file mode 100644 index 000000000..40cc49c3e --- /dev/null +++ b/internal/indexer/python_import_lifecycle_test.go @@ -0,0 +1,376 @@ +package indexer + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedPythonImportLifecycle(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, operation := range []string{"consumer_save", "provider_save", "provider_save_util", "provider_save_lib_contract", "provider_delete_util_batched", "forced_evict_util", "delete_restore_util", "consumer_save_then_restore", "provider_gains_new_module", "provider_gains_empty_module", "consumer_full", "provider_full", "warm_full_retrack", "untrack_retrack", "rename_and_back", "untrack_only", "no_change", "track", "deferred_track"} { + if operation == "deferred_track" && providerFirst { + continue // This case starts with an already tracked consumer. + } + order := "consumer_first" + if providerFirst { + order = "provider_first" + } + t.Run(backend+"/"+order+"/"+operation, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider) + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + consumerSource := pythonLifecycleConsumer + switch operation { + case "provider_gains_new_module": + consumerSource += "\nfrom PIL.extra import later\ndef after(): return later()\n" + case "provider_gains_empty_module": + consumerSource += "\nimport PIL.extra\n" + } + writeWorkspaceFile(t, consumer, "app.py", consumerSource) + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + if providerFirst { + gc.Repos = []config.RepoEntry{entries[1], entries[0]} + } + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + var g graph.Store = graph.New() + if backend == "sqlite" { + disk, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, disk.Close()) }) + g = disk + } + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + core, logs := observer.New(zap.InfoLevel) + logger := zap.New(core) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + switch operation { + case "deferred_track": + _, err = mi.TrackRepoCtx(context.Background(), entries[0]) + require.NoError(t, err) + mi.BeginParallelBatch() + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + require.NoError(t, err) + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports { + require.False(t, edge.CrossRepo, "deferred track must not resolve shared imports early") + } + } + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/app.py", "provider/src/PIL/image.py", "provider/src/PIL/util.py"}, nil)) + mi.EndBatch() + case "track": + for _, entry := range gc.Repos { + _, err = mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + default: + _, err = mi.IndexAll() + require.NoError(t, err) + } + assertImport := func() { + t.Helper() + var imports []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports && edge.Line <= 4 { + imports = append(imports, edge) + } + } + require.Len(t, imports, 4) + for _, edge := range imports { + require.True(t, edge.CrossRepo, "%#v", edge) + } + calls := make(map[string]int) + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls { + require.True(t, edge.CrossRepo, "%#v", edge) + require.NotEmpty(t, edge.Meta["python_call_stub"], "%#v", edge) + require.NotContains(t, edge.Meta, "external_call") + calls[edge.To]++ + } + } + require.Equal(t, map[string]int{ + "provider/src/PIL/image.py::Image": 1, + "provider/src/PIL/image.py::Image.show": 1, + "provider/src/PIL/image.py::f": 2, + "provider/src/PIL/util.py::helper": 1, + }, calls) + } + assertImport() + initial := pythonLifecycleBindings(g, "") + initialMarkers := map[string]string{} + for _, node := range g.GetFileNodes("consumer/app.py") { + for _, edge := range g.GetOutEdges(node.ID) { + key := "python_call_stub" + if edge.Kind == graph.EdgeImports { + key = "python_import_stub" + } + if marker, ok := edge.Meta[key].(string); ok { + initialMarkers[fmt.Sprint(edge.Kind, edge.Line)] = marker + } + } + } + logs.TakeAll() + var scans *pythonLookupScanStore + if operation == "provider_save" { + scans = &pythonLookupScanStore{Store: g} + mi.graph = scans + } + switch operation { + case "consumer_save": + writeWorkspaceFile(t, consumer, "app.py", pythonLifecycleConsumer+"def edited(): pass\n") + _, err = mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "app.py")}) + case "provider_save": + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider+"def added(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/image.py")}) + case "provider_save_util": + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\ndef added(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + case "provider_save_lib_contract": + writeWorkspaceFile(t, provider, "src/PIL/image.py", strings.Replace(pythonLifecycleProvider, "def f(): pass", "def f(x): pass", 1)) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/image.py")}) + case "provider_delete_util_batched": + require.NoError(t, os.Remove(filepath.Join(provider, "src/PIL/util.py"))) + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider+"def added(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py"), filepath.Join(provider, "src/PIL/image.py")}) + case "provider_gains_empty_module": + writeWorkspaceFile(t, provider, "src/PIL/extra.py", "") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/extra.py")}) + case "provider_gains_new_module": + writeWorkspaceFile(t, provider, "src/PIL/extra.py", "def later(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/extra.py")}) + case "delete_restore_util", "consumer_save_then_restore": + if operation == "consumer_save_then_restore" { + writeWorkspaceFile(t, consumer, "app.py", pythonLifecycleConsumer+"def edited(): pass\n") + _, err = mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "app.py")}) + require.NoError(t, err) + assertImport() + } + require.NoError(t, os.Remove(filepath.Join(provider, "src/PIL/util.py"))) + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + require.NoError(t, err) + var parked []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 10 { + parked = append(parked, edge) + } + } + require.Len(t, parked, 1) + require.Equal(t, "external-call::dep::PIL.util.helper", parked[0].To) + require.Equal(t, "unresolved::extern::PIL.util.helper::helper", parked[0].Meta["python_call_stub"]) + require.False(t, parked[0].CrossRepo) + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + case "forced_evict_util": + require.NoError(t, os.Remove(filepath.Join(provider, "src/PIL/util.py"))) + mi.GetIndexer("provider").EvictFile(filepath.Join(provider, "src/PIL/util.py")) + case "consumer_full": + _, err = mi.IndexRepo("consumer") + case "provider_full": + var result *IndexResult + result, err = mi.IndexRepo("provider") + require.NoError(t, err) + data, marshalErr := json.Marshal(result) + require.NoError(t, marshalErr) + var output struct { + Files []string `json:"cross_repo_resolve_files"` + } + require.NoError(t, json.Unmarshal(data, &output)) + require.Contains(t, output.Files, "consumer/app.py") + case "warm_full_retrack": + prior := map[string]map[string]int64{"consumer": mi.FileMtimes("consumer"), "provider": mi.FileMtimes("provider")} + restarted := NewMultiIndexer(g, reg, search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, restarted.Close(context.Background())) }) + restarted.BeginBatch() + var output struct { + Files []string `json:"cross_repo_resolve_files"` + } + for _, entry := range gc.Repos { + force := "0" + if entry.Name == "provider" { + force = "1" + } + t.Setenv("GORTEX_WARMUP_FULL_RETRACK", force) + result, restoreErr := restarted.ReconcileRepoCtx(context.Background(), entry, prior[entry.Name]) + require.NoError(t, restoreErr) + if entry.Name == "provider" { + require.True(t, result.FullRetrack) + } + data, marshalErr := json.Marshal(result) + require.NoError(t, marshalErr) + var current struct { + Files []string `json:"cross_repo_resolve_files"` + } + require.NoError(t, json.Unmarshal(data, ¤t)) + output.Files = appendUniqueSorted(output.Files, current.Files...) + if entry.Name == "provider" { + require.NotContains(t, result.DerivedInvalidation.Files, "consumer/app.py") + } + } + if backend == "sqlite" { + require.Contains(t, output.Files, "consumer/app.py") + } else { + require.Empty(t, output.Files, "memory replacement preserved the existing import") + } + err = restarted.RunPreEnrichResolveFiles(context.Background(), output.Files, nil) + restarted.EndBatch() + case "no_change": + result, noChangeErr := mi.IncrementalReindexRepo("provider", nil) + require.NoError(t, noChangeErr) + require.Zero(t, result.StaleFileCount) + case "rename_and_back": + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper_renamed(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + require.NoError(t, err) + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 10 { + require.False(t, edge.CrossRepo) + require.Equal(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)], edge.Meta["python_call_stub"]) + } + } + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/PIL/util.py")}) + case "untrack_only": + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") + case "untrack_retrack": + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") + require.NoError(t, err) + _, err = mi.TrackRepoCtx(context.Background(), entries[1]) + } + require.NoError(t, err) + if operation == "rename_and_back" { + require.Equal(t, initial, pythonLifecycleBindings(g, "")) + } + if operation == "untrack_only" { + for _, node := range g.GetFileNodes("consumer/app.py") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind != graph.EdgeImports && edge.Kind != graph.EdgeCalls { + continue + } + require.False(t, edge.CrossRepo) + require.False(t, strings.HasPrefix(edge.To, "provider/")) + key := "python_call_stub" + if edge.Kind == graph.EdgeImports { + key = "python_import_stub" + } + require.NotEmpty(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)]) + require.Equal(t, initialMarkers[fmt.Sprint(edge.Kind, edge.Line)], edge.Meta[key]) + } + } + } + if scans != nil { + // The existing cross-repo pass scans imports once for reachability. + require.Equal(t, 1, scans.importScans, "ordinary saves must not add a Python consumer-frontier scan") + } + missingUtil := operation == "provider_delete_util_batched" || operation == "forced_evict_util" + if missingUtil { + var missingImport, missingCall []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports && edge.Line == 4 { + missingImport = append(missingImport, edge) + } + } + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls && edge.Line == 10 { + missingCall = append(missingCall, edge) + } + } + require.Len(t, missingImport, 1) + require.Equal(t, "module::pypi:PIL", missingImport[0].To) + require.Equal(t, "PIL.util", missingImport[0].Meta["python_import"]) + require.Equal(t, "module::pypi:PIL", missingImport[0].Meta["python_import_stub"]) + require.False(t, missingImport[0].CrossRepo) + require.Len(t, missingCall, 1) + require.Equal(t, "external-call::dep::PIL.util.helper", missingCall[0].To) + require.Equal(t, "unresolved::extern::PIL.util.helper::helper", missingCall[0].Meta["python_call_stub"]) + require.False(t, missingCall[0].CrossRepo) + } + var cold graph.Store = graph.New() + if backend == "sqlite" { + db, openErr := store_sqlite.Open(filepath.Join(t.TempDir(), "cold.sqlite")) + require.NoError(t, openErr) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + cold = db + } + coldConfig := cm + if operation == "untrack_only" { + coldConfig = newTestConfigManager(t) + coldConfig.Global().Repos = []config.RepoEntry{entries[0]} + } + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), coldConfig, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, pythonLifecycleBindings(cold, operation), pythonLifecycleBindings(g, operation), "after %s", operation) + require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) + require.Empty(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveAll").All()) + }) + } + } + } +} + +const pythonLifecycleProvider = "class Image:\n def show(self): pass\ndef f(): pass\n" +const pythonLifecycleConsumer = "from PIL.image import Image\nfrom PIL.image import f\nimport PIL.image as image\nfrom PIL.util import helper\ndef run():\n x = Image()\n x.show()\n f()\n image.f()\n helper()\n" + +func pythonLifecycleBindings(g graph.Store, operation string) []string { + var bindings []string + for _, node := range g.GetFileNodes("consumer/app.py") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind != graph.EdgeImports && edge.Kind != graph.EdgeCalls { + continue + } + copy := *edge + parked := operation == "untrack_only" || ((operation == "provider_delete_util_batched" || operation == "forced_evict_util") && ((edge.Kind == graph.EdgeImports && edge.Line == 4) || (edge.Kind == graph.EdgeCalls && edge.Line == 10))) + if parked && !edge.CrossRepo { + copy.Meta = maps.Clone(edge.Meta) + key := "python_call_stub" + if edge.Kind == graph.EdgeImports { + key = "python_import_stub" + } + delete(copy.Meta, key) // Historical parked identity is asserted separately. + } + bindings = append(bindings, builderRenderEdge(©)) + } + } + slices.Sort(bindings) + return bindings +} + +func TestPythonAddedFileWithoutConsumersDoesNotRunGlobalResolve(t *testing.T) { + core, logs := observer.New(zap.InfoLevel) + g := graph.New() + g.AddNode(&graph.Node{ID: "consumer/app.py::run", Kind: graph.KindFunction, Name: "run", FilePath: "consumer/app.py", RepoPrefix: "consumer", Language: "python"}) + g.AddNode(&graph.Node{ID: "provider/new.py::f", Kind: graph.KindFunction, Name: "f", FilePath: "provider/new.py", RepoPrefix: "provider", Language: "python"}) + g.AddEdge(&graph.Edge{From: "consumer/app.py::run", To: "unresolved::f", Kind: graph.EdgeCalls, FilePath: "consumer/app.py"}) + mi := &MultiIndexer{graph: g, logger: zap.New(core)} + mi.resolveIncrementalRepoMutation("provider", &IndexResult{}, &graph.MutationReceipt{Complete: true}, + &reparsePendingEnrichmentBatch{pythonAddedFiles: []string{"provider/empty.py"}}) + require.Empty(t, logs.FilterMessage("cross-repo resolve: pass start").All()) +} diff --git a/internal/indexer/python_import_restub.go b/internal/indexer/python_import_restub.go new file mode 100644 index 000000000..e54782597 --- /dev/null +++ b/internal/indexer/python_import_restub.go @@ -0,0 +1,215 @@ +package indexer + +import ( + "strings" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +// Preserve importer-owned facts before their provider's files disappear. +func restubTrackedPythonRepoIncoming(g graph.Store, repo string) []string { + if repo == "" { + return nil + } + var files, ids []string + flush := func() { + incoming := foreignBridgeIncoming(g, repo, ids) + files = appendUniqueSorted(files, restubTrackedPythonImportsFromView(g, nil, incoming, true)...) + ids = ids[:0] + } + for node := range graph.RepoNodeIdentitiesSeq(g, []string{repo}, graph.KindFile, graph.KindFunction, graph.KindMethod, graph.KindType) { + file, _, _ := strings.Cut(node.ID, "::") + if !strings.HasSuffix(file, ".py") { + continue + } + ids = append(ids, node.ID) + if len(ids) == 512 { + flush() + } + } + if len(ids) > 0 { + flush() + } + return files +} + +func restubTrackedPythonImportsFromView(g graph.Store, evicted map[string]struct{}, incoming map[string][]*graph.Edge, parkCalls bool) []string { + consumers := make(map[string]struct{}) + var batch []graph.EdgeReindex + for id, edges := range incoming { + if _, changing := evicted[id]; !parkCalls && !changing { + continue + } + for _, edge := range edges { + if edge == nil || !edge.CrossRepo { + continue + } + key := resolver.PythonImportStubMeta + if edge.Kind == graph.EdgeCalls { + key = resolver.PythonCallStubMeta + } else if edge.Kind != graph.EdgeImports { + continue + } + stub, _ := edge.Meta[key].(string) + if _, gone := evicted[edge.From]; stub == "" || gone || edge.To == stub { + continue + } + consumers[edge.FilePath] = struct{}{} + // Save paths keep the existing carry/restub decision and its dataflow sites. + if edge.Kind == graph.EdgeCalls && !parkCalls { + continue + } + old := edge.To + // A package edit can change a re-export without changing the old + // definition. Reopen its dependent calls when the import is restubbed. + if edge.Kind == graph.EdgeImports && strings.HasSuffix(old, "/__init__.py") { + for row := range graph.EdgesInScopeSeq(g, nil, []string{edge.FilePath}, graph.EdgeCalls) { + call := row.Edge + if call == nil || !call.CrossRepo || call.Meta[resolver.PythonReExportFileMeta] != old { + continue + } + original, _ := call.Meta[resolver.PythonCallStubMeta].(string) + if original == "" { + continue + } + previous := call.To + graph.StashRestubProvenance(call) + call.To, call.CrossRepo, call.ConfidenceLabel = original, false, "" + delete(call.Meta, resolver.PythonReExportFileMeta) + batch = append(batch, graph.EdgeReindex{Edge: call, OldTo: previous}) + // SQLite reads return separate objects. Update the prior view too, + // so the generic carry cannot re-state the old call target. + for _, prior := range incoming[previous] { + if prior != nil && prior.From == call.From && prior.Kind == call.Kind && prior.Line == call.Line && prior.To == previous { + prior.To, prior.CrossRepo, prior.ConfidenceLabel = call.To, false, "" + delete(prior.Meta, resolver.PythonReExportFileMeta) + } + } + } + } + if edge.Kind == graph.EdgeCalls { + graph.StashRestubProvenance(edge) + edge.ConfidenceLabel = "" + delete(edge.Meta, resolver.PythonReExportFileMeta) + } + edge.To, edge.CrossRepo = stub, false + batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: old}) + } + } + if len(batch) > 0 { + g.ReindexEdges(batch) + } + return sortedStringKeys(consumers) +} + +// Retry only parked imports claimed by this provider, or owned by this repo. +func (mi *MultiIndexer) pythonConsumerFilesForRepo(repo string) []string { + lookup := mi.pythonPackageLookup() + if lookup == nil { + return nil + } + return pythonConsumerFilesForLookup(mi.graph, lookup, repo) +} + +func pythonConsumerFilesForLookup(g graph.Store, lookup *resolver.PythonPackageLookup, repo string) []string { + files := make(map[string]struct{}) + type candidate struct{ caller, top string } + candidates := make(map[candidate]map[string]struct{}) + for edge := range graph.EdgesLightSeq(g, graph.EdgeImports) { + if !strings.HasSuffix(edge.FilePath, ".py") { + continue + } + top := "" + for _, prefix := range []string{"module::pypi:", "external::", "unresolved::import::"} { + if spec, ok := strings.CutPrefix(edge.To, prefix); ok { + top, _, _ = strings.Cut(spec, ".") + break + } + } + if top == "" || !lookup.HasTop(top) { + continue + } + key := candidate{graph.RepoPrefixOfID(edge.From), top} + if candidates[key] == nil { + candidates[key] = make(map[string]struct{}) + } + candidates[key][edge.FilePath] = struct{}{} + } + // Providers may query the store; release the import cursor first. + for item, paths := range candidates { + for provider := range lookup.Providers(item.caller, item.top) { + if provider == repo || item.caller == repo { + for file := range paths { + files[file] = struct{}{} + } + break + } + } + } + return sortedStringKeys(files) +} + +// Keep candidate importers before warmup has registered every repository. +// The existing cross-repo matcher still checks providers and declarations. +func pythonConsumerFilesForAddedFiles(g graph.Store, files []string) []string { + if len(files) == 0 { + return nil + } + tops := map[string]bool{} + for _, file := range files { + rel := strings.TrimPrefix(file, graph.RepoPrefixOfID(file)+"/") + for _, path := range []string{rel, strings.TrimPrefix(rel, "src/")} { + top, _, _ := strings.Cut(path, "/") + tops[strings.TrimSuffix(top, ".py")] = true + } + } + consumers := map[string]struct{}{} + for edge := range graph.EdgesLightSeq(g, graph.EdgeImports) { + if !strings.HasSuffix(edge.FilePath, ".py") { + continue + } + for _, prefix := range []string{"module::pypi:", "external::", "unresolved::import::"} { + if spec, ok := strings.CutPrefix(edge.To, prefix); ok { + top, _, _ := strings.Cut(spec, ".") + if tops[top] { + consumers[edge.FilePath] = struct{}{} + } + break + } + } + } + return sortedStringKeys(consumers) +} + +func (mi *MultiIndexer) pythonAddedFileConsumers(files []string) []string { + if len(files) == 0 { + return nil + } + // A live warmup batch can publish provider additions before persisted + // consumers register. Raw config entries are not evidence of pending work. + metadata := mi.AllMetadata() + warming := false + mode := mi.currentBatchMode() + if mode.deferGlobalPasses { + for _, repo := range mi.graph.RepoPrefixes() { + if metadata[repo] == nil { + warming = true + break + } + } + } + if !warming && len(metadata) < 2 { + return nil + } + if !warming && !mi.pythonManifestSnapshot().declaresPythonBridge() { + return nil + } + eligible := make([]string, 0, len(files)) + for _, file := range files { + if !resolver.IsPythonTestPackageFile(file, graph.RepoPrefixOfID(file)) { + eligible = append(eligible, file) + } + } + return pythonConsumerFilesForAddedFiles(mi.graph, eligible) +} diff --git a/internal/indexer/python_import_restub_test.go b/internal/indexer/python_import_restub_test.go new file mode 100644 index 000000000..1dddca9b5 --- /dev/null +++ b/internal/indexer/python_import_restub_test.go @@ -0,0 +1,118 @@ +package indexer + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/resolver" +) + +func TestPythonRestubLeavesUnmarkedImportsUnchanged(t *testing.T) { + for _, file := range []string{"provider/module.py", "provider/main.go"} { + t.Run(file, func(t *testing.T) { + g := graph.New() + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "provider"}) + edge := &graph.Edge{From: "consumer/source", FilePath: "consumer/source", To: file, Kind: graph.EdgeImports, CrossRepo: true} + if file == "provider/main.go" { + edge.Meta = map[string]any{resolver.PythonImportStubMeta: "external::PIL.image"} + } + g.AddEdge(edge) + require.Empty(t, restubTrackedPythonRepoIncoming(g, "provider")) + require.Equal(t, file, edge.To) + require.True(t, edge.CrossRepo) + }) + } +} + +func TestPythonRestubKeepsExactImportAndCallStubs(t *testing.T) { + g := graph.New() + file, caller, target := "consumer/app.py", "consumer/app.py::run", "provider/src/PIL/image.py" + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "consumer"}) + g.AddNode(&graph.Node{ID: caller, Kind: graph.KindFunction, FilePath: file, RepoPrefix: "consumer"}) + g.AddNode(&graph.Node{ID: target, Kind: graph.KindFile, FilePath: target, RepoPrefix: "provider"}) + importEdge := &graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{ + resolver.PythonImportStubMeta: "module::pypi:PIL", "python_import": "PIL.image", + }} + g.AddNode(&graph.Node{ID: target + "::f", Kind: graph.KindFunction, FilePath: target, RepoPrefix: "provider"}) + call := &graph.Edge{From: caller, FilePath: file, To: target + "::f", Kind: graph.EdgeCalls, CrossRepo: true, + Meta: map[string]any{resolver.PythonCallStubMeta: "unresolved::extern::PIL.image::f"}} + g.AddEdge(importEdge) + g.AddEdge(call) + require.Equal(t, []string{file}, restubTrackedPythonRepoIncoming(g, "provider")) + require.Equal(t, "module::pypi:PIL", importEdge.To) + require.Equal(t, "PIL.image", importEdge.Meta["python_import"]) + require.False(t, importEdge.CrossRepo) + require.Equal(t, "unresolved::extern::PIL.image::f", call.To) + require.False(t, call.CrossRepo) +} + +func TestPythonRestubIgnoresMetadataOnlyTargets(t *testing.T) { + g := graph.New() + target := "provider/src/PIL/image.py" + edge := &graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: target, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{resolver.PythonImportStubMeta: "module::pypi:PIL"}} + g.AddEdge(edge) + files := restubTrackedPythonImportsFromView(g, map[string]struct{}{"provider/src/PIL/changed.py": {}}, map[string][]*graph.Edge{target: {edge}}, false) + require.Empty(t, files) + require.Equal(t, target, edge.To) + require.True(t, edge.CrossRepo) +} + +func TestPythonRestubClearsReExportEvidence(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"point", "batch", "park"} { + t.Run(backend+"/"+route, func(t *testing.T) { + _, g := newPythonBridgeIdentityIndexer(t, backend, nil) + file, target := "consumer/app.py", "provider/pkg/mod.py" + caller := file + "::run" + g.AddNode(&graph.Node{ID: caller, Name: "run", FilePath: file, Kind: graph.KindFunction, RepoPrefix: "consumer", Language: "python"}) + g.AddBatch([]*graph.Node{{ID: file, Name: "app.py", FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "python"}, {ID: target, Name: "mod.py", FilePath: target, Kind: graph.KindFile, RepoPrefix: "provider", Language: "python"}, {ID: target + "::f", Name: "f", FilePath: target, Kind: graph.KindFunction, RepoPrefix: "provider", Language: "python"}}, nil) + imp := &graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports, CrossRepo: true, Line: 1, Meta: map[string]any{resolver.PythonImportStubMeta: "external::pkg.mod", "python_import": "pkg.mod"}} + linkedImport := *imp + g.AddEdge(imp) + stub := "unresolved::extern::pkg.mod::f" + g.AddEdge(&graph.Edge{From: caller, FilePath: file, To: stub, Kind: graph.EdgeCalls, Line: 2, Meta: map[string]any{resolver.PythonCallStubMeta: stub}}) + resolve := func() { + cr := resolver.NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *resolver.PythonPackageLookup { return nil }) + cr.ResolveForFile("consumer", "app.py") + } + resolve() + resolve() + calls := func() []*graph.Edge { + var rows []*graph.Edge + for _, e := range g.GetOutEdges(caller) { + if e.Kind == graph.EdgeCalls { + rows = append(rows, e) + } + } + require.Len(t, rows, 1) + return rows + } + require.Equal(t, target+"::f", calls()[0].To) + cold := builderRenderEdges(calls()) + call := calls()[0] + call.Meta[resolver.PythonReExportFileMeta] = "provider/pkg/__init__.py" + g.ReindexEdges([]graph.EdgeReindex{{Edge: call, OldTo: call.To}}) + switch route { + case "point": + newTestIndexer(g).restubIncomingRefs(target) + case "batch": + stages := []*incrementalBatchStage{{graphPath: target, priorNodes: g.GetFileNodes(target)}} + restubIncomingRefsFromView(g, stages, loadIncrementalPriorView(g, stages)) + case "park": + restubTrackedPythonRepoIncoming(g, "provider") + } + require.Equal(t, stub, calls()[0].To) + require.NotContains(t, calls()[0].Meta, resolver.PythonReExportFileMeta) + g.RemoveEdge(file, "external::pkg.mod", graph.EdgeImports) + g.AddEdge(&linkedImport) + resolve() + require.Equal(t, cold, builderRenderEdges(calls())) + resolve() + require.Equal(t, cold, builderRenderEdges(calls())) + }) + } + } +} diff --git a/internal/indexer/python_import_reuse_test.go b/internal/indexer/python_import_reuse_test.go new file mode 100644 index 000000000..091f3448e --- /dev/null +++ b/internal/indexer/python_import_reuse_test.go @@ -0,0 +1,25 @@ +package indexer + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" +) + +func TestPythonReuseKeepsFreshCallStubWithoutConfirmation(t *testing.T) { + prior := &graph.Edge{Meta: map[string]any{ + "confirmed_from_origin": "ast", + "semantic_source": "python-lsp", + "python_call_stub": "unresolved::extern::PIL.image.f::f", + }} + captured := reuseSemanticMeta(prior) + values := map[reuseKey]*reuseVal{{}: {semanticMeta: captured}} + stripReuseSemanticMeta(values) + require.Equal(t, map[string]any{"python_call_stub": "unresolved::extern::PIL.image.f::f"}, captured) + fresh := &graph.Edge{To: "provider/PIL/image.py::f"} + applyReuseSemanticMeta(fresh, captured, "unresolved::extern::PIL.image::f") + require.Equal(t, "unresolved::extern::PIL.image::f", fresh.Meta["python_call_stub"]) + require.NotContains(t, fresh.Meta, "semantic_source") + require.NotContains(t, fresh.Meta, "confirmed_from_origin") +} diff --git a/internal/indexer/python_import_warm_recovery_test.go b/internal/indexer/python_import_warm_recovery_test.go new file mode 100644 index 000000000..bbcd7ade1 --- /dev/null +++ b/internal/indexer/python_import_warm_recovery_test.go @@ -0,0 +1,102 @@ +package indexer + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedPythonWarmFileRecovery(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, deferred := range []bool{false, true} { + for _, restore := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/provider_first=%v/deferred=%v/restore=%v", backend, providerFirst, deferred, restore), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, provider, "src/PIL/image.py", pythonLifecycleProvider) + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + consumerSource := pythonLifecycleConsumer + if !restore { + consumerSource += "\nimport PIL.extra\n" + } + writeWorkspaceFile(t, consumer, "app.py", consumerSource) + for i := 0; i < 6; i++ { + writeWorkspaceFile(t, provider, fmt.Sprintf("pad%d.py", i), "def pad(): pass\n") + writeWorkspaceFile(t, consumer, fmt.Sprintf("pad%d.py", i), "def pad(): pass\n") + } + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + if providerFirst { + entries[0], entries[1] = entries[1], entries[0] + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + newMI := func(g graph.Store) *MultiIndexer { + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + return mi + } + g := pythonLocalStore(t, backend) + mi := newMI(g) + _, err = mi.IndexAll() + require.NoError(t, err) + if restore { + file := filepath.Join(provider, "src/PIL/util.py") + require.NoError(t, os.Remove(file)) + _, err = mi.IncrementalReindexRepo("provider", []string{file}) + require.NoError(t, err) + } + prior := map[string]map[string]int64{"consumer": mi.FileMtimes("consumer"), "provider": mi.FileMtimes("provider")} + require.NoError(t, mi.Close(context.Background())) + if restore { + writeWorkspaceFile(t, provider, "src/PIL/util.py", "def helper(): pass\n") + } else { + writeWorkspaceFile(t, provider, "src/PIL/extra.py", "") + } + restarted := newMI(g) + if deferred { + restarted.BeginParallelBatch() + } else { + restarted.BeginBatch() + } + var files []string + for _, entry := range entries { + result, err := restarted.ReconcileRepoCtx(context.Background(), entry, prior[entry.Name]) + require.NoError(t, err) + require.False(t, result.FullRetrack, "must exercise scoped warm reconcile") + if entry.Name == "provider" { + require.Contains(t, result.CrossRepoResolveFiles, "consumer/app.py") + } + files = appendUniqueSorted(files, result.DerivedInvalidation.Files...) + files = appendUniqueSorted(files, result.CrossRepoResolveFiles...) + } + require.NoError(t, restarted.RunPreEnrichResolveFiles(context.Background(), files, nil)) + restarted.EndBatch() + cold := pythonLocalStore(t, backend) + _, err = newMI(cold).IndexAll() + require.NoError(t, err) + require.Equal(t, pythonLifecycleBindings(cold, ""), pythonLifecycleBindings(g, "")) + }) + } + } + } + } +} diff --git a/internal/indexer/python_local_lifecycle_test.go b/internal/indexer/python_local_lifecycle_test.go new file mode 100644 index 000000000..cbc1b6912 --- /dev/null +++ b/internal/indexer/python_local_lifecycle_test.go @@ -0,0 +1,88 @@ +package indexer + +import ( + "context" + "fmt" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestPythonLocalRenameAndRestore(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, repos := range []string{"solo", "duo"} { + t.Run(backend+"/"+repos, func(t *testing.T) { + dir := t.TempDir() + original := "def foo(): pass\nclass K:\n def m(self): pass\n" + writeWorkspaceFile(t, dir, "a.py", original) + writeWorkspaceFile(t, dir, "b.py", "from a import foo, K\nimport a\ndef run():\n foo()\n a.foo()\n k = K()\n k.m()\n") + entries := []config.RepoEntry{{Path: dir, Name: "solo", Workspace: "shared"}} + if repos == "duo" { + other := t.TempDir() + writeWorkspaceFile(t, other, "z.py", "def foo(): pass\n") + entries = append(entries, config.RepoEntry{Path: other, Name: "other", Workspace: "shared"}) + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + g := pythonLocalStore(t, backend) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + _, err = mi.IndexAll() + require.NoError(t, err) + before := pythonLocalBindings(g) + writeWorkspaceFile(t, dir, "a.py", "def foo2(): pass\nclass K:\n def m2(self): pass\n") + _, err = mi.IncrementalReindexRepo("solo", []string{filepath.Join(dir, "a.py")}) + require.NoError(t, err) + writeWorkspaceFile(t, dir, "a.py", original) + _, err = mi.IncrementalReindexRepo("solo", []string{filepath.Join(dir, "a.py")}) + require.NoError(t, err) + require.Equal(t, before, pythonLocalBindings(g)) + cold := pythonLocalStore(t, backend) + coldIndexer := NewMultiIndexer(cold, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, coldIndexer.Close(context.Background())) }) + _, err = coldIndexer.IndexAll() + require.NoError(t, err) + require.Equal(t, pythonLocalBindings(cold), pythonLocalBindings(g)) + }) + } + } +} + +func pythonLocalStore(t *testing.T, backend string) graph.Store { + t.Helper() + if backend == "memory" { + return graph.New() + } + disk, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, disk.Close()) }) + return disk +} + +func pythonLocalBindings(g graph.Store) []string { + var bindings []string + for _, node := range g.GetFileNodes("solo/b.py") { + for _, edge := range g.GetOutEdges(node.ID) { + if edge.Kind == graph.EdgeImports || edge.Kind == graph.EdgeCalls { + bindings = append(bindings, fmt.Sprintf("%s -%s-> %s cross=%v line=%d stub=%v", edge.From, edge.Kind, edge.To, edge.CrossRepo, edge.Line, edge.Meta["python_call_stub"])) + } + } + } + slices.Sort(bindings) + return bindings +} diff --git a/internal/indexer/python_package_reexport_test.go b/internal/indexer/python_package_reexport_test.go new file mode 100644 index 000000000..e29ed1fd0 --- /dev/null +++ b/internal/indexer/python_package_reexport_test.go @@ -0,0 +1,221 @@ +package indexer + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/search" +) + +func TestTrackedPythonPackageReExport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, tc := range []struct { + name, exports string + resolves bool + }{ + {"explicit_alias", "from .url_safe import URLSafeTimedSerializer as URLSafeTimedSerializer\n", true}, + {"plain", "from .url_safe import URLSafeTimedSerializer\n", true}, + {"ambiguous", "from .url_safe import URLSafeTimedSerializer\nfrom .other import URLSafeTimedSerializer\n", false}, + {"not_exported", "from .other import unrelated\n", false}, + } { + t.Run(fmt.Sprintf("%s/provider_first=%v/%s", backend, providerFirst, tc.name), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'itsdangerous'\n") + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", tc.exports) + writeWorkspaceFile(t, provider, "src/itsdangerous/url_safe.py", "class URLSafeTimedSerializer: pass\n") + writeWorkspaceFile(t, provider, "src/itsdangerous/other.py", "class URLSafeTimedSerializer: pass\ndef unrelated(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['itsdangerous']\n") + writeWorkspaceFile(t, consumer, "app.py", "from itsdangerous import URLSafeTimedSerializer\ndef run(): return URLSafeTimedSerializer('secret')\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + if providerFirst { + entries[0], entries[1] = entries[1], entries[0] + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + g := pythonLocalStore(t, backend) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + for _, entry := range entries { + _, err = mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + var imports, calls []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py") { + if edge.Kind == graph.EdgeImports { + imports = append(imports, edge) + } + } + require.Len(t, imports, 1) + require.Equal(t, "provider/src/itsdangerous/__init__.py", imports[0].To) + require.True(t, imports[0].CrossRepo) + external := "external-call::dep::itsdangerous.URLSafeTimedSerializer" + assertCall := func(target string, linked bool) { + t.Helper() + calls = nil + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls { + calls = append(calls, edge) + } + } + require.Len(t, calls, 1) + require.Equal(t, linked, calls[0].CrossRepo, "%#v", calls[0]) + require.Equal(t, target, calls[0].To) + require.NotEmpty(t, calls[0].Meta["python_call_stub"]) + if target == external { + require.Equal(t, true, calls[0].Meta["external_call"]) + } else { + require.NotContains(t, calls[0].Meta, "external_call") + } + } + stub := "unresolved::extern::itsdangerous.URLSafeTimedSerializer::URLSafeTimedSerializer" + target := "provider/src/itsdangerous/url_safe.py::URLSafeTimedSerializer" + if !tc.resolves { + assertCall(stub, false) + return + } + assertCall(target, true) + if tc.name != "explicit_alias" { + return + } + writeWorkspaceFile(t, provider, "src/itsdangerous/url_safe.py", "class URLSafeTimedSerializer:\n def added(self): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/itsdangerous/url_safe.py")}) + require.NoError(t, err) + assertCall(target, true) + _, err = mi.IndexRepo("provider") + require.NoError(t, err) + assertCall(target, true) + _, _, err = mi.UntrackRepoChecked(context.Background(), "provider") + require.NoError(t, err) + // A fresh consumer-only IndexAll materializes this dependency call. + assertCall(external, false) + _, err = mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: provider, Name: "provider", Workspace: "shared"}) + require.NoError(t, err) + assertCall(target, true) + // Both files change structurally, but the old class keeps its shape. + // A stale incoming view must not carry its old call target. + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", "from .other import URLSafeTimedSerializer\n") + writeWorkspaceFile(t, provider, "src/itsdangerous/url_safe.py", "class URLSafeTimedSerializer:\n def added(self): pass\ndef extra(): pass\n") + _, err = mi.IncrementalReindexRepo("provider", []string{ + filepath.Join(provider, "src/itsdangerous/__init__.py"), + filepath.Join(provider, "src/itsdangerous/url_safe.py"), + }) + require.NoError(t, err) + assertCall("provider/src/itsdangerous/other.py::URLSafeTimedSerializer", true) + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", "from .url_safe import URLSafeTimedSerializer\nfrom .other import URLSafeTimedSerializer\n") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/itsdangerous/__init__.py")}) + require.NoError(t, err) + assertCall(stub, false) + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", "") + _, err = mi.IncrementalReindexRepo("provider", []string{filepath.Join(provider, "src/itsdangerous/__init__.py")}) + require.NoError(t, err) + assertCall(stub, false) + }) + } + } + } +} + +func TestTrackedPythonPackageReExportColdPaths(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, ambiguous := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/provider_first=%v/ambiguous=%v", backend, providerFirst, ambiguous), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + exports := "from .url_safe import URLSafeTimedSerializer\n" + if ambiguous { + exports += "from .other import URLSafeTimedSerializer\n" + } + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'itsdangerous'\n") + writeWorkspaceFile(t, provider, "src/itsdangerous/__init__.py", exports) + writeWorkspaceFile(t, provider, "src/itsdangerous/url_safe.py", "class URLSafeTimedSerializer: pass\n") + writeWorkspaceFile(t, provider, "src/itsdangerous/other.py", "class URLSafeTimedSerializer: pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['itsdangerous']\n") + writeWorkspaceFile(t, consumer, "app.py", "from itsdangerous import URLSafeTimedSerializer\ndef run(): return URLSafeTimedSerializer('secret')\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + if providerFirst { + entries[0], entries[1] = entries[1], entries[0] + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: entries} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + expected := "provider/src/itsdangerous/url_safe.py::URLSafeTimedSerializer" + if ambiguous { + expected = "" + } + incrementalTarget := "" + for _, mode := range []string{"incremental", "index_all", "parallel_full"} { + t.Run(mode, func(t *testing.T) { + g := pythonLocalStore(t, backend) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + switch mode { + case "index_all": + _, err := mi.IndexAll() + require.NoError(t, err) + case "parallel_full": + // The daemon's cold-start sequence defers resolution until + // every repository is registered. No daemon is started here. + mi.BeginParallelBatch() + defer mi.EndBatch() + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + require.NoError(t, mi.RunPreEnrichResolve(context.Background(), nil, nil)) + case "incremental": + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + } + var calls []*graph.Edge + for _, edge := range g.GetOutEdges("consumer/app.py::run") { + if edge.Kind == graph.EdgeCalls { + calls = append(calls, edge) + } + } + require.Len(t, calls, 1) + call := calls[0] + target := "" + if call.CrossRepo { + target = call.To + require.Equal(t, "unresolved::extern::itsdangerous.URLSafeTimedSerializer::URLSafeTimedSerializer", call.Meta["python_call_stub"]) + require.Equal(t, "provider/src/itsdangerous/__init__.py", call.Meta["python_reexport_file"]) + } + require.Equal(t, expected, target, "call: %#v", call) + if ambiguous { + require.Equal(t, "unresolved::extern::itsdangerous.URLSafeTimedSerializer::URLSafeTimedSerializer", call.To) + require.NotContains(t, call.Meta, "external_call") + } + if mode == "incremental" { + incrementalTarget = target + } else { + require.Equal(t, incrementalTarget, target) + } + }) + } + }) + } + } + } +} diff --git a/internal/indexer/python_save_resolve_work_test.go b/internal/indexer/python_save_resolve_work_test.go new file mode 100644 index 000000000..22ecfcd78 --- /dev/null +++ b/internal/indexer/python_save_resolve_work_test.go @@ -0,0 +1,97 @@ +package indexer + +import ( + "context" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + "path/filepath" + "testing" +) + +func TestPythonSaveRunsOneIncomingResolve(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"receipt", "point"} { + t.Run(backend+"/"+route, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'dist'\n") + writeWorkspaceFile(t, provider, "pkg/__init__.py", "") + writeWorkspaceFile(t, provider, "pkg/image.py", "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['dist']\n") + source := "from pkg.image import f\ndef run(): return f()\n" + writeWorkspaceFile(t, consumer, "app.py", source) + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + core, logs := observer.New(zap.InfoLevel) + mi.logger = zap.New(core) + save := func() { + t.Helper() + if route == "point" { + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "app.py"))) + } else { + _, err := mi.IncrementalReindexRepo("consumer", []string{filepath.Join(consumer, "app.py")}) + require.NoError(t, err) + } + require.Equal(t, []string{"provider/pkg/image.py::f"}, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + } + writeWorkspaceFile(t, consumer, "app.py", source+"def added(): pass\n") + save() + require.Equal(t, 1, logs.FilterMessage("DEFERRED-TIMING master.ResolveFilesAndIncoming").Len()) + logs.TakeAll() + save() + require.LessOrEqual(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveFilesAndIncoming").Len(), 1) + }) + } + } +} + +func TestPythonAffectedReferrerGetsBridgePreparation(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname='dist'\n") + writeWorkspaceFile(t, provider, "pkg/__init__.py", "") + writeWorkspaceFile(t, provider, "pkg/image.py", "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies=['dist']\n") + writeWorkspaceFile(t, consumer, "app.py", "from pkg.image import f\ndef run(): return f()\n") + writeWorkspaceFile(t, consumer, "defs.py", "def changed(): pass\n") + entries := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: "shared"}} + mi, g := newPythonBridgeIdentityIndexer(t, backend, entries) + for _, entry := range entries { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + idx := mi.GetIndexer("consumer") + require.NoError(t, idx.IndexFileNoResolve(filepath.Join(consumer, "app.py"))) + writeWorkspaceFile(t, consumer, "defs.py", "def changed(arg): pass\n") + require.NoError(t, idx.IndexFileNoResolve(filepath.Join(consumer, "defs.py"))) + core, logs := observer.New(zap.InfoLevel) + mi.logger = zap.New(core) + var masterFiles []string + idx.incrementalCatchupHook = func(kind string, files []string) { + if kind == "resolve" { + masterFiles = append(masterFiles, files...) + } + } + batch := &reparsePendingEnrichmentBatch{deferResolverCatchup: true, deferredAffectedFiles: map[string]struct{}{"consumer/app.py": {}}} + mi.resolveIncrementalRepoMutation("consumer", &IndexResult{DerivedInvalidation: DerivedInvalidationPlan{Files: []string{"consumer/defs.py"}}}, nil, batch) + require.Equal(t, []string{"consumer/defs.py"}, masterFiles) + outgoing := logs.FilterMessage("resolver: outgoing files phases").All() + require.Len(t, outgoing, 1, "affected referrers did not pass through the master callback") + require.EqualValues(t, 1, outgoing[0].ContextMap()["input_files"], "only the affected file needs preparation") + require.Equal(t, []string{"provider/pkg/image.py::f"}, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls)) + cold, cg := newPythonBridgeIdentityIndexer(t, backend, entries) + for _, entry := range entries { + _, err := cold.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } + require.Equal(t, pythonLifecycleBindings(cg, ""), pythonLifecycleBindings(g, "")) + }) + } +} diff --git a/internal/indexer/python_tracked_import.go b/internal/indexer/python_tracked_import.go new file mode 100644 index 000000000..1a740ffa5 --- /dev/null +++ b/internal/indexer/python_tracked_import.go @@ -0,0 +1,169 @@ +package indexer + +import ( + "path" + "regexp" + "strings" + "sync" + + "github.com/pelletier/go-toml/v2" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/modules" + "github.com/zzet/gortex/internal/resolver" +) + +var pythonDistributionSeparators = regexp.MustCompile(`[-_.]+`) + +func normalizePythonDistribution(name string) string { + return pythonDistributionSeparators.ReplaceAllString(strings.ToLower(name), "-") +} + +// pythonManifestEntry holds a distribution identity and declared dependencies. +// Declaration admission reads these entries without discovering provider files. +type pythonManifestEntry struct { + name string + deps map[string]bool +} + +func readPythonPackageEntries(trees map[string]manifestTree) map[string]*pythonManifestEntry { + if len(trees) < 2 { + return nil + } + entries := map[string]*pythonManifestEntry{} + for repo, tree := range trees { + src, _ := tree.readFile("pyproject.toml") + var manifest struct { + Project struct{ Name string } `toml:"project"` + Tool struct { + Poetry struct { + Name string + Group map[string]struct{ Dependencies map[string]any } `toml:"group"` + } `toml:"poetry"` + } `toml:"tool"` + } + // A malformed manifest supplies no identity or dependencies. + if len(src) > 0 && toml.Unmarshal(src, &manifest) != nil { + rejectBridgeManifest(tree) + continue + } + name := manifest.Project.Name + if name == "" { + name = manifest.Tool.Poetry.Name + } + deps := map[string]bool{} + specs := modules.ParsePyProject(src) + for _, group := range manifest.Tool.Poetry.Group { + for dep := range group.Dependencies { + if dep != "python" { + deps[normalizePythonDistribution(dep)] = true + } + } + } + for _, file := range tree.matchFiles("requirements*.txt") { + src, _ := tree.readFile(file) + specs = append(specs, modules.ParseRequirementsTxt(src)...) + } + for _, dep := range specs { + deps[normalizePythonDistribution(dep.Path)] = true + } + entries[repo] = &pythonManifestEntry{name: normalizePythonDistribution(name), deps: deps} + } + return entries +} + +func (mi *MultiIndexer) buildPythonPackageLookup(trees map[string]manifestTree, entries map[string]*pythonManifestEntry) *resolver.PythonPackageLookup { + // Only declared distributions can supply providers. Discover top-level names + // from their root/src directory entries, then probe exact indexed node IDs. + packages := map[string]map[string][]graph.FileNodeIdentity{} + for repo, pkg := range entries { + declared := false + for caller, consumer := range entries { + if caller != repo && pkg.name != "" && consumer.deps[pkg.name] { + declared = true + break + } + } + if !declared { + continue + } + tree := trees[repo] + for _, dir := range []string{"", "src"} { + files := tree.matchFiles(path.Join(dir, "*.py")) + for _, pkg := range tree.matchDirs(path.Join(dir, "*")) { + files = append(files, pkg+"/__init__.py") + } + for _, file := range files { + name := strings.TrimSuffix(strings.TrimPrefix(file, dir+"/"), "/__init__.py") + if dir == "" && pythonRootToolingFile(name) { + continue + } + top := strings.TrimSuffix(name, ".py") + if resolver.IsPythonTestPackageFile(file, "") { + continue // Root/src test packages are not provider exports. + } + node := mi.graph.GetNode(repo + "/" + file) + if node == nil || node.Kind != graph.KindFile || node.RepoPrefix != repo { + continue + } + if packages[top] == nil { + packages[top] = map[string][]graph.FileNodeIdentity{} + } + packages[top][repo] = append(packages[top][repo], graph.FileNodeIdentity{ + ID: node.ID, FilePath: node.FilePath, RepoPrefix: repo, WorkspaceID: node.WorkspaceID, + }) + } + } + } + if len(packages) == 0 { + return nil + } + localTops := map[string]map[string]bool{} + var localMu sync.Mutex + return &resolver.PythonPackageLookup{ + HasTop: func(top string) bool { return len(packages[top]) > 0 }, + Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + consumer := entries[caller] + if consumer == nil || len(packages[top]) == 0 { + return nil + } + localMu.Lock() + defer localMu.Unlock() + local, seen := localTops[caller] + if !seen { + local = map[string]bool{} + // Record both the literal top-level name and the src-layout name: + // a root `src` package and src/ must each stay local, and an + // extra local name only blocks a cross-repo link. + for file := range graph.FileNodeIdentitiesSeq(mi.graph, []string{caller}) { + if !strings.HasSuffix(file.FilePath, ".py") { + continue + } + rel := strings.TrimPrefix(file.FilePath, caller+"/") + for _, path := range []string{rel, strings.TrimPrefix(rel, "src/")} { + name, _, _ := strings.Cut(path, "/") + local[strings.TrimSuffix(name, ".py")] = true + } + } + localTops[caller] = local + } + if local[top] { + return nil + } + providers := map[string][]graph.FileNodeIdentity{} + for repo, files := range packages[top] { + if repo != caller && consumer.deps[entries[repo].name] { + providers[repo] = files + } + } + return providers + }, + } +} + +func pythonRootToolingFile(name string) bool { + switch name { + case "setup.py", "conftest.py", "noxfile.py", "fabfile.py", "manage.py": + return true + } + return false +} diff --git a/internal/indexer/python_tracked_import_test.go b/internal/indexer/python_tracked_import_test.go new file mode 100644 index 000000000..ead3a3ab1 --- /dev/null +++ b/internal/indexer/python_tracked_import_test.go @@ -0,0 +1,408 @@ +package indexer + +import ( + "context" + "fmt" + "iter" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/resolver" + "github.com/zzet/gortex/internal/search" +) + +func TestMultiIndexer_TrackedPythonImport(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, tc := range []struct { + name, manifest, requirements, entry, statement, providerName, workspace, localFile, extraLocal string + duplicate, cloneMissing, resolves bool + }{ + {name: "src module distribution differs", manifest: "[project]\ndependencies = ['pillow >= 1']", entry: "src/PIL/image.py", statement: "import PIL.image", resolves: true}, + {name: "package init", manifest: "[project]\ndependencies = ['pillow']", entry: "PIL/image/__init__.py", statement: "import PIL.image", resolves: true}, + {name: "requirements", requirements: "PILLOW[extra]>=1; python_version > '3'\n", entry: "src/PIL/image.py", statement: "import PIL.image", resolves: true}, + {name: "from package", manifest: "[project.optional-dependencies]\nimages = ['pillow']", entry: "src/PIL/__init__.py", statement: "from PIL import image", resolves: true}, + {name: "poetry dependency", manifest: "[tool.poetry.dependencies]\npillow = {path = '../provider'}", entry: "PIL/image.py", statement: "from PIL.image import f", resolves: true}, + {name: "poetry group normalized", manifest: "[tool.poetry.group.dev.dependencies]\nimage-dist = '*'", providerName: "Image._Dist", entry: "PIL/image.py", statement: "import PIL.image", resolves: true}, + {name: "top level module", manifest: "[project]\ndependencies = ['pillow']", entry: "image.py", statement: "import image", resolves: true}, + {name: "consumer namespace root", manifest: "[project]\ndependencies = ['pillow']", entry: "tests/helpers.py", statement: "import tests.helpers", localFile: "tests/helpers.py"}, + {name: "consumer namespace src", manifest: "[project]\ndependencies = ['pillow']", entry: "tests/helpers.py", statement: "import tests.helpers", localFile: "src/tests/helpers.py"}, + {name: "consumer root src package", manifest: "[project]\ndependencies = ['pillow']", entry: "src/foo.py", statement: "import src.foo", localFile: "src/__init__.py"}, + {name: "consumer src init with src namespace", manifest: "[project]\ndependencies = ['pillow']", entry: "tests/helpers.py", statement: "import tests.helpers", localFile: "src/__init__.py", extraLocal: "src/tests/helpers.py"}, + {name: "stdlib package", manifest: "[project]\ndependencies = ['pillow']", entry: "json/__init__.py", statement: "import json"}, + {name: "root setup script", manifest: "[project]\ndependencies = ['pillow']", entry: "setup.py", statement: "import setup"}, + {name: "root conftest script", manifest: "[project]\ndependencies = ['pillow']", entry: "conftest.py", statement: "import conftest"}, + {name: "root noxfile script", manifest: "[project]\ndependencies = ['pillow']", entry: "noxfile.py", statement: "import noxfile"}, + {name: "root fabfile script", manifest: "[project]\ndependencies = ['pillow']", entry: "fabfile.py", statement: "import fabfile"}, + {name: "root manage script", manifest: "[project]\ndependencies = ['pillow']", entry: "manage.py", statement: "import manage"}, + {name: "malformed consumer", manifest: "[project", entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "undeclared", entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "same named undeclared repo", manifest: "[project]\ndependencies = ['other']", providerName: "PIL", entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "ambiguous", manifest: "[project]\ndependencies = ['pillow']", duplicate: true, entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "ambiguous before module probe", manifest: "[project]\ndependencies = ['pillow']", duplicate: true, cloneMissing: true, entry: "src/PIL/image.py", statement: "import PIL.image"}, + {name: "workspace refusal", manifest: "[project]\ndependencies = ['pillow']", workspace: "private", entry: "src/PIL/image.py", statement: "import PIL.image"}, + } { + t.Run(backend+"/"+tc.name, func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + name := tc.providerName + if name == "" { + name = "Pillow" + } + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = '"+name+"'\n") + packageRoot := "PIL/__init__.py" + if strings.HasPrefix(tc.entry, "src/") { + packageRoot = "src/" + packageRoot + } + writeWorkspaceFile(t, provider, packageRoot, "") + if strings.HasPrefix(tc.entry, "tests/") { + writeWorkspaceFile(t, provider, "tests/__init__.py", "") + } + if tc.statement == "import src.foo" { + writeWorkspaceFile(t, provider, "src/__init__.py", "") + } + if tc.localFile != "" { + writeWorkspaceFile(t, consumer, tc.localFile, "def local(): pass\n") + } + if tc.extraLocal != "" { + writeWorkspaceFile(t, consumer, tc.extraLocal, "def local(): pass\n") + } + writeWorkspaceFile(t, provider, tc.entry, "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", tc.manifest) + writeWorkspaceFile(t, consumer, "requirements-dev.txt", tc.requirements) + writeWorkspaceFile(t, consumer, "app.py", tc.statement+"\n") + ws := tc.workspace + if ws == "" { + ws = "shared" + } + repos := []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: "provider", Workspace: ws}} + if tc.duplicate { + clone := t.TempDir() + writeWorkspaceFile(t, clone, "pyproject.toml", "[tool.poetry]\nname = 'pillow'\n") + writeWorkspaceFile(t, clone, "src/PIL/__init__.py", "") + if !tc.cloneMissing { + writeWorkspaceFile(t, clone, tc.entry, "def f(): pass\n") + } + repos = append(repos, config.RepoEntry{Path: clone, Name: "clone", Workspace: "shared"}) + } + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + gc := &config.GlobalConfig{Repos: repos} + gc.SetConfigPath(cfgPath) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(cfgPath) + require.NoError(t, err) + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) + _, err = mi.IndexAll() + require.NoError(t, err) + var imports []*graph.Edge + for _, e := range g.GetOutEdges("consumer/app.py") { + if e.Kind == graph.EdgeImports { + imports = append(imports, e) + } + } + require.Len(t, imports, 1) + require.Equal(t, tc.resolves, imports[0].CrossRepo, "target: %s", imports[0].To) + if tc.resolves { + require.Equal(t, "provider/"+tc.entry, imports[0].To) + } else { + require.NotEqual(t, "provider/"+tc.entry, imports[0].To) + require.NotEqual(t, "clone/"+tc.entry, imports[0].To) + } + }) + } + } +} + +func TestPythonPackageLookupNoProviders(t *testing.T) { + for _, count := range []int{1, 2} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + mi := &MultiIndexer{graph: graph.New(), repos: map[string]*RepoMetadata{}} + for i := 0; i < count; i++ { + mi.repos[fmt.Sprint(i)] = &RepoMetadata{RootPath: t.TempDir()} + } + require.Nil(t, mi.pythonPackageLookup()) + }) + } +} + +type pythonLookupScanStore struct { + graph.Store + globalScans int + importScans int + nodeReads int +} + +func (s *pythonLookupScanStore) GetNode(id string) *graph.Node { + s.nodeReads++ + return s.Store.GetNode(id) +} + +func (s *pythonLookupScanStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { + if repos == nil { + s.globalScans++ + } + return graph.FileNodeIdentitiesSeq(s.Store, repos) +} + +func TestPythonPackageLookupAvoidsGlobalFileScan(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, provider, "src/PIL/__init__.py", "") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['pillow']\n") + g := &pythonLookupScanStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "provider/src/PIL/__init__.py", FilePath: "provider/src/PIL/__init__.py", Kind: graph.KindFile, RepoPrefix: "provider"}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{ + "consumer": {RootPath: consumer}, "provider": {RootPath: provider}, + }} + lookup := mi.pythonPackageLookup() + require.NotNil(t, lookup) + require.Len(t, lookup.Providers("consumer", "PIL"), 1) + require.Zero(t, g.globalScans) +} + +func TestPythonPackageLookupUsesInstalledManifestTree(t *testing.T) { + for _, entry := range []string{"PIL.py", "PIL/__init__.py", "src/PIL.py", "src/PIL/__init__.py"} { + for _, mode := range []string{"pyproject", "requirements", "layout", "undeclared", "missing_layout"} { + t.Run(entry+"/"+mode, func(t *testing.T) { + provider, consumer, providerSnapshot, consumerSnapshot := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir() + providerName, checkoutDeps := "pillow", "[project]\ndependencies = ['pillow']\n" + if mode == "pyproject" { + providerName, checkoutDeps = "wrong-checkout", "" + } + snapshotDeps := "[project]\ndependencies = ['pillow']\n" + if mode == "requirements" || mode == "undeclared" { + snapshotDeps = "" + } + if mode == "requirements" { + checkoutDeps = "" + writeWorkspaceFile(t, consumer, "requirements-dev.txt", "other\n") + writeWorkspaceFile(t, consumerSnapshot, "requirements-dev.txt", "pillow >= 1\n") + } + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = '"+providerName+"'\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", checkoutDeps) + writeWorkspaceFile(t, providerSnapshot, "pyproject.toml", "[project]\nname = 'pillow'\n") + writeWorkspaceFile(t, consumerSnapshot, "pyproject.toml", snapshotDeps) + if mode != "layout" { + writeWorkspaceFile(t, provider, entry, "") + } + if mode != "missing_layout" { + writeWorkspaceFile(t, providerSnapshot, entry, "") + } + g := graph.New() + id := "provider/" + entry + g.AddNode(&graph.Node{ID: id, FilePath: id, Kind: graph.KindFile, RepoPrefix: "provider"}) + providerIdx, consumerIdx := sideChannelIndexer(t, g, provider), sideChannelIndexer(t, g, consumer) + providerIdx.SetContentSource(snapshotOf(t, providerSnapshot)) + consumerIdx.SetContentSource(snapshotOf(t, consumerSnapshot)) + mi := &MultiIndexer{graph: g, + repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}, + indexers: map[string]*Indexer{"provider": providerIdx, "consumer": consumerIdx}} + lookup := mi.pythonPackageLookup() + if mode == "undeclared" || mode == "missing_layout" { + require.Nil(t, lookup, "checkout-only declaration/layout must not supply a provider") + return + } + require.NotNil(t, lookup) + require.Equal(t, []graph.FileNodeIdentity{{ID: id, FilePath: id, RepoPrefix: "provider"}}, lookup.Providers("consumer", "PIL")["provider"]) + }) + } + } +} + +func (s *pythonLookupScanStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + if len(kinds) == 1 && kinds[0] == graph.EdgeImports { + s.importScans++ + } + return graph.EdgesLightSeq(s.Store, kinds...) +} + +func TestPythonAddedFileConsumerFrontier(t *testing.T) { + g := &pythonLookupScanStore{Store: graph.New()} + for _, row := range []struct{ file, target string }{ + {"consumer/a.py", "module::pypi:PIL"}, + {"consumer/b.py", "external::PIL.extra"}, + {"consumer/c.py", "unresolved::import::PIL.extra"}, + {"consumer/d.py", "module::pypi:other"}, + {"consumer/e.py", "provider/src/PIL/image.py"}, + {"consumer/f.rs", "external::PIL.extra"}, + } { + g.AddEdge(&graph.Edge{From: row.file, FilePath: row.file, To: row.target, Kind: graph.EdgeImports}) + } + require.Empty(t, pythonConsumerFilesForAddedFiles(g, nil)) + require.Zero(t, g.importScans) + require.Equal(t, []string{"consumer/a.py", "consumer/b.py", "consumer/c.py"}, pythonConsumerFilesForAddedFiles(g, []string{"provider/src/PIL/extra.py"})) + require.Equal(t, 1, g.importScans) +} + +// A one-connection store cannot hydrate package identities under an open cursor. +type pythonCursorGuardStore struct { + graph.Store + t *testing.T + cursorOpen bool +} + +func (s *pythonCursorGuardStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + return func(yield func(*graph.Edge) bool) { + s.cursorOpen = true + defer func() { s.cursorOpen = false }() + for edge := range graph.EdgesLightSeq(s.Store, kinds...) { + if !yield(edge) { + return + } + } + } +} + +func (s *pythonCursorGuardStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { + require.False(s.t, s.cursorOpen, "package lookup must not query while the import cursor is open") + return graph.FileNodeIdentitiesSeq(s.Store, repos) +} + +func TestPythonConsumerLookupClosesImportCursor(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var store graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store = db + } + g := &pythonCursorGuardStore{Store: store, t: t} + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'library'\n") + writeWorkspaceFile(t, provider, "mylib.py", "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['library']\n") + for _, file := range []string{"provider/mylib.py", "consumer/app.py"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "python"}) + } + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: "external::mylib", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + require.Equal(t, []string{"consumer/app.py"}, mi.pythonConsumerFilesForRepo("provider")) + }) + } +} + +func TestSingleRepoPythonAddedFileSkipsCrossRepoScan(t *testing.T) { + root := t.TempDir() + writeWorkspaceFile(t, root, "app.py", "import missing\n") + g := &pythonLookupScanStore{Store: graph.New()} + reg := newTestRegistry() + reg.Register(languages.NewPythonExtractor()) + mi := NewMultiIndexer(g, reg, search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + _, err := mi.TrackRepoCtx(t.Context(), config.RepoEntry{Path: root, Name: "solo", Workspace: "shared"}) + require.NoError(t, err) + g.importScans = 0 + writeWorkspaceFile(t, root, "missing.py", "pass\n") + _, err = mi.IncrementalReindexRepo("solo", []string{filepath.Join(root, "missing.py")}) + require.NoError(t, err) + require.Zero(t, g.importScans, "a lone Python repo has no foreign provider to recover") +} + +func TestPythonAddedFileGateDoesNotBuildLookup(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname='library'\n") + writeWorkspaceFile(t, provider, "mylib.py", "") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies=['library']\n") + g := &pythonLookupScanStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "provider/mylib.py", FilePath: "provider/mylib.py", Kind: graph.KindFile, RepoPrefix: "provider"}) + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: "external::mylib", Kind: graph.EdgeImports}) + mi := &MultiIndexer{graph: g, repos: map[string]*RepoMetadata{"provider": {RootPath: provider}, "consumer": {RootPath: consumer}}} + require.Equal(t, []string{"consumer/app.py"}, mi.pythonAddedFileConsumers([]string{"provider/mylib.py"})) + require.Zero(t, g.nodeReads, "the added-file candidate gate must not build package lookup") +} + +func TestPythonAddedFileNilGlobal(t *testing.T) { + mi := &MultiIndexer{graph: graph.New(), configMgr: &config.ConfigManager{}} + require.NotPanics(t, func() { require.Empty(t, mi.pythonAddedFileConsumers([]string{"solo/new.py"})) }) +} + +func TestPythonConsumerLookupFiltersAndDeduplicates(t *testing.T) { + g := graph.New() + for i, target := range []string{"external::known", "external::known.sub", "external::known", "external::unknown", "local/file.py", "module::python:os"} { + file := fmt.Sprintf("consumer/f%d.py", i) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports}) + } + calls := 0 + lookup := &resolver.PythonPackageLookup{ + HasTop: func(top string) bool { return top == "known" }, + Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + calls++ + require.Equal(t, "consumer", caller) + require.Equal(t, "known", top) + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/known.py"}}} + }, + } + require.Equal(t, []string{"consumer/f0.py", "consumer/f1.py", "consumer/f2.py"}, pythonConsumerFilesForLookup(g, lookup, "provider")) + require.Equal(t, 1, calls, "provider lookup is per (caller, top), not per import") +} + +func TestTrackedPythonProviderTestPackagesRefused(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, dir := range []string{"", "src/"} { + for _, top := range []string{"tests", "test"} { + for _, local := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s%s/local=%v", backend, dir, top, local), func(t *testing.T) { + provider, consumer := t.TempDir(), t.TempDir() + providerAlias := "provider" + if dir == "src/" && top == "test" { + providerAlias = "tests" + } + writeWorkspaceFile(t, provider, "pyproject.toml", "[project]\nname = 'image-distribution'\n") + writeWorkspaceFile(t, provider, dir+top+"/__init__.py", "") + writeWorkspaceFile(t, provider, dir+top+"/helpers.py", "def f(): pass\n") + writeWorkspaceFile(t, provider, dir+"image.py", "def f(): pass\n") + writeWorkspaceFile(t, consumer, "pyproject.toml", "[project]\ndependencies = ['image-distribution']\n") + writeWorkspaceFile(t, consumer, "app.py", "import "+top+".helpers\nimport image\ndef run():\n "+top+".helpers.f()\n") + if local { + writeWorkspaceFile(t, consumer, top+"/__init__.py", "") + writeWorkspaceFile(t, consumer, top+"/helpers.py", "def f(): pass\n") + } + mi, g := newPythonBridgeIdentityIndexer(t, backend, []config.RepoEntry{{Path: consumer, Name: "consumer", Workspace: "shared"}, {Path: provider, Name: providerAlias, Workspace: "shared"}}) + _, err := mi.IndexAll() + require.NoError(t, err) + check := func() { + var targets []string + for _, e := range g.GetOutEdges("consumer/app.py") { + if e.Kind != graph.EdgeImports { + continue + } + targets = append(targets, e.To) + require.NotEqual(t, providerAlias+"/"+dir+top+"/helpers.py", e.To) + if e.To == providerAlias+"/"+dir+"image.py" { + require.True(t, e.CrossRepo) + } + } + require.Contains(t, targets, providerAlias+"/"+dir+"image.py") + if local { + require.Contains(t, revisionCallTargets(g, "consumer/app.py::run", graph.EdgeCalls), "consumer/"+top+"/helpers.py::f") + } + for _, e := range g.GetOutEdges("consumer/app.py::run") { + require.NotEqual(t, providerAlias+"/"+dir+top+"/helpers.py::f", e.To) + } + } + check() + require.NoError(t, mi.GetIndexer("consumer").IndexFile(filepath.Join(consumer, "app.py"))) + check() + }) + } + } + } + } +} diff --git a/internal/indexer/repo_eviction_scope.go b/internal/indexer/repo_eviction_scope.go index bed6e0478..a181486a5 100644 --- a/internal/indexer/repo_eviction_scope.go +++ b/internal/indexer/repo_eviction_scope.go @@ -11,6 +11,8 @@ import ( // Store eviction is the compatible fallback. func evictRepoCurrentGeneration(target graph.Store, repoPrefix string) (nodesRemoved, edgesRemoved int) { restubTrackedNpmRepoIncoming(target, repoPrefix) + restubTrackedCargoRepoIncoming(target, repoPrefix) + restubTrackedPythonRepoIncoming(target, repoPrefix) if scoped, ok := target.(graph.CurrentGenerationRepoEvicter); ok { return scoped.EvictRepoCurrentGeneration(repoPrefix) } diff --git a/internal/indexer/repository_untrack.go b/internal/indexer/repository_untrack.go index 3915be809..b2f2ccf85 100644 --- a/internal/indexer/repository_untrack.go +++ b/internal/indexer/repository_untrack.go @@ -189,7 +189,7 @@ func (mi *MultiIndexer) untrackRepoCheckedRetainingAdmission( // retry does not repeat a completed destructive scan. refresh := func(sw *search.Swappable) error { if !state.payloadPurged { - removedNodes, removedEdges, purgeErr := mi.purgeRepositoryPayload(repoPrefix, state.metadata) + removedNodes, removedEdges, purgeErr := mi.purgeRepositoryPayload(repoPrefix, state.metadata, &state.bridgeConsumerFiles) if purgeErr != nil { return purgeErr } @@ -287,17 +287,24 @@ func (mi *MultiIndexer) finalizeCompletedRepositoryUntrackConfig( } func (mi *MultiIndexer) purgeRepositoryPayload( - repoPrefix string, meta *RepoMetadata, + repoPrefix string, meta *RepoMetadata, bridgeConsumers *[]string, ) (nodesRemoved, edgesRemoved int, err error) { restubTrackedNpmRepoIncoming(mi.graph, repoPrefix) parked := restubGoModuleRepoIncoming(mi.graph, repoPrefix) + var localConsumers []string + if bridgeConsumers == nil { + bridgeConsumers = &localConsumers + } + *bridgeConsumers = appendUniqueSorted(*bridgeConsumers, restubTrackedCargoRepoIncoming(mi.graph, repoPrefix)...) + *bridgeConsumers = appendUniqueSorted(*bridgeConsumers, restubTrackedPythonRepoIncoming(mi.graph, repoPrefix)...) if purger, ok := mi.graph.(interface{ PurgeRepo(string) error }); ok { if err := purger.PurgeRepo(repoPrefix); err != nil { return 0, 0, fmt.Errorf("purge repository payload: %w", err) } - if err := mi.finalizeUntrackedGoModuleCalls(parked); err != nil { + if err := mi.finalizeUntrackedGoModuleCalls(repoPrefix, parked); err != nil { return 0, 0, err } + mi.finalizeUntrackedBridgeConsumers(*bridgeConsumers) if meta != nil { return meta.NodeCount, meta.EdgeCount, nil } @@ -305,12 +312,20 @@ func (mi *MultiIndexer) purgeRepositoryPayload( } nodesRemoved, edgesRemoved, err = evictRepoAllGenerations(mi.graph, repoPrefix) if err == nil { - err = mi.finalizeUntrackedGoModuleCalls(parked) + err = mi.finalizeUntrackedGoModuleCalls(repoPrefix, parked) + if err == nil { + mi.finalizeUntrackedBridgeConsumers(*bridgeConsumers) + } } return nodesRemoved, edgesRemoved, err } -func (mi *MultiIndexer) finalizeUntrackedGoModuleCalls(parked []graph.EdgeIdentity) error { +func (mi *MultiIndexer) finalizeUntrackedGoModuleCalls(removedRepo string, parked []graph.EdgeIdentity) error { + // A refused tie parks nothing, so drop the projection before the early return. + mu := mi.graph.ResolveMutex() + mu.Lock() + mi.goModuleRetry.InvalidatePackageRepo(removedRepo) + mu.Unlock() if len(parked) == 0 { return nil } @@ -322,3 +337,31 @@ func (mi *MultiIndexer) finalizeUntrackedGoModuleCalls(parked []graph.EdgeIdenti mi.reconcileRetargetedTestCalls(cr.TakeRetargetedTestCallFiles()) return nil } + +// Removal must settle parked callers too; an immediate retrack used to hide +// their unclassified terminals and unfinished provenance handoff. +func (mi *MultiIndexer) finalizeUntrackedBridgeConsumers(files []string) { + if len(files) == 0 { + return + } + mi.runMasterResolveFiles(files, false) + mi.synthesizeDemotedGoModuleCalls(files) // Scoped synthesis honors each caller's config for every language. + if master := mi.newMasterResolver(true); master != nil { + master.ReconcileBridgeCallerTerminals(files) + } + mu := mi.graph.ResolveMutex() + mu.Lock() + defer mu.Unlock() + var batch []graph.EdgeReindex + for row := range graph.EdgesInScopeSeq(mi.graph, nil, files, graph.EdgeCalls) { + edge := row.Edge + if !graph.HasRestubProvenance(edge) { + continue + } + graph.RestoreRestubProvenance(edge) + batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: edge.To}) + } + if len(batch) > 0 { + mi.graph.ReindexEdges(batch) + } +} diff --git a/internal/indexer/repository_untrack_retry_test.go b/internal/indexer/repository_untrack_retry_test.go index b51350cb0..5ca497c69 100644 --- a/internal/indexer/repository_untrack_retry_test.go +++ b/internal/indexer/repository_untrack_retry_test.go @@ -473,3 +473,109 @@ func BenchmarkRepositoryCleanupFailureRetry(b *testing.B) { }) } } + +type bridgeRetryCleanupStore struct { + graph.Store + failed bool +} + +func (s *bridgeRetryCleanupStore) PurgeRepo(prefix string) error { + if !s.failed { + s.failed = true + return errInjectedRepositoryPurge + } + _, _, err := evictRepoAllGenerations(s.Store, prefix) + return err +} + +func (s *bridgeRetryCleanupStore) CountNodesByNameClass(names []string, kinds []graph.NodeKind) map[string]graph.NodeNameClassCount { + if counter, ok := s.Store.(graph.NodeNameClassCounter); ok { + return counter.CountNodesByNameClass(names, kinds) + } + allowed := map[graph.NodeKind]bool{} + for _, kind := range kinds { + allowed[kind] = true + } + out := map[string]graph.NodeNameClassCount{} + for _, name := range names { + count := graph.NodeNameClassCount{} + for _, node := range s.FindNodesByName(name) { + if graph.IsStub(node.ID) { + count.Stub++ + } else if allowed[node.Kind] { + count.Real++ + } + } + out[name] = count + } + return out +} + +type bridgeUnknownCounterStore struct{ graph.Store } + +func (s *bridgeUnknownCounterStore) PurgeRepo(prefix string) error { + return s.Store.(interface{ PurgeRepo(string) error }).PurgeRepo(prefix) +} + +type bridgeTerminalLSP struct{} + +func (bridgeTerminalLSP) SupportsPath(string) bool { return true } +func (bridgeTerminalLSP) Definition(string, int, string) (string, int, bool) { + panic("terminal reconciliation must not call LSP") +} + +func TestRepositoryCleanupRetainsBridgeCallersAcrossPurgeFailure(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"native_counter", "lsp_supported", "unknown_counter"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + var base graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(t.TempDir() + "/graph.sqlite") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }) + base = db + } + store := &bridgeRetryCleanupStore{Store: base} + source, target := "consumer/src/lib.rs::run", "provider/src/lib.rs::helper" + store.AddBatch([]*graph.Node{{ID: "consumer/src/lib.rs", Name: "lib.rs", FilePath: "consumer/src/lib.rs", Kind: graph.KindFile, RepoPrefix: "consumer", Language: "rust"}, {ID: source, Name: "run", FilePath: "consumer/src/lib.rs", Kind: graph.KindFunction, RepoPrefix: "consumer", Language: "rust"}, {ID: "provider/src/lib.rs", Name: "lib.rs", FilePath: "provider/src/lib.rs", Kind: graph.KindFile, RepoPrefix: "provider", Language: "rust"}, {ID: target, Name: "helper", FilePath: "provider/src/lib.rs", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}}, []*graph.Edge{{From: source, FilePath: "consumer/src/lib.rs", To: target, Kind: graph.EdgeCalls, Line: 2, CrossRepo: true, Origin: graph.OriginASTResolved, Confidence: 0.85, Meta: map[string]any{"cargo_call_stub": "unresolved::helper"}}}) + var activeStore graph.Store = store + if mode == "unknown_counter" { + activeStore = &bridgeUnknownCounterStore{Store: store} + } + mi := NewMultiIndexer(activeStore, newTestRegistry(), nil, newTestConfigManager(t), zap.NewNop()) + if mode == "lsp_supported" { + mi.resolverLSPHelper = bridgeTerminalLSP{} + } + mi.repos["consumer"] = &RepoMetadata{RepoPrefix: "consumer", RootPath: t.TempDir()} + mi.repos["provider"] = &RepoMetadata{RepoPrefix: "provider", RootPath: t.TempDir()} + if _, _, err := mi.UntrackRepoChecked(t.Context(), "provider"); !errors.Is(err, errInjectedRepositoryPurge) { + t.Fatalf("first untrack = %v", err) + } + if _, _, err := mi.UntrackRepoChecked(t.Context(), "provider"); err != nil { + t.Fatal(err) + } + rows := store.GetOutEdges(source) + if len(rows) != 1 { + t.Fatalf("calls = %d", len(rows)) + } + call := rows[0] + if call.To != "unresolved::helper" || call.CrossRepo || graph.HasRestubProvenance(call) || (call.Meta["resolve_terminal"] == true) != (mode == "native_counter") { + t.Fatalf("unsettled caller after purge retry: %s", builderRenderEdge(call)) + } + settled := builderRenderEdge(call) + if _, _, err := mi.UntrackRepoChecked(t.Context(), "provider"); err != nil { + t.Fatal(err) + } + if got := builderRenderEdge(store.GetOutEdges(source)[0]); got != settled { + t.Fatalf("no-change untrack altered caller: %s", got) + } + }) + } + } +} diff --git a/internal/indexer/workspace_resolve.go b/internal/indexer/workspace_resolve.go index 1a88aac47..6ef795a73 100644 --- a/internal/indexer/workspace_resolve.go +++ b/internal/indexer/workspace_resolve.go @@ -4,6 +4,7 @@ import ( "context" "path/filepath" "sort" + "strings" "sync" "sync/atomic" @@ -497,6 +498,10 @@ func (mi *MultiIndexer) newCrossRepoResolver() *resolver.CrossRepoResolver { cr.SetNpmAliasResolver(mi.npmAliasResolver()) cr.SetNpmDependencyLookup(mi.npmDependencyLookup()) cr.SetNpmPackageLookup(mi.npmPackageLookup()) + cr.SetCargoCrateLookup(mi.cargoCrateLookup()) + if mi.IsMultiRepo() { + cr.SetPythonPackageLookup(mi.pythonPackageLookup) + } cr.SetPathAliasResolver(mi.pathAliasResolver()) cr.SetWorkspaceMembership(mi.workspaceMembershipResolver()) mi.applyRemoteStitch(cr) @@ -509,7 +514,7 @@ func (mi *MultiIndexer) runCrossRepoResolve(reconcileContracts bool) { } } -func (mi *MultiIndexer) runCrossRepoResolveContext(ctx context.Context, reconcileContracts bool) error { +func (mi *MultiIndexer) runCrossRepoResolveContext(ctx context.Context, reconcileContracts bool, cargoRetargetRepo ...string) error { cr := mi.newCrossRepoResolver() if cr == nil { return nil @@ -527,6 +532,7 @@ func (mi *MultiIndexer) runCrossRepoResolveContext(ctx context.Context, reconcil } mi.synthesizeDeferredGoModuleCalls(idx, edges) } + cr.SetCargoImportRetargetRepo(cargoRetargetRepo...) if _, err := cr.ResolveAllContext(ctx); err != nil { return err } @@ -544,15 +550,11 @@ func (mi *MultiIndexer) runCrossRepoResolveContext(ctx context.Context, reconcil // for one coordinated mutation frontier. Its caller owns the repository lane // and global topology-writer gate, so the graph cannot expose a half-resolved // batch between same-repo, semantic, and cross-repository phases. -func (mi *MultiIndexer) runCrossRepoResolveFiles(filePaths []string) { - mi.runCrossRepoResolveMutationFiles(filePaths, filePaths) -} - -func (mi *MultiIndexer) runCrossRepoResolveMutationFiles(resolutionFiles, materializationFiles []string) { - mi.runCrossRepoResolveMutationFrontiers(resolutionFiles, materializationFiles, materializationFiles) +func (mi *MultiIndexer) runCrossRepoResolveFiles(filePaths []string, cargoRetargetRepo ...string) { + mi.runCrossRepoResolveMutationFrontiers(filePaths, filePaths, filePaths, nil, cargoRetargetRepo...) } -func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles []string) { +func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles, masterResolvedFiles []string, cargoRetargetRepo ...string) { if len(resolutionFiles) == 0 && len(edgeSourceFiles) == 0 && len(definitionFiles) == 0 { return } @@ -560,9 +562,48 @@ func (mi *MultiIndexer) runCrossRepoResolveMutationFrontiers(resolutionFiles, ed if cr == nil { return } - cr.ResolveMutationFrontiers(resolutionFiles, edgeSourceFiles, definitionFiles) + cr.SetCargoImportRetargetRepo(cargoRetargetRepo...) + var bridgeFiles []string + for _, file := range resolutionFiles { + if strings.HasSuffix(file, ".rs") || strings.HasSuffix(file, ".py") || strings.HasSuffix(file, ".pyi") { + bridgeFiles = append(bridgeFiles, file) + } + } + // Match cold ordering: tracked-provider refusal must remain authoritative + // after fallback has handled genuinely external calls. + prepared := make(map[string]bool, len(masterResolvedFiles)) + for _, file := range masterResolvedFiles { + prepared[file] = true + } + var outgoingFiles []string + for _, file := range bridgeFiles { + if !prepared[file] { + outgoingFiles = append(outgoingFiles, file) + } + } + if len(outgoingFiles) > 0 { + if master := mi.newMasterResolver(false); master != nil { + master.ResolveFilesOutgoing(outgoingFiles) + mi.reconcileRetargetedTestCalls(master.TakeRetargetedTestCallFiles()) + } + } + if len(bridgeFiles) > 0 { + mi.synthesizeDemotedGoModuleCalls(bridgeFiles) + } + _, admission := cr.ResolveMutationFrontiersBounded(resolutionFiles, edgeSourceFiles, definitionFiles) + if !admission.IncomingRefused && len(bridgeFiles) > 0 { + if master := mi.newMasterResolver(true); master != nil { + master.ReconcileBridgeCallerTerminals(bridgeFiles) + } + } mi.finishGoModuleCalls(cr) mi.reconcileRetargetedTestCalls(cr.TakeRetargetedTestCallFiles()) + if admission.Refused && !admission.IncomingRefused { + // The bounded import retry cannot finish this revision. Reuse the + // full fallback before another mutation repeats its identity sweep. + mi.logger.Info("cross-repo resolve: completing deferred Go module imports") + mi.runCrossRepoResolve(false) + } } // crossWorkspaceLookup builds a resolver.CrossWorkspaceDepLookup from @@ -708,7 +749,7 @@ func (mi *MultiIndexer) resolveDeferredGoModules(idx *Indexer) error { } func (mi *MultiIndexer) resolveIncomingGoModules(repo string) error { - if mi.graph == nil || len(mi.graph.RepoPrefixes()) < 2 || mi.trackedGoModules()[repo] == "" { + if mi.graph == nil || len(mi.graph.RepoPrefixes()) < 2 || len(mi.trackedGoModules()) == 0 { return nil } cr := mi.newCrossRepoResolver() diff --git a/internal/parser/languages/python.go b/internal/parser/languages/python.go index 4926d8b37..6402b32a8 100644 --- a/internal/parser/languages/python.go +++ b/internal/parser/languages/python.go @@ -934,9 +934,11 @@ func (e *PythonExtractor) emitImport(m parser.QueryResult, filePath, fileID stri func (e *PythonExtractor) emitImportFrom(m parser.QueryResult, filePath, fileID string, src []byte, result *parser.ExtractionResult, imports map[string]string) { mod := m.Captures["import.module"] pyEmitImportNode(filePath, fileID, mod.Text, "", mod.StartLine+1, result) + bindings := make(map[string]any) result.Edges = append(result.Edges, &graph.Edge{ From: fileID, To: "unresolved::import::" + mod.Text, Kind: graph.EdgeImports, FilePath: filePath, Line: mod.StartLine + 1, + Meta: map[string]any{"python_bindings": bindings}, }) def, ok := m.Captures["importfrom.def"] @@ -963,6 +965,7 @@ func (e *PythonExtractor) emitImportFrom(m parser.QueryResult, filePath, fileID continue } imports[name] = mod.Text + "." + name + bindings[name] = name case "aliased_import": var importedName, alias string for j, _nc := 0, int(child.NamedChildCount()); j < _nc; j++ { @@ -978,6 +981,7 @@ func (e *PythonExtractor) emitImportFrom(m parser.QueryResult, filePath, fileID continue } imports[alias] = mod.Text + "." + importedName + bindings[alias] = importedName } } } @@ -1041,12 +1045,14 @@ func (e *PythonExtractor) emitImportFromRelative(m parser.QueryResult, filePath, // `from numpy import array` path uses (one import edge to the // originating module) — the only difference is that for `from . // import X` the module name comes from the import list. + var moduleImport *graph.Edge if modPath != "" { pyEmitImportNode(filePath, fileID, modPath, "", importLine, result) - result.Edges = append(result.Edges, &graph.Edge{ + moduleImport = &graph.Edge{ From: fileID, To: "unresolved::pyrel::" + stem, Kind: graph.EdgeImports, FilePath: filePath, Line: importLine, - }) + } + result.Edges = append(result.Edges, moduleImport) } def, ok := m.Captures["importfrom_rel.def"] @@ -1054,9 +1060,23 @@ func (e *PythonExtractor) emitImportFromRelative(m parser.QueryResult, filePath, return } stmt := def.Node + // Keep exact package bindings for the tracked-import call bridge. Ordinary + // imports and function-local bindings do not expose package attributes. + var exports map[string]any + if moduleImport != nil && strings.HasSuffix(filePath, "/__init__.py") && stmt.Parent().Type() == "module" { + exports = make(map[string]any) + moduleImport.Meta = map[string]any{"python_reexports": exports, "python_reexport_stem": stem} + } moduleSeen := false emittedSub := map[string]bool{} emitSubmodule := func(name, alias string) { + if exports != nil && name != "" { + binding := alias + if binding == "" { + binding = name + } + exports[binding] = name + } if modPath != "" || name == "" { return } diff --git a/internal/parser/languages/python_a22_relative_test.go b/internal/parser/languages/python_a22_relative_test.go index 60864906b..bdcd2a13c 100644 --- a/internal/parser/languages/python_a22_relative_test.go +++ b/internal/parser/languages/python_a22_relative_test.go @@ -120,3 +120,34 @@ func TestPyResolveRelativeStem_CornerCases(t *testing.T) { "filePath=%q dots=%d mod=%q", c.filePath, c.dots, c.mod) } } + +func TestPythonPackageReExportMetadata(t *testing.T) { + for _, tc := range []struct { + name, file, source string + want map[string]any + }{ + {"plain and alias", "pkg/__init__.py", "from .impl import Plain, Original as Public\n", map[string]any{"Plain": "Plain", "Public": "Original"}}, + {"ordinary module", "pkg/app.py", "from .impl import Plain\n", nil}, + {"function local", "pkg/__init__.py", "def local():\n from .impl import Plain\n", nil}, + {"conditional", "pkg/__init__.py", "if flag:\n from .impl import Plain\n", nil}, + {"wildcard", "pkg/__init__.py", "from .impl import *\n", map[string]any{}}, + } { + t.Run(tc.name, func(t *testing.T) { + result, err := NewPythonExtractor().Extract(tc.file, []byte(tc.source)) + require.NoError(t, err) + var imports []*graph.Edge + for _, edge := range result.Edges { + if edge.Kind == graph.EdgeImports { + imports = append(imports, edge) + } + } + require.Len(t, imports, 1) + if tc.want == nil { + require.NotContains(t, imports[0].Meta, "python_reexports") + } else { + require.Equal(t, tc.want, imports[0].Meta["python_reexports"]) + require.Equal(t, "pkg/impl", imports[0].Meta["python_reexport_stem"]) + } + }) + } +} diff --git a/internal/parser/languages/python_test.go b/internal/parser/languages/python_test.go index 21a0d0c24..72a91d71e 100644 --- a/internal/parser/languages/python_test.go +++ b/internal/parser/languages/python_test.go @@ -515,3 +515,11 @@ func pyNodeIDs(nodes []*graph.Node) []string { } return out } + +func TestPyExtractor_ImportedReceiverBindings(t *testing.T) { + result, err := NewPythonExtractor().Extract("app.py", []byte("from pkg.image import Image as Picture, Other\n")) + require.NoError(t, err) + imports := edgesOfKind(result.Edges, graph.EdgeImports) + require.Len(t, imports, 1) + require.Equal(t, map[string]any{"Picture": "Image", "Other": "Other"}, imports[0].Meta["python_bindings"]) +} diff --git a/internal/resolver/bridge_import_reachability_test.go b/internal/resolver/bridge_import_reachability_test.go new file mode 100644 index 000000000..d5a594d2b --- /dev/null +++ b/internal/resolver/bridge_import_reachability_test.go @@ -0,0 +1,350 @@ +package resolver + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestTrackedBridgeImportsDoNotAuthorizeBareNames(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, tc := range []struct{ language, file, provider, marker, spec string }{ + {"go", "main.go", "lib.go", "go_import", "example.org/lib"}, + {"rust", "src/lib.rs", "src/lib.rs", CargoImportStubMeta, "external::library/action"}, + {"python", "app.py", "pkg/image.py", PythonImportStubMeta, "external::pkg.image"}, + } { + t.Run(backend+"/"+tc.language, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file, provider := "consumer/"+tc.file, "provider/"+tc.provider + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "consumer", Language: tc.language}) + g.AddNode(&graph.Node{ID: file + "::run", Name: "run", Kind: graph.KindFunction, FilePath: file, RepoPrefix: "consumer", Language: tc.language}) + g.AddNode(&graph.Node{ID: provider, Kind: graph.KindFile, FilePath: provider, RepoPrefix: "provider", Language: tc.language}) + // Neither imported module exports hidden. Two other modules do. + for _, decoy := range []string{"provider/other", "provider/decoy"} { + g.AddNode(&graph.Node{ID: decoy + "::hidden", Name: "hidden", Kind: graph.KindFunction, FilePath: decoy, RepoPrefix: "provider", Language: tc.language}) + g.AddNode(&graph.Node{ID: decoy + "::Client.Close", Name: "Close", Kind: graph.KindMethod, FilePath: decoy, RepoPrefix: "provider", Language: tc.language, Meta: map[string]any{"receiver": "Client"}}) + } + g.AddEdge(&graph.Edge{From: file, To: provider, Kind: graph.EdgeImports, FilePath: file, Line: 1, CrossRepo: true, Meta: map[string]any{tc.marker: tc.spec}}) + // A foreign-typed resp.Body.Close() has no evidence for Client.Close. + for i, target := range []string{"unresolved::hidden", "unresolved::*.Close"} { + g.AddEdge(&graph.Edge{From: file + "::run", To: target, Kind: graph.EdgeCalls, FilePath: file, Line: i + 2, Meta: map[string]any{"receiver_type": "io.ReadCloser"}}) + } + cr := NewCrossRepo(g) + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + calls := outEdgesOfKind(g, file+"::run", graph.EdgeCalls) + require.Len(t, calls, 2) + for _, call := range calls { + assert.True(t, graph.IsUnresolvedTarget(call.To), "unimported symbol must stay unresolved: %s", call.To) + assert.False(t, call.CrossRepo) + } + }) + } + } +} + +func TestCargoCallsStayWithinImportedItem(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scenario := range []string{"function", "glob", "constructor", "method", "foreign_type", "ambiguous"} { + t.Run(backend+"/"+scenario, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file, imported := "consumer/src/lib.rs", "provider/src/util.rs" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "rust"}) + kind, name, target, item := graph.KindFunction, "action", "unresolved::action", "action" + meta := map[string]any{} + // Given a glob use, the ordinary function call still needs exact file evidence. + if scenario == "glob" { + item = "*" + } + if scenario == "constructor" { + kind, name, target, item = graph.KindMethod, "new", "unresolved::new", "Foo" + meta["rust_path"] = "Foo::new" + } + if scenario == "method" || scenario == "foreign_type" { + kind, name, target, item = graph.KindMethod, "close", "unresolved::*.close", "Foo" + meta["receiver_type"] = "Foo" + if scenario == "foreign_type" { + meta["receiver_type"] = "Foreign" + } + } + // A same-named decoy in another module must never compete with util. + for _, path := range []string{"provider/src/aaa.rs", imported} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: "provider", Language: "rust"}) + g.AddNode(&graph.Node{ID: path + "::" + name, FilePath: path, Name: name, Kind: kind, RepoPrefix: "provider", Language: "rust", Meta: map[string]any{"receiver": "Foo"}}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: imported, Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::library/util/" + item, CargoImportModuleMeta: "library::util"}}) + if scenario == "ambiguous" { + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/src/aaa.rs", Kind: graph.EdgeImports, Line: 2, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::library/aaa/" + item, CargoImportModuleMeta: "library::aaa"}}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeCalls, Line: 3, Meta: meta}) + _, err := NewCrossRepo(g).ResolveAllContext(t.Context()) + require.NoError(t, err) + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 1) + if scenario == "foreign_type" || scenario == "ambiguous" { + require.Equal(t, target, calls[0].To) + require.False(t, calls[0].CrossRepo) + } else { + require.Equal(t, imported+"::"+name, calls[0].To) + require.True(t, calls[0].CrossRepo) + } + }) + } + } +} + +func TestPythonMethodsStayWithinImportedType(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scenario := range []string{"method", "foreign_type", "ambiguous", "local_type", "other_file_type", "alias", "unbound"} { + t.Run(backend+"/"+scenario, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file := "consumer/app.py" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "python"}) + for _, provider := range []string{"provider/pkg/aaa.py", "provider/pkg/image.py"} { + g.AddNode(&graph.Node{ID: provider, FilePath: provider, Kind: graph.KindFile, RepoPrefix: "provider", Language: "python"}) + g.AddNode(&graph.Node{ID: provider + "::Image.show", FilePath: provider, Name: "show", Kind: graph.KindMethod, RepoPrefix: "provider", Language: "python", Meta: map[string]any{"receiver": "Image"}}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/pkg/image.py", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image", "python_bindings": map[string]any{"Image": "Image"}}}) + if scenario == "ambiguous" { + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "provider/pkg/aaa.py", Kind: graph.EdgeImports, Line: 2, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.aaa", "python_import": "pkg.aaa", "python_bindings": map[string]any{"Image": "Image"}}}) + } + // Given a same-named type, only a definition in this caller file shadows the import. + if scenario == "local_type" || scenario == "other_file_type" { + typeFile := file + if scenario == "other_file_type" { + typeFile = "consumer/other.py" + } + g.AddNode(&graph.Node{ID: typeFile + "::Image", Name: "Image", FilePath: typeFile, Kind: graph.KindType, RepoPrefix: "consumer", Language: "python"}) + } + if scenario == "alias" || scenario == "unbound" { + imports := outEdgesOfKind(g, file, graph.EdgeImports) + imports[0].Meta["python_bindings"] = map[string]any{"Picture": "Image"} + g.ReindexEdges([]graph.EdgeReindex{{Edge: imports[0], OldTo: imports[0].To}}) + } + receiver := "Image" + if scenario == "alias" { + receiver = "Picture" + } + if scenario == "foreign_type" { + receiver = "Foreign" + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::*.show", Kind: graph.EdgeCalls, Line: 3, Meta: map[string]any{"receiver_type": receiver}}) + _, err := NewCrossRepo(g).ResolveAllContext(t.Context()) + require.NoError(t, err) + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 1) + if scenario == "method" || scenario == "alias" || scenario == "other_file_type" { + require.Equal(t, "provider/pkg/image.py::Image.show", calls[0].To) + require.True(t, calls[0].CrossRepo) + } else { + require.Equal(t, "unresolved::*.show", calls[0].To) + require.False(t, calls[0].CrossRepo) + } + }) + } + } +} + +type bridgeNameCountStore struct { + graph.Store + broad, scoped int + fileReads int + fileBatchReads int + fileBatchPaths []string +} + +func (s *bridgeNameCountStore) GetFileNodes(file string) []*graph.Node { + s.fileReads++ + return s.Store.GetFileNodes(file) +} + +func (s *bridgeNameCountStore) GetFileNodesByPaths(paths []string) map[string][]*graph.Node { + s.fileBatchReads++ + s.fileBatchPaths = append(s.fileBatchPaths, paths...) + return s.Store.GetFileNodesByPaths(paths) +} + +func (s *bridgeNameCountStore) FindNodesByName(name string) []*graph.Node { + s.broad++ + return s.Store.FindNodesByName(name) +} +func (s *bridgeNameCountStore) FindNodesByNameInRepo(name, repo string) []*graph.Node { + s.scoped++ + return s.Store.FindNodesByNameInRepo(name, repo) +} +func TestBridgeCandidatesLookupOncePerProvider(t *testing.T) { + for _, language := range []string{"rust", "python"} { + t.Run(language, func(t *testing.T) { + g := &bridgeNameCountStore{Store: graph.New()} + cr := NewCrossRepo(g) + file := "consumer/app.py" + name := "show" + target := "*.show" + if language == "rust" { + file = "consumer/src/lib.rs" + name = "action" + target = "action" + } + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: language}) + var imports []*graph.Edge + for i := 0; i < 20; i++ { + provider := fmt.Sprintf("provider/pkg/mod%d.py", i) + marker := map[string]any{PythonImportStubMeta: "external::pkg.mod", "python_import": fmt.Sprintf("pkg.mod%d", i), "python_bindings": map[string]any{"Image": "Image"}} + if language == "rust" { + provider = fmt.Sprintf("provider/src/mod%d.rs", i) + marker = map[string]any{CargoImportStubMeta: "external::library/action", CargoImportModuleMeta: "library"} + } + imports = append(imports, &graph.Edge{From: file, FilePath: file, To: provider, Meta: marker}) + if i == 19 { + kind := graph.KindFunction + if language == "python" { + kind = graph.KindMethod + } + g.AddNode(&graph.Node{ID: provider + "::" + name, FilePath: provider, Name: name, Kind: kind, RepoPrefix: "provider", Language: language, Meta: map[string]any{"receiver": "Image"}}) + } + } + cr.bridgeImportsByFile = map[string][]*graph.Edge{file: imports} + e := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{"receiver_type": "Image"}} + if language == "rust" { + delete(e.Meta, "receiver_type") + require.True(t, cr.resolveCargoImportedCall(e, target)) + } else { + require.True(t, cr.resolvePythonImportedMethod(e, target)) + } + require.Zero(t, g.broad, "bridge lookups must not scan all repositories") + require.Equal(t, 1, g.scoped, "one provider lookup") + }) + } +} + +func TestPythonUnimportedReceiverAvoidsNameLookups(t *testing.T) { + // Given a bridge import that binds a different name, not the call receiver. + g := &bridgeNameCountStore{Store: graph.New()} + file := "consumer/tests.py" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "python"}) + cr := NewCrossRepo(g) + cr.bridgeImportsByFile = map[string][]*graph.Edge{file: {{From: file, FilePath: file, To: "provider/pkg/image.py", Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image", "python_bindings": map[string]any{"Different": "Image"}}}}} + edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{"receiver_type": "TestCase"}} + // When a typed method is considered by the tracked bridge. + require.False(t, cr.resolvePythonImportedMethod(edge, "*.assertEqual")) + // Then it performs no file-node or store name lookup. + require.Zero(t, g.broad) + require.Zero(t, g.scoped) + require.Zero(t, g.fileReads) +} + +func TestPythonReceiverTypesReadOncePerFilePerPass(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scoped := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/scoped=%v", backend, scoped), func(t *testing.T) { + // Given eight calls on one imported receiver in a single file. + g := &bridgeNameCountStore{Store: goImportReviewStore(t, backend)} + file, provider := "consumer/app.py", "provider/pkg/image.py" + for _, path := range []string{file, provider} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "python"}) + } + g.AddNode(&graph.Node{ID: provider + "::Image.show", Name: "show", FilePath: provider, Kind: graph.KindMethod, RepoPrefix: "provider", Language: "python", Meta: map[string]any{"receiver": "Image"}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: provider, Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image", "python_bindings": map[string]any{"Image": "Image"}}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: provider, Kind: graph.EdgeImports, Line: 20, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_import": "pkg.image", "python_bindings": map[string]any{"Different": "Other"}}}) + for i := 0; i < 8; i++ { + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::*.show", Kind: graph.EdgeCalls, Line: i + 2, Meta: map[string]any{"receiver_type": "Image"}}) + } + cr := NewCrossRepo(g) + resolve := func() { + if scoped { + cr.resolveScopedLocked(g.GetOutEdges(file)) + } else { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + } + } + // When a full or scoped pass resolves the calls. + resolve() + // Then local type names are read once, not once per edge. + require.Zero(t, g.fileReads) + require.Equal(t, 1, g.fileBatchReads) + require.Len(t, g.fileBatchPaths, 1) + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 8) + for _, call := range calls { + require.Equal(t, provider+"::Image.show", call.To) + } + // A later pass must refresh the snapshot after a same-file type is added. + g.AddNode(&graph.Node{ID: file + "::Image", Name: "Image", FilePath: file, Kind: graph.KindType, RepoPrefix: "consumer", Language: "python"}) + var batch []graph.EdgeReindex + for _, call := range calls { + old := call.To + call.To = "unresolved::*.show" + call.CrossRepo = false + batch = append(batch, graph.EdgeReindex{Edge: call, OldTo: old}) + } + g.ReindexEdges(batch) + resolve() + require.Zero(t, g.fileReads) + require.Equal(t, 2, g.fileBatchReads) + require.Len(t, g.fileBatchPaths, 2) + for _, call := range outEdgesOfKind(g, file, graph.EdgeCalls) { + require.Equal(t, "unresolved::*.show", call.To) + } + }) + } + } +} + +func TestPythonReceiverTypeSnapshotBatchesFiles(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, count := range []int{0, 6} { + t.Run(fmt.Sprintf("%s/files=%d", backend, count), func(t *testing.T) { + // Given several from-import callers and an unrelated wildcard importer. + g := &bridgeNameCountStore{Store: goImportReviewStore(t, backend)} + provider := "provider/pkg/image.py" + g.AddNode(&graph.Node{ID: provider, FilePath: provider, Kind: graph.KindFile, RepoPrefix: "provider", Language: "python"}) + g.AddEdge(&graph.Edge{From: "consumer/wild.py", FilePath: "consumer/wild.py", To: provider, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_bindings": map[string]any{}}}) + for i := 0; i < count; i++ { + file := fmt.Sprintf("consumer/caller%d.py", i) + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "python"}) + g.AddNode(&graph.Node{ID: file + "::Local", FilePath: file, Kind: graph.KindType, Name: "Local", RepoPrefix: "consumer", Language: "python"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: provider, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg.image", "python_bindings": map[string]any{"Image": "Image"}}}) + } + // When a full pass builds the import evidence index, with no typed-call filter. + cr := NewCrossRepo(g) + cr.buildReachableReposIndex(nil) + // Then qualifying files are fetched in one batch and no point reads. + require.Zero(t, g.fileReads) + expectedBatches := 0 + if count > 0 { + expectedBatches = 1 + } + require.Equal(t, expectedBatches, g.fileBatchReads) + require.Len(t, g.fileBatchPaths, count) + require.Len(t, cr.pythonLocalTypesByFile, count) + for _, types := range cr.pythonLocalTypesByFile { + require.True(t, types["Local"]) + } + }) + } + } +} diff --git a/internal/resolver/cargo_crate.go b/internal/resolver/cargo_crate.go new file mode 100644 index 000000000..955f29098 --- /dev/null +++ b/internal/resolver/cargo_crate.go @@ -0,0 +1,343 @@ +package resolver + +import ( + "context" + "iter" + "path" + "strings" + + "github.com/zzet/gortex/internal/graph" +) + +// CargoImportStubMeta preserves the import's exact target before a tracked bind. +const CargoImportStubMeta = "cargo_import_stub" + +// CargoImportModuleMeta records the module path actually matched by file probing. +const CargoImportModuleMeta = "cargo_import_module" + +// CargoCallStubMeta retains the extractor's function or method target before linking. +const CargoCallStubMeta = "cargo_call_stub" + +// CargoCrate identifies a provider library and its exposed crate name. +type CargoCrate struct { + Root string // graph-path library file + Name string // exposed provider crate name, never the consumer's alias +} + +// CargoCrateLookup returns providers declared by the crate owning callerFile. +// The resolver owns workspace eligibility. +type CargoCrateLookup func(callerFile, crateName string) []CargoCrate + +// SetCargoCrateLookup installs manifest evidence for Rust import/file edges. +func (cr *CrossRepoResolver) SetCargoCrateLookup(lookup CargoCrateLookup) { cr.cargoCrates = lookup } + +// SetCargoImportRetargetRepo refreshes marked bindings only after a provider file is added. +func (cr *CrossRepoResolver) SetCargoImportRetargetRepo(repos ...string) { + cr.cargoRetargetRepos = make(map[string]bool, len(repos)) + for _, repo := range repos { + if repo != "" { + cr.cargoRetargetRepos[repo] = true + } + } +} + +func (cr *CrossRepoResolver) cargoImportTarget(e *graph.Edge, spec string) (*graph.Node, string) { + if cr.cargoCrates == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { + return nil, "" + } + segs := strings.Split(spec, "/") + switch segs[0] { + case "", "crate", "self", "super", "std", "core", "alloc": + return nil, "" + } + for _, seg := range segs { + if seg == ".." || seg == "." || seg == "" { + return nil, "" + } + } + crates := cr.cargoCrates(e.FilePath, segs[0]) + if len(crates) == 0 { + return nil, "" + } + callerRepo, callerWS := cr.callerRepoPrefix(e), cr.callerWorkspaceID(e) + var root *graph.Node + for _, crate := range crates { + candidate := cr.cachedGetNode(crate.Root) + if candidate == nil || candidate.Kind != graph.KindFile || candidate.Language != "rust" { + continue + } + if candidate.RepoPrefix == callerRepo { + return nil, "" // same-repo resolution stays unchanged + } + if !cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(candidate), crate.Name+strings.TrimPrefix(spec, segs[0])) { + continue + } + if root != nil { + return nil, "" // two eligible providers, even if only one has the module + } + root = candidate + } + if root == nil { + return nil, "" + } + // Same longest-prefix module-file probes as resolveRustUseFile, anchored + // at the manifest's library root rather than a guessed src directory. + for n := len(segs); n > 1; n-- { + stem := path.Join(append([]string{path.Dir(root.ID)}, segs[1:n]...)...) + var found *graph.Node + for _, id := range []string{stem + ".rs", stem + "/mod.rs"} { + if node := cr.cachedGetNode(id); node != nil && node.Kind == graph.KindFile && node.Language == "rust" { + if found != nil { + return nil, "" + } + found = node + } + } + if found != nil { + return found, strings.Join(segs[:n], "::") + } + } + return root, segs[0] // item or glob import from the crate itself +} + +// Per-repo resolution converts missing imports to external stubs. Retry just +// those Rust imports before the cross-repo unresolved pass builds reachability. +func (cr *CrossRepoResolver) resolveCargoImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats) (reopened []*graph.Edge, err error) { + if cr.cargoCrates == nil { + return reopened, ctx.Err() + } + // Keep metadata out of the scan. Only declared, tracked crates need hydration. + var eligible []*graph.Edge + for e := range edges { + if err := ctx.Err(); err != nil { + return reopened, err + } + if e == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { + continue + } + spec := cr.cargoImportSpec(e) + if spec == "" && (!e.CrossRepo || !cr.cargoRetargetRepos[graph.RepoPrefixOfID(e.To)]) { + continue + } + if spec != "" { + name, _, _ := strings.Cut(spec, "/") + if len(cr.cargoCrates(e.FilePath, name)) == 0 && !e.CrossRepo { + continue + } + } + eligible = append(eligible, e) + } + // Close the light cursor before hydration or writes; retain metadata for + // only one bounded page, as in the tracked npm bridge. + for start := 0; start < len(eligible); start += resolvePendingPageRows { + if err := ctx.Err(); err != nil { + return reopened, err + } + page := eligible[start:min(start+resolvePendingPageRows, len(eligible))] + sites := make([]graph.EdgeSite, 0, len(page)) + for _, e := range page { + sites = append(sites, graph.EdgeSite{From: e.From, Line: e.Line, Kind: graph.EdgeImports}) + } + full := cr.graph.GetEdgeCandidates(nil, sites) + var batch []graph.EdgeReindex + for _, projected := range page { + for _, e := range full.Site(projected.From, projected.Line, graph.EdgeImports) { + if e.To != projected.To { + continue + } + node, module := cr.cargoImportTarget(e, cr.cargoImportSpec(e)) + if node == nil && e.CrossRepo { + if stub, _ := e.Meta[CargoImportStubMeta].(string); stub != "" { + oldTo := e.To + e.To, e.CrossRepo = stub, false + delete(e.Meta, CargoImportModuleMeta) + clearEdgeTerminal(e) + batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) + projected.To, projected.CrossRepo = e.To, false + stats.Unresolved++ + reopened = append(reopened, cr.restubCargoImportCalls(e.FilePath, oldTo)...) + } + } else if node != nil && node.ID != e.To { + oldTo := e.To + cr.bindCargoImport(e, node, module, stats) + batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) + projected.To, projected.CrossRepo = e.To, e.CrossRepo + } else if node != nil && e.Meta[CargoImportModuleMeta] != module { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + e.Meta[CargoImportModuleMeta] = module + batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: e.To}) + } + } + } + if len(batch) > 0 { + cr.graph.ReindexEdges(batch) + } + } + return reopened, ctx.Err() +} + +func (cr *CrossRepoResolver) cargoImportSpec(e *graph.Edge) string { + if e == nil || e.Kind != graph.EdgeImports || !strings.HasSuffix(e.FilePath, ".rs") { + return "" + } + target := e.To + if e.CrossRepo && cr.cargoRetargetRepos[graph.RepoPrefixOfID(e.To)] { + target, _ = e.Meta[CargoImportStubMeta].(string) + target = strings.Replace(target, "unresolved::import::", "external::", 1) + } + spec, external := strings.CutPrefix(target, "external::") + if !external { + spec, external = strings.CutPrefix(target, "unresolved::import::") + } + if !external { + return "" + } + name, _, _ := strings.Cut(spec, "/") + switch name { + case "", "crate", "self", "super", "std", "core", "alloc": + return "" + } + return spec +} + +func (cr *CrossRepoResolver) bindCargoImport(e *graph.Edge, node *graph.Node, module string, stats *CrossRepoStats) { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + if original, _ := e.Meta[CargoImportStubMeta].(string); original == "" { + e.Meta[CargoImportStubMeta] = e.To + } + e.Meta[CargoImportModuleMeta] = module + e.To, e.CrossRepo = node.ID, true + clearEdgeTerminal(e) + stats.Resolved++ + stats.CrossRepoEdges++ + stats.ByRepo[node.RepoPrefix]++ +} + +// A tracked use authorizes only its item or receiver in the linked module file. +func (cr *CrossRepoResolver) resolveCargoImportedCall(e *graph.Edge, target string) bool { + if e.Kind != graph.EdgeCalls || !strings.HasSuffix(e.FilePath, ".rs") { + return false + } + name := strings.TrimPrefix(target, "*.") + receiver := edgeReceiverType(e) + scoped, _ := e.Meta["rust_path"].(string) + if owner, _, ok := strings.Cut(scoped, "::"); ok { + receiver = owner + } + names := map[string]bool{name: true} + for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { + stub, _ := imp.Meta[CargoImportStubMeta].(string) + spec := strings.TrimPrefix(strings.TrimPrefix(stub, "external::"), "unresolved::import::") + for _, binding := range parseRustUseBindings(spec) { + if receiver == "" && (binding.local == name || imp.Alias == name) { + names[rustUseBindingName(binding.source)] = true + } + } + } + candidates := make(map[string]map[string][]*graph.Node) + for name, nodes := range cr.bridgeCandidates(e, names, false) { + candidates[name] = make(map[string][]*graph.Node) + for _, node := range nodes { + candidates[name][node.FilePath] = append(candidates[name][node.FilePath], node) + } + } + var match *graph.Node + for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { + stub, _ := imp.Meta[CargoImportStubMeta].(string) + if stub == "" { + continue + } + spec := strings.TrimPrefix(strings.TrimPrefix(stub, "external::"), "unresolved::import::") + workspaceSpec := spec + if cr.cargoCrates != nil { + alias, _, _ := strings.Cut(spec, "/") + for _, crate := range cr.cargoCrates(imp.FilePath, alias) { + if graph.RepoPrefixOfID(crate.Root) == graph.RepoPrefixOfID(imp.To) { + workspaceSpec = crate.Name + strings.TrimPrefix(spec, alias) + break + } + } + } + bindings := parseRustUseBindings(spec) + if strings.HasSuffix(spec, "/*") { + bindings = []rustUseBinding{{source: normalizeRustUsePath(spec), local: "*"}} + } + groupBase, _, grouped := strings.Cut(normalizeRustUsePath(spec), "{") + groupBase = strings.TrimSuffix(groupBase, "::") + module, _ := imp.Meta[CargoImportModuleMeta].(string) + for _, binding := range bindings { + split := strings.LastIndex(binding.source, "::") + if split < 0 || module == "" || binding.source[:split] != module { + continue + } + if grouped && binding.source[:split] != groupBase { + continue + } + item, local := rustUseBindingName(binding.source), binding.local + if imp.Alias != "" { + local = imp.Alias + } + lookupName := name + if receiver == "" && local == name { + lookupName = item + } + for _, candidate := range candidates[lookupName][imp.To] { + if candidate.FilePath != imp.To || candidate.Language != "rust" || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), workspaceSpec) { + continue + } + switch candidate.Kind { + case graph.KindFunction: + if strings.HasPrefix(target, "*.") || receiver != "" || (item != "*" && local != name) { + continue + } + case graph.KindMethod: + if receiver == "" || receiver != local || nodeReceiverType(candidate) != item { + continue + } + default: + continue + } + if match != nil && match.ID != candidate.ID { + return false + } + match = candidate + } + } + } + if match == nil { + return false + } + e.To, e.CrossRepo = match.ID, true + e.Confidence, e.Origin = 0.85, graph.OriginASTResolved + return true +} + +// A refused import no longer authorizes calls through its former target file. +func (cr *CrossRepoResolver) restubCargoImportCalls(file, target string) []*graph.Edge { + var batch []graph.EdgeReindex + var reopened []*graph.Edge + for row := range graph.EdgesInScopeSeq(cr.graph, nil, []string{file}, graph.EdgeCalls) { + call := row.Edge + if call == nil || !call.CrossRepo { + continue + } + original, _ := call.Meta[CargoCallStubMeta].(string) + if original == "" || !strings.HasPrefix(call.To, target+"::") { + continue + } + old := call.To + graph.StashRestubProvenance(call) + call.To, call.CrossRepo, call.ConfidenceLabel = original, false, "" + clearEdgeTerminal(call) + reopened = append(reopened, call) + batch = append(batch, graph.EdgeReindex{Edge: call, OldTo: old}) + } + if len(batch) > 0 { + cr.graph.ReindexEdges(batch) + } + return reopened +} diff --git a/internal/resolver/cargo_crate_test.go b/internal/resolver/cargo_crate_test.go new file mode 100644 index 000000000..90c8cc3cc --- /dev/null +++ b/internal/resolver/cargo_crate_test.go @@ -0,0 +1,381 @@ +package resolver + +import ( + "context" + "fmt" + "iter" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestCargoImportUntrackedSkipsNodeReads(t *testing.T) { + g := &resolverBatchCountingStore{Store: graph.New()} + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(_, _ string) []CargoCrate { return nil }) + e := &graph.Edge{From: "app/src/lib.rs", To: "external::serde/Item", Kind: graph.EdgeImports, FilePath: "app/src/lib.rs"} + node, module := cr.cargoImportTarget(e, "serde/Item") + assert.Nil(t, node) + assert.Empty(t, module) + assert.Zero(t, g.getNodeCalls, "unknown crates must be rejected before caller node reads") +} + +type cargoPagingStore struct { + graph.Store + fullScans, lightScans, pages, maxSites int + cursorOpen bool +} + +func (s *cargoPagingStore) EdgesByKind(kind graph.EdgeKind) iter.Seq[*graph.Edge] { + s.fullScans++ + return s.Store.EdgesByKind(kind) +} + +func (s *cargoPagingStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + s.lightScans++ + return func(yield func(*graph.Edge) bool) { + s.cursorOpen = true + defer func() { s.cursorOpen = false }() + for e := range graph.EdgesLightSeq(s.Store, kinds...) { + if !yield(e) { + return + } + } + } +} + +func (s *cargoPagingStore) GetEdgeCandidates(endpoints []graph.EdgeEndpoint, sites []graph.EdgeSite) graph.EdgeCandidateSet { + if s.cursorOpen { + panic("hydration while light cursor is open") + } + s.pages++ + s.maxSites = max(s.maxSites, len(sites)) + return s.Store.GetEdgeCandidates(endpoints, sites) +} + +func TestCargoImportLightPaging(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var store graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store = db + } + g := &cargoPagingStore{Store: store} + const count = resolvePendingPageRows + 1 + edges := make([]*graph.Edge, 0, count+1) + for line := 1; line <= count; line++ { + edges = append(edges, &graph.Edge{From: "app/src/lib.rs", To: "external::other_crate/Item", Kind: graph.EdgeImports, FilePath: "app/src/lib.rs", Line: line, Meta: map[string]any{"marker": "preserved"}}) + } + edges = append(edges, &graph.Edge{From: "app/src/lib.rs", To: "external::serde/Item", Kind: graph.EdgeImports, FilePath: "app/src/lib.rs", Line: count + 1}) + g.AddBatch([]*graph.Node{ + {ID: "app/src/lib.rs", Kind: graph.KindFile, FilePath: "app/src/lib.rs", RepoPrefix: "app", Language: "rust"}, + {ID: "lib/src/lib.rs", Kind: graph.KindFile, FilePath: "lib/src/lib.rs", RepoPrefix: "lib", Language: "rust"}, + }, edges) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(_, name string) []CargoCrate { + if name != "other_crate" { + return nil + } + return []CargoCrate{{Root: "lib/src/lib.rs", Name: "other_crate"}} + }) + stats := &CrossRepoStats{ByRepo: map[string]int{}} + _, err := cr.resolveCargoImports(context.Background(), graph.EdgesLightSeq(g, graph.EdgeImports), stats) + require.NoError(t, err) + assert.Zero(t, g.fullScans) + assert.Equal(t, 1, g.lightScans) + assert.Equal(t, 2, g.pages) + assert.Equal(t, resolvePendingPageRows, g.maxSites) + assert.Equal(t, count, stats.Resolved) + imports := g.GetOutEdges("app/src/lib.rs") + require.Len(t, imports, count+1) + for _, edge := range imports { + if edge.Line == count+1 { + assert.Equal(t, "external::serde/Item", edge.To) + assert.False(t, edge.CrossRepo) + continue + } + assert.Equal(t, "lib/src/lib.rs", edge.To) + assert.True(t, edge.CrossRepo) + assert.Equal(t, "preserved", edge.Meta["marker"]) + } + }) + } +} + +func TestCargoGroupedAliasedUses(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, use := range []string{"other_crate/util/{action, Other}", "other_crate/util/action as act"} { + t.Run(backend+"/"+use, func(t *testing.T) { + g := goImportReviewStore(t, backend) + file, provider := "consumer/src/lib.rs", "provider/src/util.rs" + for _, path := range []string{file, "provider/src/lib.rs", provider} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + g.AddNode(&graph.Node{ID: provider + "::action", Name: "action", FilePath: provider, Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + local := "action" + if strings.Contains(use, " as ") { + local = "act" + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "external::" + use, Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::" + local, Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { + return []CargoCrate{{Root: "provider/src/lib.rs", Name: "other_crate"}} + }) + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + require.Equal(t, provider, outEdgesOfKind(g, file, graph.EdgeImports)[0].To) + require.Equal(t, provider+"::action", outEdgesOfKind(g, file, graph.EdgeCalls)[0].To) + }) + } + } +} + +func TestScopedBridgeImportHydration(t *testing.T) { + g := &cargoPagingStore{Store: graph.New()} + for i := 0; i < 12; i++ { + file := fmt.Sprintf("app/%d.rs", i) + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "app", Language: "rust"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "lib/src/lib.rs", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: "external::other/action"}}) + } + g.AddNode(&graph.Node{ID: "lib/src/lib.rs", FilePath: "lib/src/lib.rs", Kind: graph.KindFile, RepoPrefix: "lib", Language: "rust"}) + g.AddEdge(&graph.Edge{From: "app/0.rs", FilePath: "app/0.rs", To: "unresolved::action", Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.ResolveForFile("app", "0.rs") + require.Equal(t, 1, g.maxSites, "only the pending caller's import metadata may be hydrated") + g.maxSites = 0 + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + require.Equal(t, 12, g.maxSites, "full passes retain corpus-wide import evidence") +} + +func TestCargoGroupedUsesRejectNestedPathDecoys(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, spec := range []string{"other/util/{action, sub/helper}", "other/{util/action, net/fetch}", "other/{self, action}"} { + t.Run(backend+"/"+spec, func(t *testing.T) { + // Given a real nested item and a same-named decoy in the group's linked file. + g := goImportReviewStore(t, backend) + file, root, module := "consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs" + paths := []string{file, root, module, "provider/src/util/sub.rs", "provider/src/net.rs"} + for _, path := range paths { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + decoy, name := module, "helper" + if strings.HasPrefix(spec, "other/{") { + decoy, name = root, "fetch" + } + if strings.Contains(spec, "{self,") { + name = "action" + } + for _, path := range []string{decoy, "provider/src/util/sub.rs", "provider/src/net.rs"} { + g.AddNode(&graph.Node{ID: path + "::" + name, FilePath: path, Name: name, Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + } + g.AddNode(&graph.Node{ID: module + "::action", FilePath: module, Name: "action", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "external::" + spec, Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::" + name, Kind: graph.EdgeCalls, Line: 2}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::action", Kind: graph.EdgeCalls, Line: 3}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return []CargoCrate{{Root: root, Name: "other"}} }) + // When the tracked bridge resolves the grouped import and its calls. + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + // Then the nested leaf never binds to a decoy in that file. + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 2) + for _, call := range calls { + if call.Line == 2 { + if strings.Contains(spec, "{self,") { + require.Equal(t, root+"::action", call.To) + } else { + require.Equal(t, "unresolved::"+name, call.To) + require.False(t, call.CrossRepo) + } + } + if call.Line == 3 && decoy == module { + require.Equal(t, module+"::action", call.To) + } + } + }) + } + } +} + +func TestScopedCallerFileUsesWarmedNodeCache(t *testing.T) { + // Given many pending edges with one caller node. + g := &resolverBatchCountingStore{Store: graph.New()} + file, caller := "app/file.go", "app/file.go::run" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "app", Language: "go"}) + g.AddNode(&graph.Node{ID: caller, FilePath: file, Kind: graph.KindFunction, RepoPrefix: "app", Language: "go"}) + var edges []*graph.Edge + for i := 0; i < 30; i++ { + edges = append(edges, &graph.Edge{From: caller, FilePath: file, To: "unresolved::missing", Kind: graph.EdgeCalls, Line: i + 1}) + } + g.AddBatch(nil, edges) + // When the scoped resolver builds its pending caller-file set. + cr := NewCrossRepo(g) + cr.resolveScopedLocked(edges) + // Then the batched cache, not one GetNode per edge, supplies caller placement. + require.Zero(t, g.getNodeCalls) +} + +func TestCargoCallRequiresExactParentModule(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, spec := range []string{"other/sys/{open}", "other/sys/open", "other/util/sys/{open}", "other/util/sys/open", "other/util/{open}", "other/util/open"} { + t.Run(backend+"/"+spec, func(t *testing.T) { + // Given a missing sys module and decoys in its ancestor files. + g := goImportReviewStore(t, backend) + file, root, module := "consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs" + for _, path := range []string{file, root, module} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + for _, path := range []string{root, module} { + g.AddNode(&graph.Node{ID: path + "::open", FilePath: path, Name: "open", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "external::" + spec, Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "unresolved::open", Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return []CargoCrate{{Root: root, Name: "other"}} }) + // When the import falls back to the nearest indexed module file. + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + // Then only an exact parent module authorizes its member call. + imp := outEdgesOfKind(g, file, graph.EdgeImports)[0] + matched := "other::util" + if strings.HasPrefix(spec, "other/sys/") { + matched = "other" + } + require.Equal(t, matched, imp.Meta[CargoImportModuleMeta]) + call := outEdgesOfKind(g, file, graph.EdgeCalls)[0] + if strings.Contains(spec, "/sys/") { + require.Equal(t, "unresolved::open", call.To) + require.False(t, call.CrossRepo) + } else { + require.Equal(t, module+"::open", call.To) + require.True(t, call.CrossRepo) + } + }) + } + } +} + +func TestCargoAliasedCallsWorkspacePolicy(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, allowed := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/allowed=%v", backend, allowed), func(t *testing.T) { + g := goImportReviewStore(t, backend) + for _, file := range []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs"} { + ws := "private" + if strings.HasPrefix(file, "consumer/") { + ws = "shared" + } + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), WorkspaceID: ws, Language: "rust"}) + } + g.AddNode(&graph.Node{ID: "provider/src/util.rs::helper", Name: "helper", FilePath: "provider/src/util.rs", Kind: graph.KindFunction, RepoPrefix: "provider", WorkspaceID: "private", Language: "rust"}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "external::oc/util/helper", Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "unresolved::helper", Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(_, alias string) []CargoCrate { + if alias != "oc" { + return nil + } + return []CargoCrate{{Root: "provider/src/lib.rs", Name: "other_crate"}} + }) + module := "oc" + if allowed { + module = "other_crate" + } + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { + return []CrossWorkspaceDepRule{{Workspace: "private", Modules: []string{module}}} + }) + for pass := 0; pass < 2; pass++ { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + imports := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeImports) + require.Len(t, imports, 1) + require.Equal(t, allowed, imports[0].CrossRepo) + calls := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeCalls) + require.Len(t, calls, 1) + require.Equal(t, allowed, calls[0].CrossRepo) + if allowed { + require.Equal(t, "provider/src/util.rs::helper", calls[0].To) + } + } + }) + } + } +} + +func TestCargoUnresolvedImportAndCallSinglePass(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + for _, file := range []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "rust"}) + } + g.AddNode(&graph.Node{ID: "provider/src/util.rs::helper", Name: "helper", FilePath: "provider/src/util.rs", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "unresolved::import::other_crate/util/helper", Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "unresolved::helper", Kind: graph.EdgeCalls, Line: 2}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { + return []CargoCrate{{Root: "provider/src/lib.rs", Name: "other_crate"}} + }) + for pass := 0; pass < 2; pass++ { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + rows := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeCalls) + require.Len(t, rows, 1) + require.Equal(t, "provider/src/util.rs::helper", rows[0].To) + require.True(t, rows[0].CrossRepo) + } + }) + } +} + +func TestCargoImportAuthorizedCallEvidence(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"function", "path_constructor", "wildcard_method"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + g := goImportReviewStore(t, backend) + file, provider := "consumer/src/lib.rs", "provider/src/lib.rs" + for _, id := range []string{file, provider} { + g.AddNode(&graph.Node{ID: id, FilePath: id, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(id), Language: "rust"}) + } + name, item, target, kind := "helper", "helper", "unresolved::helper", graph.KindFunction + meta := map[string]any{} + if mode != "function" { + item, kind = "Foo", graph.KindMethod + name = "new" + target = "unresolved::new" + meta["rust_path"] = "Foo::new" + } + if mode == "wildcard_method" { + name, target = "go", "unresolved::*.go" + delete(meta, "rust_path") + meta["receiver_type"] = "Foo" + } + g.AddNode(&graph.Node{ID: provider + "::" + name, Name: name, FilePath: provider, Kind: kind, RepoPrefix: "provider", Language: "rust", Meta: map[string]any{"receiver": "Foo"}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "external::other/" + item, Kind: graph.EdgeImports, Line: 1}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeCalls, Line: 2, Meta: meta}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return []CargoCrate{{Root: provider, Name: "other"}} }) + for pass := 0; pass < 2; pass++ { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + rows := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, rows, 1) + require.Equal(t, provider+"::"+name, rows[0].To) + require.Equal(t, 0.85, rows[0].Confidence) + require.Equal(t, graph.OriginASTResolved, rows[0].Origin) + } + }) + } + } +} diff --git a/internal/resolver/cargo_import_retarget_test.go b/internal/resolver/cargo_import_retarget_test.go new file mode 100644 index 000000000..b7e68bb3a --- /dev/null +++ b/internal/resolver/cargo_import_retarget_test.go @@ -0,0 +1,226 @@ +package resolver + +import ( + "context" + "fmt" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +type cargoRetargetWriteStore struct { + graph.Store + writes int +} + +func (s *cargoRetargetWriteStore) ReindexEdges(batch []graph.EdgeReindex) { + s.writes += len(batch) + s.Store.ReindexEdges(batch) +} + +func TestCargoRetargetOnlyMarkedProviderImports(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scenario := range []string{"unresolved_marker", "external_marker", "unmarked", "other_provider", "already_correct", "already_correct_legacy", "disabled", "light_projection", "multiple_enabled", "nil_input", "wrong_kind"} { + t.Run(backend+"/"+scenario, func(t *testing.T) { + var native graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + native = db + } + g := &cargoRetargetWriteStore{Store: native} + file, root, module := "consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs" + for _, path := range []string{file, root, module, "other/src/lib.rs"} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), WorkspaceID: "shared", Language: "rust"}) + } + original := "unresolved::import::other_crate/util" + if scenario == "external_marker" { + original = "external::other_crate/util" + } + edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeImports, To: root, CrossRepo: true, Line: 1, + Meta: map[string]any{CargoImportStubMeta: original}} + if scenario == "unmarked" { + edge.Meta = nil + } + if scenario == "other_provider" { + edge.To = "other/src/lib.rs" + } + if scenario == "already_correct" || scenario == "already_correct_legacy" { + edge.To = module + if scenario == "already_correct" { + edge.Meta[CargoImportModuleMeta] = "other_crate::util" + } + } + priorTarget := edge.To + g.AddEdge(edge) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return []CargoCrate{{Root: root, Name: "other_crate"}} }) + if scenario == "multiple_enabled" { + cr.SetCargoImportRetargetRepo("other", "provider") + } else if scenario != "disabled" { + cr.SetCargoImportRetargetRepo("provider") + } + stats := &CrossRepoStats{ByRepo: map[string]int{}} + edges := slices.Values([]*graph.Edge{edge}) + switch scenario { + case "light_projection": + edges = graph.EdgesLightSeq(g, graph.EdgeImports) + case "nil_input": + edges = slices.Values([]*graph.Edge{nil}) + case "wrong_kind": + projected := *edge + projected.Kind = graph.EdgeCalls + edges = slices.Values([]*graph.Edge{&projected}) + } + _, err := cr.resolveCargoImports(context.Background(), edges, stats) + require.NoError(t, err) + imports := outEdgesOfKind(g, file, graph.EdgeImports) + require.Len(t, imports, 1) + changed := scenario == "unresolved_marker" || scenario == "external_marker" || scenario == "light_projection" || scenario == "multiple_enabled" || scenario == "already_correct_legacy" + if changed { + require.Equal(t, module, imports[0].To) + require.Equal(t, 1, g.writes) + } else { + require.Equal(t, priorTarget, imports[0].To) + require.Zero(t, g.writes) + } + if scenario != "unmarked" { + require.Equal(t, original, imports[0].Meta[CargoImportStubMeta]) + } + if changed || scenario == "already_correct" { + require.Equal(t, "other_crate::util", imports[0].Meta[CargoImportModuleMeta]) + } + if scenario == "already_correct_legacy" { + require.Zero(t, stats.Resolved, "refreshing metadata is not a new binding") + } + + priorWrites := g.writes + if scenario == "nil_input" || scenario == "wrong_kind" { + require.Zero(t, stats.Resolved) + return + } + _, err = cr.resolveCargoImports(context.Background(), slices.Values(imports), stats) + require.NoError(t, err) + require.Equal(t, priorWrites, g.writes, "an unchanged native match must not reindex") + }) + } + } +} + +func TestCargoRevisitedImportRestoresStubOnRefusal(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, reason := range []string{"undeclared", "ambiguous"} { + for _, stub := range []string{"external::other_crate/util", "unresolved::import::other_crate/util"} { + t.Run(backend+"/"+reason+"/"+stub, func(t *testing.T) { + g := goImportReviewStore(t, backend) + for _, file := range []string{"consumer/src/lib.rs", "provider/src/lib.rs", "provider/src/util.rs", "fork/src/lib.rs"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "rust"}) + } + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "provider/src/util.rs", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{CargoImportStubMeta: stub, CargoImportModuleMeta: "other_crate::util"}}) + g.AddEdge(&graph.Edge{From: "consumer/src/lib.rs", FilePath: "consumer/src/lib.rs", To: "provider/src/util.rs::action", Kind: graph.EdgeCalls, Line: 2, CrossRepo: true, Meta: map[string]any{CargoCallStubMeta: "unresolved::action"}}) + cr := NewCrossRepo(g) + cr.SetCargoImportRetargetRepo("provider") + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { + if reason == "undeclared" { + return nil + } + return []CargoCrate{{Root: "provider/src/lib.rs", Name: "other_crate"}, {Root: "fork/src/lib.rs", Name: "other_crate"}} + }) + _, err := cr.resolveCargoImports(t.Context(), graph.EdgesLightSeq(g, graph.EdgeImports), &CrossRepoStats{ByRepo: map[string]int{}}) + require.NoError(t, err) + imports := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeImports) + require.Len(t, imports, 1) + require.Equal(t, stub, imports[0].To) + require.False(t, imports[0].CrossRepo) + require.NotContains(t, imports[0].Meta, CargoImportModuleMeta) + calls := outEdgesOfKind(g, "consumer/src/lib.rs", graph.EdgeCalls) + require.Len(t, calls, 1) + require.Equal(t, "unresolved::action", calls[0].To, "warm call must equal a cold unbound call") + require.False(t, calls[0].CrossRepo) + }) + } + } + } +} + +func TestCargoRefusedImportWarmMatchesCold(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scoped := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/scoped=%v", backend, scoped), func(t *testing.T) { + targets := []string{} + for _, warm := range []bool{false, true} { + g := goImportReviewStore(t, backend) + file, provider := "consumer/src/lib.rs", "provider/src/util.rs" + for _, path := range []string{file, "provider/src/lib.rs", provider} { + g.AddNode(&graph.Node{ID: path, FilePath: path, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(path), Language: "rust"}) + } + g.AddNode(&graph.Node{ID: provider + "::action", FilePath: provider, Name: "action", Kind: graph.KindFunction, RepoPrefix: "provider", Language: "rust"}) + imp, call := "external::other_crate/util/action", "unresolved::action" + if warm { + imp, call = provider, provider+"::action" + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: imp, Kind: graph.EdgeImports, Line: 1, CrossRepo: warm, Meta: map[string]any{CargoImportStubMeta: "external::other_crate/util/action"}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: call, Kind: graph.EdgeCalls, Line: 2, CrossRepo: warm, Meta: map[string]any{CargoCallStubMeta: "unresolved::action"}}) + cr := NewCrossRepo(g) + cr.SetCargoCrateLookup(func(string, string) []CargoCrate { return nil }) + cr.SetCargoImportRetargetRepo("provider") + if scoped { + cr.ResolveForFile("consumer", "src/lib.rs") + } else { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + } + edge := outEdgesOfKind(g, file, graph.EdgeCalls)[0] + require.False(t, edge.CrossRepo) + targets = append(targets, edge.To) + } + require.Equal(t, targets[0], targets[1]) + require.Equal(t, "unresolved::action", targets[0]) + }) + } + } +} + +func TestCargoRestubCallsPreservesUnrelatedLinks(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + // Given a revoked import and three marked calls with different binding evidence. + g := goImportReviewStore(t, backend) + file, target := "consumer/src/lib.rs", "provider/src/util.rs" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "rust"}) + for _, row := range []struct { + to string + cross bool + line int + }{ + {target + "::action", true, 1}, + {"other/src/lib.rs::action", true, 2}, + {target + "::local", false, 3}, + } { + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, To: row.to, Line: row.line, CrossRepo: row.cross, Meta: map[string]any{CargoCallStubMeta: "unresolved::action"}}) + } + // When only calls through the revoked target are restubbed. + reopened := NewCrossRepo(g).restubCargoImportCalls(file, target) + // Then the other provider and non-cross-repo call keep their links. + require.Len(t, reopened, 1) + for _, call := range outEdgesOfKind(g, file, graph.EdgeCalls) { + switch call.Line { + case 1: + require.Equal(t, "unresolved::action", call.To) + require.False(t, call.CrossRepo) + case 2: + require.Equal(t, "other/src/lib.rs::action", call.To) + require.True(t, call.CrossRepo) + case 3: + require.Equal(t, target+"::local", call.To) + require.False(t, call.CrossRepo) + } + } + }) + } +} diff --git a/internal/resolver/cross_repo.go b/internal/resolver/cross_repo.go index 4fe26a8da..c6e68afaf 100644 --- a/internal/resolver/cross_repo.go +++ b/internal/resolver/cross_repo.go @@ -111,8 +111,11 @@ type CrossRepoResolver struct { // importedDirsByFile retains the package placement from those same import // rows. A Go call the per-repo package guard refused must not regain a // same-repo name-only binding just because the repository is reachable. - importedDirsByFile map[string]map[string]struct{} - jsTSBindingsByFile map[string]map[string]string + importedDirsByFile map[string]map[string]struct{} + jsTSBindingsByFile map[string]map[string]string + bridgeImportsByFile map[string][]*graph.Edge + pythonReexportsByFile map[string][]*graph.Edge + pythonLocalTypesByFile map[string]map[string]bool // depModuleIndex bridges Go imports to dep:: contract // nodes from the caller's go.mod. Same shape and rationale as // the field of the same name on Resolver — see resolver.go for @@ -131,17 +134,19 @@ type CrossRepoResolver struct { // dangling edge). Off (the default, and the whole-pass-locked path) it is // a no-op: nothing can mutate the graph mid-pass, so every edge and // candidate is live by construction. - validateLiveness bool - crossWorkspaceLookup CrossWorkspaceDepLookup - goModules map[string]string - goModulePaths goModulePathIndex - goModuleReplacements map[string][]GoModuleReplacement - goModuleRetry *GoModuleDepRetry - goModuleRevision uint64 - goModuleDemotedMu sync.Mutex - goModuleDemotedFiles map[string]struct{} - goModuleSourceFiles map[string]struct{} - goModuleSourceSites map[graph.EdgeSite]map[string]struct{} + validateLiveness bool + crossWorkspaceLookup CrossWorkspaceDepLookup + goModules map[string]string + goModulePaths goModulePathIndex + goModuleRepos map[string][]string + goModuleReplacements map[string][]GoModuleReplacement + goModuleRetry *GoModuleDepRetry + goModuleImportsDeferred int + goModuleRevision uint64 + goModuleDemotedMu sync.Mutex + goModuleDemotedFiles map[string]struct{} + goModuleSourceFiles map[string]struct{} + goModuleSourceSites map[graph.EdgeSite]map[string]struct{} // npmAlias rewrites a JS/TS import specifier that matches an // npm-alias dependency key in the importing file's nearest- // ancestor package.json. Same contract as the field of the @@ -153,6 +158,10 @@ type CrossRepoResolver struct { // npm_dependency.go. npmDep NpmDependencyLookup npmPackages NpmPackageLookup + + cargoCrates CargoCrateLookup + cargoRetargetRepos map[string]bool + pythonPackages func() *PythonPackageLookup // pathAlias expands a JS/TS tsconfig/jsconfig path-alias / baseUrl // import specifier to the repo-prefixed file stem it targets. Same // contract as the field of the same name on Resolver — see @@ -393,12 +402,30 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS if err := ctx.Err(); err != nil { return stats, err } + // Whole-graph fallback has no exact provider frontier. Discard package + // projections even if the module revision's terminal retry is complete. + cr.mu.Lock() + if len(cr.goModules) > 0 || cr.goModuleRetry != nil { + state := cr.goModuleImportState() + clear(state.packages) + state.importsSeeded = false // A full pass must include newly published consumers. + } + cr.mu.Unlock() + if err := cr.seedGoModuleImportRegistry(ctx); err != nil { + return stats, err + } cr.mu.Lock() defer cr.mu.Unlock() if err := ctx.Err(); err != nil { return stats, err } - if err := cr.retryGoModuleDeps(ctx, stats); err != nil { + cr.goModuleImportsDeferred = 0 + for repo := range cr.goModules { + if err := cr.restoreGoModuleImports(ctx, repo, stats); err != nil { + return stats, err + } + } + if err := cr.retryGoModuleDeps(ctx, stats, true); err != nil { return stats, err } // Fresh placeholder-source set per pass — same rationale as ResolveAll @@ -410,6 +437,12 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS if err := cr.resolveTrackedNpmImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { return stats, err } + if _, err := cr.resolveCargoImports(ctx, graph.EdgesLightSeq(cr.graph, graph.EdgeImports), stats); err != nil { + return stats, err + } + if _, err := cr.resolveTrackedPythonImports(ctx, cr.trackedPythonImportEdges(), stats, false); err != nil { + return stats, err + } // Share the master resolver's stable high-water/keyset stream. The cold // residual can exceed 200k edges; retaining it plus the cross-repo name, // raw-name, repo and qualified-name caches was the second whole-corpus heap @@ -440,7 +473,7 @@ func (cr *CrossRepoResolver) ResolveAllContext(ctx context.Context) (*CrossRepoS defer cr.clearDirIndexes() cr.buildDepModuleIndex() defer cr.clearDepModuleIndex() - cr.buildReachableReposIndex() + cr.buildReachableReposIndex(nil) defer cr.clearReachableReposIndex() if resolveHotCacheEnabled() { @@ -695,6 +728,21 @@ func (cr *CrossRepoResolver) ResolveForFile(repoPrefix, relPath string) *CrossRe func (cr *CrossRepoResolver) resolveScopedLocked(edges []*graph.Edge) *CrossRepoStats { stats := &CrossRepoStats{ByRepo: make(map[string]int)} _ = cr.resolveTrackedNpmImports(context.Background(), slices.Values(edges), stats) + cargoReopened, _ := cr.resolveCargoImports(context.Background(), slices.Values(edges), stats) + reopened, _ := cr.resolveTrackedPythonImports(context.Background(), slices.Values(edges), stats, true) + reopened = append(reopened, cargoReopened...) + if len(reopened) > 0 { + seen := make(map[*graph.Edge]bool, len(edges)) + for _, edge := range edges { + seen[edge] = true + } + for _, edge := range reopened { + if !seen[edge] { + edges = append(edges, edge) + seen[edge] = true + } + } + } pending := make([]*graph.Edge, 0, len(edges)) for _, edge := range edges { if edge != nil && graph.IsUnresolvedTarget(edge.To) { @@ -705,14 +753,18 @@ func (cr *CrossRepoResolver) resolveScopedLocked(edges []*graph.Edge) *CrossRepo return stats } + cr.warmLookupCache(pending) + defer cr.clearLookupCache() cr.buildDirIndexes() defer cr.clearDirIndexes() cr.buildDepModuleIndex() defer cr.clearDepModuleIndex() - cr.buildReachableReposIndex() + callerFiles := make(map[string]bool) + for _, edge := range pending { + callerFiles[cr.callerFileID(edge)] = true + } + cr.buildReachableReposIndex(callerFiles) defer cr.clearReachableReposIndex() - cr.warmLookupCache(pending) - defer cr.clearLookupCache() var reindexBatch []graph.EdgeReindex for _, edge := range pending { @@ -815,10 +867,14 @@ func (cr *CrossRepoResolver) clearDirIndexes() { // resolves imports — including cross-repo imports, with a precise // import-path match — so by the time this index is built the import // graph is settled enough to be trustworthy evidence. -func (cr *CrossRepoResolver) buildReachableReposIndex() { +func (cr *CrossRepoResolver) buildReachableReposIndex(callerFiles map[string]bool) { idx := make(map[string]map[string]struct{}) dirs := make(map[string]map[string]struct{}) cr.jsTSBindingsByFile = make(map[string]map[string]string) + cr.bridgeImportsByFile = make(map[string][]*graph.Edge) + cr.pythonReexportsByFile = make(map[string][]*graph.Edge) + cr.pythonLocalTypesByFile = make(map[string]map[string]bool) + var pythonFiles []string // Materialise metadata-free import projections and batch-load only target // placement fields. A per-edge point lookup here is a query round-trip per // import on a disk backend, which under the cross-repo pass's import @@ -826,6 +882,9 @@ func (cr *CrossRepoResolver) buildReachableReposIndex() { var imports []*graph.Edge ids := make(map[string]struct{}) for e := range graph.EdgesLightSeq(cr.graph, graph.EdgeImports) { + if callerFiles != nil && !callerFiles[e.From] { + continue + } imports = append(imports, e) if e.To != "" { ids[e.To] = struct{}{} @@ -845,12 +904,12 @@ func (cr *CrossRepoResolver) buildReachableReposIndex() { page := imports[start:min(start+resolvePendingPageRows, len(imports))] var sites []graph.EdgeSite for _, e := range page { - if e.CrossRepo && isJSTSPath(e.From) { + if e.CrossRepo && (isJSTSPath(e.From) || strings.HasSuffix(e.FilePath, ".go") || strings.HasSuffix(e.FilePath, ".rs") || isPythonSourcePath(e.FilePath)) { sites = append(sites, graph.EdgeSite{From: e.From, Line: e.Line, Kind: graph.EdgeImports}) } } - // Markers belong to the persisted import, including imports outside a - // scoped resolution frontier. Hydrate only cross-repo JS/TS imports. + // Full passes retain every import; scoped passes load metadata only + // for the caller files represented by pending edges. full := cr.graph.GetEdgeCandidates(nil, sites) for _, e := range page { for _, candidate := range full.Site(e.From, e.Line, graph.EdgeImports) { @@ -899,7 +958,15 @@ func (cr *CrossRepoResolver) buildReachableReposIndex() { } } } - if npmLinked { + if e.Meta[CargoImportStubMeta] != nil || e.Meta[PythonImportStubMeta] != nil { + cr.bridgeImportsByFile[e.From] = append(cr.bridgeImportsByFile[e.From], e) + } + if bindings, _ := e.Meta["python_bindings"].(map[string]any); isPythonSourcePath(e.FilePath) && e.Meta[PythonImportStubMeta] != nil && len(bindings) > 0 && cr.pythonLocalTypesByFile[e.From] == nil { + cr.pythonLocalTypesByFile[e.From] = make(map[string]bool) + pythonFiles = append(pythonFiles, e.FilePath) + } + // A linked module file is not evidence for every symbol in its repo. + if npmLinked || e.Meta["go_import"] != nil || e.Meta[CargoImportStubMeta] != nil || e.Meta[PythonImportStubMeta] != nil { continue } set := idx[e.From] @@ -910,6 +977,41 @@ func (cr *CrossRepoResolver) buildReachableReposIndex() { set[to.RepoPrefix] = struct{}{} } } + // Publish package exports before ResolveAllContext starts its workers. + // The map and its edge slices are read-only until this index is cleared. + packageIDs := make(map[string]struct{}) + for _, imports := range cr.bridgeImportsByFile { + for _, imp := range imports { + if imp.CrossRepo && strings.HasSuffix(imp.To, "/__init__.py") { + packageIDs[imp.To] = struct{}{} + } + } + } + if len(packageIDs) > 0 { + ids := make([]string, 0, len(packageIDs)) + for id := range packageIDs { + ids = append(ids, id) + } + rows := cr.graph.GetOutEdgesByNodeIDs(ids) + for _, id := range ids { + var reexports []*graph.Edge + for _, edge := range rows[id] { + if edge.Kind == graph.EdgeImports { + reexports = append(reexports, edge) + } + } + cr.pythonReexportsByFile[id] = reexports + } + } + if len(pythonFiles) > 0 { + for file, nodes := range cr.graph.GetFileNodesByPaths(pythonFiles) { + for _, node := range nodes { + if node.Kind == graph.KindType { + cr.pythonLocalTypesByFile[file][node.Name] = true + } + } + } + } cr.reachableReposByFile = idx cr.importedDirsByFile = dirs } @@ -918,6 +1020,9 @@ func (cr *CrossRepoResolver) clearReachableReposIndex() { cr.reachableReposByFile = nil cr.importedDirsByFile = nil cr.jsTSBindingsByFile = nil + cr.bridgeImportsByFile = nil + cr.pythonReexportsByFile = nil + cr.pythonLocalTypesByFile = nil } // A package-guard refusal is stronger than the cross-repo pass's same-repo @@ -1170,6 +1275,35 @@ func (cr *CrossRepoResolver) scopedCandidates(e *graph.Edge, name string) []*gra return out } +// bridgeCandidates loads each name once per relevant provider, not per import. +func (cr *CrossRepoResolver) bridgeCandidates(e *graph.Edge, names map[string]bool, includeReachable bool) map[string][]*graph.Node { + repos := make(map[string]bool) + file := cr.callerFileID(e) + for _, imp := range cr.bridgeImportsByFile[file] { + repos[graph.RepoPrefixOfID(imp.To)] = true + } + if includeReachable { + repos[cr.callerRepoPrefix(e)] = true + for repo := range cr.reachableReposByFile[file] { + repos[repo] = true + } + } + result := make(map[string][]*graph.Node) + for name := range names { + for repo := range repos { + if repo == "" { + continue + } + if byRepo, ok := cr.nodesByNameRepo[name]; ok { + result[name] = append(result[name], byRepo[repo]...) + } else { + result[name] = append(result[name], cr.graph.FindNodesByNameInRepo(name, repo)...) + } + } + } + return result +} + // cachedGetNode consults the per-pass id cache first, falling through to // the store on a miss (positive-only: absence means "not pre-warmed"). func (cr *CrossRepoResolver) cachedGetNode(id string) *graph.Node { @@ -1234,12 +1368,46 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b } switch { + case cr.resolveCargoImportedCall(e, target) || cr.resolvePythonImportedMethod(e, target): + stats.Resolved++ + stats.CrossRepoEdges++ + stats.ByRepo[graph.RepoPrefixOfID(e.To)]++ case strings.HasSuffix(e.FilePath, ".go") && strings.HasPrefix(target, "extern::"): cr.resolveGoExtern(e, strings.TrimPrefix(target, "extern::"), stats) case strings.HasPrefix(target, "import::"): cr.resolveImport(e, strings.TrimPrefix(target, "import::"), stats) case strings.HasPrefix(target, "*."): cr.resolveMethodCall(e, strings.TrimPrefix(target, "*."), stats) + case strings.HasPrefix(target, "extern::") && e.Kind == graph.EdgeCalls && cr.pythonPackages != nil && e.Meta[PythonCallStubMeta] == oldTo: + module, symbol, _ := strings.Cut(strings.TrimPrefix(target, "extern::"), "::") + callerRepo, callerWS := cr.callerRepoPrefix(e), cr.callerWorkspaceID(e) + reachable := cr.reachabilityChecker(e) + importedDirs := cr.importedDirsByFile[cr.callerFileID(e)] + find := func(name string) ([]*graph.Node, error) { + candidates := cr.bridgeCandidates(e, map[string]bool{name: true}, true)[name] + var eligible []*graph.Node + for _, node := range candidates { + if cr.linkedPythonCallCandidate(e, node, module) { + eligible = append(eligible, node) + continue + } + _, imported := importedDirs[filePathDir(node.FilePath)] + if imported && reachable(node.RepoPrefix) && cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(node), module) { + eligible = append(eligible, node) + } + } + return eligible, nil + } + if resolvePythonModuleExtern(e, module, symbol, callerRepo, &ResolveStats{}, find) || cr.resolvePythonPackageReExport(e, module, symbol) { + graph.RestoreRestubProvenance(e) + stats.Resolved++ + if e.CrossRepo { + stats.CrossRepoEdges++ + stats.ByRepo[graph.RepoPrefixOfID(e.To)]++ + } + } else { + stats.Unresolved++ + } case e.Kind == graph.EdgeExtends || e.Kind == graph.EdgeImplements || e.Kind == graph.EdgeComposes: // Type-hierarchy edges never resolve to a function/method. // CrossRepoResolver has no type-only resolution path, and a @@ -1260,9 +1428,30 @@ func (cr *CrossRepoResolver) resolveEdge(e *graph.Edge, stats *CrossRepoStats, b } } + stashed := graph.HasRestubProvenance(e) + if stashed { + graph.RestoreRestubProvenance(e) + } if e.To != oldTo { + clearEdgeTerminal(e) + if e.Kind == graph.EdgeCalls && e.CrossRepo && cr.cargoCrates != nil && strings.HasSuffix(e.FilePath, ".rs") { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + e.Meta[CargoCallStubMeta] = oldTo + } + if e.Kind == graph.EdgeCalls && e.CrossRepo && cr.pythonPackages != nil && isPythonSourcePath(e.FilePath) { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + if original, _ := e.Meta[PythonCallStubMeta].(string); original == "" { + e.Meta[PythonCallStubMeta] = oldTo + } + } *batch = append(*batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) cr.noteRetargetedCall(e) + } else if stashed { + *batch = append(*batch, graph.EdgeReindex{Edge: e, OldTo: oldTo}) } } @@ -1282,7 +1471,7 @@ func (cr *CrossRepoResolver) filterLiveReindex(batch []graph.EdgeReindex) []grap sites = append(sites, graph.EdgeSite{ From: reindex.Edge.From, Line: reindex.Edge.Line, Kind: reindex.Edge.Kind, }) - if !isSyntheticResolveTarget(reindex.Edge.To) { + if reindex.Edge.To != reindex.OldTo && !isSyntheticResolveTarget(reindex.Edge.To) { targetIDs = append(targetIDs, reindex.Edge.To) } } @@ -1304,7 +1493,7 @@ func (cr *CrossRepoResolver) filterLiveReindex(batch []graph.EdgeReindex) []grap if !live { continue } - if !isSyntheticResolveTarget(edge.To) && targets[edge.To] == nil { + if edge.To != reindex.OldTo && !isSyntheticResolveTarget(edge.To) && targets[edge.To] == nil { edge.To = reindex.OldTo continue } @@ -1520,6 +1709,11 @@ func (cr *CrossRepoResolver) resolveImport(e *graph.Edge, importPath string, sta // evidence fallbacks below before deciding that the import is ambiguous. } + if node, module := cr.cargoImportTarget(e, importPath); node != nil { + cr.bindCargoImport(e, node, module, stats) + return + } + // Look for file nodes whose directory matches the import path. Two // inverted indexes (built once per Resolve* pass) replace what used // to be an O(N) scan of the entire graph per import edge. diff --git a/internal/resolver/cross_repo_incremental.go b/internal/resolver/cross_repo_incremental.go index 8727395c7..03e539c32 100644 --- a/internal/resolver/cross_repo_incremental.go +++ b/internal/resolver/cross_repo_incremental.go @@ -2,6 +2,10 @@ package resolver import ( "context" + "maps" + "path" + "slices" + "strings" "go.uber.org/zap" @@ -26,7 +30,7 @@ func (cr *CrossRepoResolver) ResolveMutationFiles(resolutionFiles, materializati } // CrossRepoIncomingAdmission is one cross-repository mutation pass's bounded -// incoming-stub admission: the completeness fact plus the typed refusal that +// incoming-stub and Go-import admission: the completeness fact plus the typed refusal that // produced it (*graph.BoundedLocalizationLimitError, nil when nothing was // refused). // @@ -37,7 +41,8 @@ func (cr *CrossRepoResolver) ResolveMutationFiles(resolutionFiles, materializati // one that refused to admit the incoming work it had. type CrossRepoIncomingAdmission struct { graph.IncomingSourceAdmission - Refusal error + Refusal error + IncomingRefused bool // Separates incoming proof from independent Go import recovery. } // ResolveMutationFrontiers preserves the distinct edge-source and changed- @@ -64,14 +69,52 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed return stats, admission } + providerRepos := make(map[string]bool) + goMutation := false + goManifestMutation := false + for _, files := range [][]string{resolutionFiles, edgeSourceFiles} { + for _, file := range files { + if goModuleManifestPath(file) { + goManifestMutation = true + goMutation = true + } + if strings.HasSuffix(file, ".go") { + goMutation = true + } + } + } + for _, file := range definitionFiles { + if goModuleManifestPath(file) { + goManifestMutation = true + goMutation = true + } + repo := graph.RepoPrefixOfID(file) + if strings.HasSuffix(file, ".go") && !strings.HasSuffix(file, "_test.go") && cr.goModules[repo] != "" { + providerRepos[repo] = true + goMutation = true + } + } + importBudget := newGoModuleImportRecoveryBudget() + // Provider and build-list recovery need the seed before the pass lock. + // Ordinary consumer frontiers keep local resolution ahead of the bridge. + if len(providerRepos) > 0 || goManifestMutation { + if err := cr.seedGoModuleImportRegistry(context.Background()); err != nil { + cr.logger.Warn("cross-repo resolve: seed Go module imports deferred", zap.Error(err)) + } + } cr.mu.Lock() defer cr.mu.Unlock() + cr.goModuleImportsDeferred = 0 + budget := &graph.IncomingSourceBudget{} if len(resolutionFiles) > 0 { frontier := collectIncrementalFileFrontier(cr.graph, resolutionFiles, nil) - admission = CrossRepoIncomingAdmission{ - IncomingSourceAdmission: frontier.incomingAdmission, - Refusal: frontier.incomingRefusal, + admission.Keys += frontier.incomingAdmission.Keys + admission.Inspected += frontier.incomingAdmission.Inspected + admission.Dropped += frontier.incomingAdmission.Dropped + admission.Limit = frontier.incomingAdmission.Limit + if frontier.incomingAdmission.Refused { + admission.Refused, admission.Refusal = true, frontier.incomingRefusal } logIncomingAdmissionRefusal(cr.logger, "cross_repo_preparation", len(frontier.stubKeys), frontier.incomingAdmission, frontier.incomingRefusal) @@ -86,10 +129,30 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed } } } + // The per-repo pass has made missing Rust imports external already. + // Keep outgoing imports for the tracked-crate retry too. + if cr.cargoCrates != nil { + for _, edges := range frontier.outByNode { + for _, edge := range edges { + if cr.cargoImportSpec(edge) != "" { + frontier.pending = append(frontier.pending, edge) + } + } + } + } + if cr.pythonPackages != nil { + for _, edges := range frontier.outByNode { + for _, edge := range edges { + if trackedPythonImportSpec(edge) != "" { + frontier.pending = append(frontier.pending, edge) + } + } + } + } pending := make([]*graph.Edge, 0, len(frontier.pending)) seen := make(map[graph.EdgeIdentity]struct{}, len(frontier.pending)) for _, edge := range frontier.pending { - if edge == nil || !graph.IsUnresolvedTarget(edge.To) && trackedNpmImportSpec(edge) == "" { + if edge == nil || !graph.IsUnresolvedTarget(edge.To) && trackedNpmImportSpec(edge) == "" && cr.cargoImportSpec(edge) == "" && trackedPythonImportSpec(edge) == "" { continue } identity := graph.EdgeIdentityFor(edge) @@ -100,13 +163,27 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed pending = append(pending, edge) } if len(pending) > 0 { - stats = cr.resolveScopedLocked(pending) + resolved := cr.resolveScopedLocked(pending) + stats.Resolved += resolved.Resolved + stats.Unresolved += resolved.Unresolved + stats.CrossRepoEdges += resolved.CrossRepoEdges + for repo, count := range resolved.ByRepo { + stats.ByRepo[repo] += count + } + } + // Preserve ordinary local-file resolution, but count each import only + // once if the module bridge subsequently lifts its dependency fallback. + counted := make(map[graph.EdgeIdentity]bool) + for _, e := range pending { + if e.Kind == graph.EdgeImports { + counted[graph.EdgeIdentityFor(e)] = !graph.IsUnresolvedTarget(e.To) && !strings.HasPrefix(e.To, "external::") + } } + _ = cr.resolveGoModuleImports(context.Background(), resolutionFiles, stats, counted) } if len(definitionFiles) > 0 && len(cr.goModules) > 0 && !admission.Refused { - budget := &graph.IncomingSourceBudget{} - if admission.Inspected > 0 { - if err := budget.Charge(admission.Inspected); err != nil { + if frontierInspected := admission.Inspected; frontierInspected > 0 { + if err := budget.Charge(frontierInspected); err != nil { admission.Refused, admission.Refusal = true, err } } @@ -124,11 +201,48 @@ func (cr *CrossRepoResolver) ResolveMutationFrontiersBounded(resolutionFiles, ed } // A newly tracked module also makes old dep/stdlib terminals eligible. // This indexed sweep runs at most once per module/config revision. - if !admission.Refused { - if err := cr.retryGoModuleDeps(context.Background(), stats); err != nil { + if goMutation && !admission.Refused { + if err := cr.retryGoModuleDeps(context.Background(), stats, false); err != nil { cr.logger.Error("cross-repo resolve: retry Go module terminals", zap.Error(err)) } } + if !goMutation { + cr.retryPendingGoModuleEdges(stats) + } + // Import recovery has its own ceiling: a popular provider must not starve + // incoming calls or the revision-keyed call-terminal retry above. A skipped + // provider stays stale until a full pass; it must not reopen the revision. + for _, repo := range slices.Sorted(maps.Keys(providerRepos)) { + ids := cr.goModuleImportIDs(repo) + if len(ids) > importBudget.Remaining() { + cr.logger.Warn("cross-repo resolve: provider consumers stay stale until next full pass", zap.String("repo", repo), zap.Int("imports", len(ids)), zap.Int("remaining", importBudget.Remaining())) + continue + } + if err := importBudget.Charge(len(ids)); err != nil { + cr.logger.Warn("cross-repo resolve: provider consumers stay stale until next full pass", zap.String("repo", repo), zap.Error(err)) + continue + } + for start := 0; start < len(ids); start += externalCallMutationChunk { + if err := cr.restoreGoModuleImportBatch(context.Background(), repo, ids[start:min(start+externalCallMutationChunk, len(ids))], stats); err != nil { + cr.logger.Error("cross-repo resolve: restore Go module imports", zap.Error(err)) + } + } + } + + admission.IncomingRefused = admission.Refused + if deferred := cr.goModuleImportsDeferred; deferred > 0 { + admission.Refused = true + admission.Dropped += deferred + admission.Limit = graph.MaxIncomingSourceCandidateRows + if admission.Refusal == nil { + admission.Refusal = &graph.BoundedLocalizationLimitError{Resource: "Go module import recovery", Limit: graph.MaxIncomingSourceCandidateRows} + } + if state := cr.goModuleRetry; state != nil { + state.mu.Lock() + state.complete = false + state.mu.Unlock() + } + } DetectCrossRepoEdgesForMutationFiles(cr.graph, edgeSourceFiles, definitionFiles) return stats, admission } @@ -149,3 +263,14 @@ func DetectCrossRepoEdgesForMutationFiles(g graph.Store, edgeSourceFiles, defini } return materializeCrossRepoCandidates(g, crossRepoCandidatesForMutationFiles(g, edgeSourceFiles, definitionFiles)) } + +// Keep the Go build-list inputs aligned with the indexer's generation input +// boundary. Source-only suffix checks miss workspace and vendor changes. +func goModuleManifestPath(file string) bool { + file = strings.ReplaceAll(file, "\\", "/") + switch path.Base(file) { + case "go.mod", "go.sum", "go.work", "go.work.sum": + return true + } + return file == "vendor/modules.txt" || strings.HasSuffix(file, "/vendor/modules.txt") +} diff --git a/internal/resolver/cross_repo_meta_liveness_test.go b/internal/resolver/cross_repo_meta_liveness_test.go new file mode 100644 index 000000000..e62244712 --- /dev/null +++ b/internal/resolver/cross_repo_meta_liveness_test.go @@ -0,0 +1,44 @@ +package resolver + +import ( + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "testing" +) + +func TestCrossRepoChunkedRestubMetadataPersists(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, chunk := range []string{"0", "1"} { + t.Run(backend+"/chunk="+chunk, func(t *testing.T) { + t.Setenv("GORTEX_RESOLVE_CHUNK", chunk) + g := goImportReviewStore(t, backend) + call := &graph.Edge{From: "caller/main.py::run", To: "provider/gone.py::missing", Kind: graph.EdgeCalls, FilePath: "caller/main.py", Line: 2, Confidence: 0.85, Origin: graph.OriginASTResolved} + graph.StashRestubProvenance(call) + call.To = "unresolved::missing" + g.AddBatch([]*graph.Node{{ID: "caller/main.py", Kind: graph.KindFile, FilePath: "caller/main.py", RepoPrefix: "caller", Language: "python"}, {ID: call.From, Name: "run", Kind: graph.KindFunction, FilePath: call.FilePath, RepoPrefix: "caller", Language: "python"}}, []*graph.Edge{call}) + for i := 0; i < 2; i++ { + _, err := NewCrossRepo(g).ResolveAllContext(t.Context()) + require.NoError(t, err) + rows := g.GetOutEdges(call.From) + require.Len(t, rows, 1) + require.Equal(t, "unresolved::missing", rows[0].To) + require.False(t, graph.HasRestubProvenance(rows[0])) + require.Empty(t, rows[0].Origin) + require.Zero(t, rows[0].Confidence) + } + }) + } + } +} + +func TestCrossRepoMetaReindexStillRequiresLiveEdge(t *testing.T) { + g := graph.New() + cr := NewCrossRepo(g) + call := &graph.Edge{From: "a/main.py::run", To: "unresolved::missing", Kind: graph.EdgeCalls, FilePath: "a/main.py", Line: 2} + require.Empty(t, cr.filterLiveReindex([]graph.EdgeReindex{{Edge: call, OldTo: call.To}})) + g.AddEdge(call) + copy := *call + copy.To = "b/gone.py::missing" + require.Empty(t, cr.filterLiveReindex([]graph.EdgeReindex{{Edge: ©, OldTo: call.To}})) + require.Equal(t, call.To, copy.To) +} diff --git a/internal/resolver/export_test.go b/internal/resolver/export_test.go new file mode 100644 index 000000000..bf2bea459 --- /dev/null +++ b/internal/resolver/export_test.go @@ -0,0 +1,26 @@ +package resolver + +import ( + "slices" + + "github.com/zzet/gortex/internal/graph" +) + +// SetGoModuleImportRecoveryBudgetForTest changes only the bounded import budget. +// External resolver tests use it to exercise the real indexer continuation. +func SetGoModuleImportRecoveryBudgetForTest(remaining int) (restore func()) { + previous := newGoModuleImportRecoveryBudget + newGoModuleImportRecoveryBudget = func() *graph.IncomingSourceBudget { + budget := &graph.IncomingSourceBudget{} + if err := budget.Charge(graph.MaxIncomingSourceCandidateRows - remaining); err != nil { + panic(err) + } + return budget + } + return func() { newGoModuleImportRecoveryBudget = previous } +} + +// IsGoModuleBindingRecheckForTest shares the production kind list with recorders. +func IsGoModuleBindingRecheckForTest(kinds []graph.NodeKind) bool { + return slices.Equal(kinds, goModuleBindingRecheckKinds) +} diff --git a/internal/resolver/external_calls.go b/internal/resolver/external_calls.go index 00a86478e..c3c433712 100644 --- a/internal/resolver/external_calls.go +++ b/internal/resolver/external_calls.go @@ -258,6 +258,14 @@ func synthesizeExternalCalls(g graph.Store, collect externalCallCandidateSeq) in // edge on its bookkeeping-string terminal. continue } + // A later tracked Python provider needs every original symbol. + // Leave colliding calls on their distinct terminals rather than + // collapsing them onto the package-only synthetic node. + if stub, _ := e.Meta[PythonCallStubMeta].(string); stub != "" && + ((i > 0 && sameExternalCallDestination(pending[i-1], c)) || + (i+1 < len(pending) && sameExternalCallDestination(c, pending[i+1]))) { + continue + } if canDrop && last != nil && sameExternalCallDestination(*last, c) { converged = append(converged, e) continue diff --git a/internal/resolver/go_module_deferral_integration_test.go b/internal/resolver/go_module_deferral_integration_test.go new file mode 100644 index 000000000..e5c4fb326 --- /dev/null +++ b/internal/resolver/go_module_deferral_integration_test.go @@ -0,0 +1,143 @@ +package resolver_test + +import ( + "context" + "fmt" + "iter" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/resolver" + "github.com/zzet/gortex/internal/search" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +type goDeferralStore struct { + graph.Store + rechecks int + importSeeds int +} + +func (s *goDeferralStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { + if resolver.IsGoModuleBindingRecheckForTest(kinds) { + s.rechecks++ + } + if repos == nil && len(kinds) == 1 && kinds[0] == graph.KindImport { + s.importSeeds++ + } + return graph.RepoNodeIdentitiesSeq(s.Store, repos, kinds...) +} + +func newGoDeferralIndexer(t *testing.T, backend string) (*indexer.MultiIndexer, *goDeferralStore, *observer.ObservedLogs) { + t.Helper() + var store graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store = db + } + g := &goDeferralStore{Store: store} + cm, err := config.NewConfigManager(filepath.Join(t.TempDir(), "config.yaml")) + require.NoError(t, err) + reg := parser.NewRegistry() + reg.Register(languages.NewGoExtractor()) + core, logs := observer.New(zap.InfoLevel) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.New(core)) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + mi.BeginBatch() + for _, repo := range []struct{ name, module string }{{"provider", "example.org/library"}, {"consumer", "example.org/app"}} { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "go.mod"), []byte("module "+repo.module+"\n"), 0600)) + _, err := mi.TrackRepo(config.RepoEntry{Path: root, Name: repo.name, Workspace: "shared"}) + require.NoError(t, err) + } + mi.ResetBatch() + // Persisted graph identities are enough for the restart recovery boundary; + // no source parse or dependency fetch is needed to construct this fixture. + g.AddNode(&graph.Node{ID: "provider/lib.go", FilePath: "provider/lib.go", Name: "lib.go", Kind: graph.KindFile, Language: "go", RepoPrefix: "provider", WorkspaceID: "shared"}) + for i := range 2 { + file := fmt.Sprintf("consumer/f%d.go", i) + g.AddBatch([]*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, Language: "go", RepoPrefix: "consumer", WorkspaceID: "shared"}, + {ID: file + "::import::example.org/library", FilePath: file, Kind: graph.KindImport, Language: "go", RepoPrefix: "consumer", WorkspaceID: "shared", StartLine: 1, Meta: map[string]any{"path": "example.org/library"}}, + }, []*graph.Edge{{From: file, FilePath: file, To: "dep::example.org/library", Kind: graph.EdgeImports, Line: 1}}) + } + g.rechecks = 0 + g.importSeeds = 0 + logs.TakeAll() + return mi, g, logs +} + +func requireGoDeferralImports(t *testing.T, g graph.Store) { + t.Helper() + for i := range 2 { + file := fmt.Sprintf("consumer/f%d.go", i) + require.Contains(t, g.GetOutEdges(file), &graph.Edge{From: file, FilePath: file, To: "provider/lib.go", Kind: graph.EdgeImports, Line: 1, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/library", "go_import_fallback": "dep::example.org/library"}}) + } +} + +func TestGoModuleMutationDeferralRunsFullFallback(t *testing.T) { + t.Cleanup(resolver.SetGoModuleImportRecoveryBudgetForTest(1)) + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, logs := newGoDeferralIndexer(t, backend) + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/f0.go"}, nil)) + require.Equal(t, 1, logs.FilterMessage("Go module import identity recheck deferred to full pass").Len(), "the committed test must exercise the bounded refusal") + require.Equal(t, 1, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len()) + requireGoDeferralImports(t, g) + require.Equal(t, 2, g.rechecks, "one bounded recheck and one full continuation") + require.Equal(t, 2, g.importSeeds, "the bounded revision seeds once and the full continuation reseeds once") + for range 2 { + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{"consumer/f0.go"}, nil)) + } + require.Equal(t, 2, g.rechecks, "later Go saves must not repeat the all-repository recheck") + require.Equal(t, 2, g.importSeeds, "later saves must not start another full pass or reseed") + require.Equal(t, 1, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len()) + requireGoDeferralImports(t, g) + }) + } +} + +func TestGoModuleProviderSkipsDoNotRunFullFallback(t *testing.T) { + t.Cleanup(resolver.SetGoModuleImportRecoveryBudgetForTest(1)) + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + mi, g, logs := newGoDeferralIndexer(t, backend) + mi.RunGlobalResolve() + requireGoDeferralImports(t, g) + g.rechecks = 0 + g.importSeeds = 0 + logs.TakeAll() + for _, file := range []string{"provider/lib.go", "provider/lib.go", "consumer/f0.go"} { + require.NoError(t, mi.RunPreEnrichResolveFiles(context.Background(), []string{file}, nil)) + } + require.Equal(t, 2, logs.FilterMessage("cross-repo resolve: provider consumers stay stale until next full pass").Len()) + require.Zero(t, logs.FilterMessage("Go module import identity recheck deferred to full pass").Len()) + require.Zero(t, logs.FilterMessage("cross-repo resolve: completing deferred Go module imports").Len()) + require.Zero(t, g.rechecks, "two provider saves and a third Go save must not recheck all repositories") + require.Zero(t, g.importSeeds, "a full pass always reseeds imports, so no reseed proves no full pass ran") + requireGoDeferralImports(t, g) + }) + } +} + +func TestGoModuleDeferralCounterExactBindingKinds(t *testing.T) { + g := &goDeferralStore{Store: graph.New()} + for range g.RepoNodeIdentitiesSeq(nil, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { + } + require.Equal(t, 1, g.rechecks) + g.rechecks = 0 + for range g.RepoNodeIdentitiesSeq(nil, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindImport) { + } + require.Zero(t, g.rechecks, "four unrelated kinds must not count as the binding recheck") +} diff --git a/internal/resolver/go_module_import_recovery_test.go b/internal/resolver/go_module_import_recovery_test.go new file mode 100644 index 000000000..26be57830 --- /dev/null +++ b/internal/resolver/go_module_import_recovery_test.go @@ -0,0 +1,209 @@ +package resolver + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestGoModuleImportBudgetDoesNotStarveCalls(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, providerFirst := range []bool{false, true} { + for _, restart := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/providerFirst=%v/restart=%v", backend, providerFirst, restart), func(t *testing.T) { + newStore := func() graph.Store { + t.Helper() + if backend == "memory" { + return graph.New() + } + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db + } + g := newStore() + modules := map[string]string{"a": "example.org/app", "b": "example.org/lib"} + state := &GoModuleDepRetry{} + resolver := func(store graph.Store, retry *GoModuleDepRetry) *CrossRepoResolver { + cr := NewCrossRepo(store) + cr.SetGoModules(modules) + cr.SetGoModuleDepRetry(retry, 1) + return cr + } + seed := func(store graph.Store) { + t.Helper() + if bulk, ok := store.(graph.BulkLoader); ok { + bulk.BeginBulkLoad() + defer func() { require.NoError(t, bulk.FlushBulk()) }() + } + var consumers []*graph.Node + var edges []*graph.Edge + for i := 0; i < graph.MaxIncomingSourceCandidateRows+1; i++ { + f := fmt.Sprintf("a/f%d.go", i) + consumers = append(consumers, &graph.Node{ID: f, Kind: graph.KindFile, FilePath: f, RepoPrefix: "a", Language: "go"}, &graph.Node{ID: f + "::import::example.org/lib", Kind: graph.KindImport, FilePath: f, RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/lib"}}) + edges = append(edges, &graph.Edge{From: f, To: "b/lib.go", Kind: graph.EdgeImports, FilePath: f, Line: 2, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/lib", "go_import_fallback": "dep::example.org/lib"}}) + } + consumers = append(consumers, &graph.Node{ID: "a/main.go::Caller", Name: "Caller", Kind: graph.KindFunction, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + consumers = append(consumers, + &graph.Node{ID: "a/new.go", Kind: graph.KindFile, FilePath: "a/new.go", RepoPrefix: "a", Language: "go"}, + &graph.Node{ID: "a/new.go::import::example.org/lib/new", Kind: graph.KindImport, FilePath: "a/new.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/lib/new"}}) + edges = append(edges, &graph.Edge{From: "a/new.go", To: "dep::example.org/lib", Kind: graph.EdgeImports, FilePath: "a/new.go", Line: 2}) + provider := &graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"} + if providerFirst { + store.AddNode(provider) + } + store.AddBatch(consumers, edges) + if !providerFirst { + store.AddNode(provider) + } + } + seed(g) + cr := resolver(g, state) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + // The provider now adds the missing package and function. + addNew := func(store graph.Store) { + store.AddBatch([]*graph.Node{ + {ID: "b/new/new.go", Kind: graph.KindFile, FilePath: "b/new/new.go", RepoPrefix: "b", Language: "go"}, + {ID: "b/new/new.go::New", Name: "New", Kind: graph.KindFunction, FilePath: "b/new/new.go", RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{ + {From: "a/main.go::Caller", To: "unresolved::extern::example.org/lib/new::New", Kind: graph.EdgeCalls, FilePath: "a/main.go", Line: 3}, + }) + } + addNew(g) + if restart { + state = &GoModuleDepRetry{} + } + cr = resolver(g, state) + // When the import recovery exceeds its own ceiling, the call leg must still bind. + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/new/new.go"}) + assert.True(t, admission.Refused, "deferred imports must be reported even while independent calls bind") + assert.Error(t, admission.Refusal) + assert.False(t, state.complete, "deferred import recovery remains retryable") + var calls []*graph.Edge + for _, e := range g.GetOutEdges("a/main.go::Caller") { + if e.Kind == graph.EdgeCalls { + calls = append(calls, e) + } + } + require.Len(t, calls, 1) + assert.Equal(t, "b/new/new.go::New", calls[0].To) + // Refused imports remain eligible for the next full pass, not lost or marked complete. + imports := g.GetOutEdges("a/f0.go") + require.Len(t, imports, 1) + assert.Equal(t, "b/lib.go", imports[0].To) + if !restart { + rows := g.GetOutEdges("a/new.go") + require.Len(t, rows, 1) + assert.Equal(t, "dep::example.org/lib", rows[0].To) + } + _, err := cr.ResolveAllContext(context.Background()) + require.NoError(t, err) + cold := newStore() + seed(cold) + addNew(cold) + _, err = resolver(cold, &GoModuleDepRetry{}).ResolveAllContext(t.Context()) + require.NoError(t, err) + files := []string{"a/new.go", "a/main.go::Caller"} + for i := 0; i < graph.MaxIncomingSourceCandidateRows+1; i++ { + files = append(files, fmt.Sprintf("a/f%d.go", i)) + } + project := func(store graph.Store) map[string][]string { + rows := make(map[string][]string) + for file, edges := range store.GetOutEdgesByNodeIDs(files) { + for _, e := range edges { + rows[file] = append(rows[file], fmt.Sprintf("%s %s cross=%v", e.Kind, e.To, e.CrossRepo)) + } + sort.Strings(rows[file]) + } + return rows + } + assert.Equal(t, project(cold), project(g), "all imports and the new call must match cold, including multiplicity") + freshImports := g.GetOutEdges("a/new.go") + require.Len(t, freshImports, 1, "one source import must bind exactly once") + for _, e := range freshImports { + assert.Equal(t, "b/new/new.go", e.To) + } + }) + } + } + } +} + +// A restart seeds all identities once; cancellation still stops filtered rows. +func TestGoModuleImportSeedCompletionAndCancellation(t *testing.T) { + for _, mode := range []string{"complete", "cancel"} { + t.Run(mode, func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + count := graph.MaxIncomingSourceCandidateRows + 10 + if mode == "cancel" { + count = externalCallMutationChunk*2 + 1 + } + var nodes []*graph.Node + for i := 0; i < count; i++ { + nodes = append(nodes, &graph.Node{ID: fmt.Sprintf("a/f%d.go::import::untracked.org/lib", i), Kind: graph.KindImport, FilePath: fmt.Sprintf("a/f%d.go", i), RepoPrefix: "a", Language: "go"}) + } + nodes = append(nodes, &graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + g.AddBatch(nodes, nil) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.org/lib"}) + if mode == "cancel" { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + g.afterBatch = cancel + require.ErrorIs(t, cr.RetryGoModuleDeps(ctx), context.Canceled) + assert.LessOrEqual(t, g.importRows, externalCallMutationChunk+1) + } else { + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/lib.go"}) + assert.False(t, admission.Refused) + assert.Equal(t, count, g.importRows) + assert.True(t, cr.goModuleRetry.importsSeeded) + cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b/lib.go"}) + assert.Equal(t, 1, g.importScans) + } + require.NotNil(t, cr.goModuleRetry) + if mode == "cancel" { + assert.False(t, cr.goModuleRetry.importsSeeded, "a cancelled seed remains eligible for retry") + } + }) + } +} + +func TestGoModuleImportRestoresEveryAliasStatement(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + const file = "a/main.go" + const id = file + "::import::example.org/lib" + g.AddBatch([]*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "a", Language: "go"}, + {ID: id, FilePath: file, StartLine: 3, Kind: graph.KindImport, RepoPrefix: "a", Language: "go", Meta: map[string]any{"path": "example.org/lib"}}, + {ID: "b/lib.go", FilePath: "b/lib.go", Kind: graph.KindFile, RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{ + {From: file, To: id, Kind: graph.EdgeContains, FilePath: file, Line: 2}, + {From: file, To: id, Kind: graph.EdgeContains, FilePath: file, Line: 3}, + {From: "other.go", To: id, Kind: graph.EdgeContains, FilePath: "other.go", Line: 4}, + {From: file, To: "b/lib.go", Kind: graph.EdgeImports, FilePath: file, Line: 3, CrossRepo: true, Meta: map[string]any{"go_import": "example.org/lib"}}, + }) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"a": "example.org/app", "b": "example.org/lib"}) + for range 2 { + require.NoError(t, cr.restoreGoModuleImportBatch(t.Context(), "b", []string{id}, &CrossRepoStats{ByRepo: map[string]int{}})) + } + var lines []int + for _, edge := range outEdgesOfKind(g, file, graph.EdgeImports) { + require.Equal(t, "b/lib.go", edge.To) + lines = append(lines, edge.Line) + } + sort.Ints(lines) + require.Equal(t, []int{2, 3}, lines, "one import per retained statement, with no duplicates") + }) + } +} diff --git a/internal/resolver/go_module_import_retry_state_test.go b/internal/resolver/go_module_import_retry_state_test.go new file mode 100644 index 000000000..7f98db968 --- /dev/null +++ b/internal/resolver/go_module_import_retry_state_test.go @@ -0,0 +1,137 @@ +package resolver + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func goImportReviewStore(t *testing.T, backend string) graph.Store { + t.Helper() + if backend == "memory" { + return graph.New() + } + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "review.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + return s +} + +func TestGoModuleImportFullPassNewConsumer(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + g.AddNode(&graph.Node{ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + state := &GoModuleDepRetry{} + resolve := func() { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.org/lib"}) + cr.SetGoModuleDepRetry(state, 1) + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + } + for _, repo := range []string{"a", "c"} { + file := repo + "/main.go" + g.AddBatch([]*graph.Node{ + {ID: file, Name: "main.go", Kind: graph.KindFile, FilePath: file, RepoPrefix: repo, Language: "go"}, + {ID: file + "::import::example.org/lib", Name: "lib", Kind: graph.KindImport, FilePath: file, RepoPrefix: repo, Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.org/lib"}}, + }, []*graph.Edge{{From: file, To: "dep::example.org/lib", Kind: graph.EdgeImports, FilePath: file, Line: 2}}) + resolve() + rows := g.GetOutEdges(file) + require.Len(t, rows, 1) + assert.Equal(t, "b/lib.go", rows[0].To, "a full pass must include current import identities even with shared retry state") + assert.True(t, rows[0].CrossRepo) + } + }) + } +} + +func TestGoModuleImportUnconfiguredRetryState(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"full", "consumer", "provider", "retry"} { + for _, configured := range []bool{false, true} { + name := backend + "/" + route + "/unconfigured" + if configured { + name = backend + "/" + route + "/removed_modules" + } + t.Run(name, func(t *testing.T) { + g := goImportReviewStore(t, backend) + g.AddBatch([]*graph.Node{ + {ID: "a/main.go", Name: "main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, + {ID: "a/main.go::Caller", Name: "Caller", Kind: graph.KindFunction, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, + {ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}, + {ID: "b/lib.go::Hello", Name: "Hello", Kind: graph.KindFunction, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{{From: "a/main.go::Caller", To: "dep::example.org/lib::Hello", Kind: graph.EdgeCalls, FilePath: "a/main.go", Line: 3}}) + bound := NewCrossRepo(g) + bound.SetGoModules(map[string]string{"b": "example.org/lib"}) + _, err := bound.ResolveAllContext(t.Context()) + require.NoError(t, err) + require.Equal(t, "b/lib.go::Hello", g.GetOutEdges("a/main.go::Caller")[0].To) + cr := NewCrossRepo(g) + if configured { + cr.SetGoModules(map[string]string{}) + cr.SetGoModuleDepRetry(&GoModuleDepRetry{}, 1) + } + switch route { + case "consumer": + cr.ResolveMutationFrontiers([]string{"a/main.go"}, nil, nil) + case "provider": + cr.ResolveMutationFrontiers(nil, nil, []string{"b/lib.go"}) + case "retry": + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + } + _, err = cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + var rows []*graph.Edge + for _, edge := range g.GetOutEdges("a/main.go::Caller") { + if edge.Kind == graph.EdgeCalls { + rows = append(rows, edge) + } + } + require.Len(t, rows, 1) + if configured { + assert.Equal(t, "dep::example.org/lib::Hello", rows[0].To) + assert.False(t, rows[0].CrossRepo) + } else { + assert.Equal(t, "b/lib.go::Hello", rows[0].To) + assert.True(t, rows[0].CrossRepo) + assert.Nil(t, cr.goModuleRetry, "no module context must not allocate retry state") + } + }) + } + } + } +} + +func TestGoModuleImportReplacementDirectory(t *testing.T) { + for _, tc := range []struct{ name, directory, path, want string }{ + {"local", "a/third_party/library", "example.org/library", "a/third_party/library/lib.go"}, + {"sub_package", "a/third_party/library", "example.org/library/sub", "a/third_party/library/sub/a.go"}, + {"denied", "", "example.org/library", ""}, + {"missing", "a/missing", "example.org/library", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + g := graph.New() + for _, file := range []string{"a/third_party/library/lib.go", "a/third_party/library/sub/a.go", "b/lib.go", "b/sub/a.go"} { + repo := file[:1] + g.AddNode(&graph.Node{ID: file, Name: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: repo, Language: "go"}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"a": "example.org/caller", "b": "example.org/library"}) + cr.SetGoModuleReplacements(map[string][]GoModuleReplacement{"a": {{Module: "example.org/library", Directory: tc.directory}}}) + cr.goModuleImportState() + match := cr.goModuleImportCandidate(tc.path, "a", "") + if tc.want == "" { + assert.Nil(t, match, "a replacement must never fall through to the remote module") + } else { + require.NotNil(t, match) + assert.Equal(t, tc.want, match.ID) + assert.Equal(t, "a", match.RepoPrefix) + } + }) + } +} diff --git a/internal/resolver/go_module_import_seed_test.go b/internal/resolver/go_module_import_seed_test.go new file mode 100644 index 000000000..affd4b892 --- /dev/null +++ b/internal/resolver/go_module_import_seed_test.go @@ -0,0 +1,335 @@ +package resolver + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +// Given more unrelated import identities than a save's recovery ceiling, the +// first save after restart must still discover and recover a small provider. +func TestGoModuleImportLargeRestartSeed(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, mode := range []string{"new_package", "representative"} { + t.Run(backend+"/"+mode, func(t *testing.T) { + newStore := func() graph.Store { + t.Helper() + if backend == "memory" { + return graph.New() + } + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db + } + var nodes []*graph.Node + for i := 0; i < graph.MaxIncomingSourceCandidateRows+1; i++ { + file := fmt.Sprintf("noise/f%d.go", i) + nodes = append(nodes, &graph.Node{ID: file + "::import::untracked.org/library", Kind: graph.KindImport, FilePath: file, RepoPrefix: "noise", Language: "go"}) + } + path, target := "example.org/lib/sub", "dep::example.org/lib" + first, second := "b/sub/z.go", "b/sub/a.go" + if mode == "representative" { + path, target = "example.org/lib", "b/z.go" + first, second = "b/a.go", "b/0.go" + } + nodes = append(nodes, + &graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}, + &graph.Node{ID: "a/main.go::import::" + path, Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": path}}, + &graph.Node{ID: "b/z.go", Kind: graph.KindFile, FilePath: "b/z.go", RepoPrefix: "b", Language: "go"}) + edge := &graph.Edge{From: "a/main.go", To: target, Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2} + if mode == "representative" { + edge.CrossRepo = true + edge.Meta = map[string]any{"go_import": path, "go_import_fallback": "dep::example.org/lib"} + } + g := &goImportCountingStore{Store: newStore()} + g.AddBatch(nodes, []*graph.Edge{edge}) + state := &GoModuleDepRetry{} // unchanged warm restart has no process-local registry + resolver := func(store graph.Store, retry *GoModuleDepRetry) *CrossRepoResolver { + cr := NewCrossRepo(store) + cr.SetGoModules(map[string]string{"a": "example.org/app", "b": "example.org/lib"}) + cr.SetGoModuleDepRetry(retry, 1) + return cr + } + for _, file := range []string{first, second} { + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "b", Language: "go"}) + cr := resolver(g, state) + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{file}) + assert.False(t, admission.Refused) + rows := g.GetOutEdges("a/main.go") + require.Len(t, rows, 1) + assert.Equal(t, file, rows[0].To, "save must recover the new package or representative without a full pass") + assert.True(t, rows[0].CrossRepo) + assert.True(t, state.importsSeeded, "first save must complete the registry even above the recovery ceiling") + } + assert.Equal(t, 1, g.importScans, "later saves must not repeat the global identity scan") + assert.Equal(t, len(nodes)-2, g.importRows, "visit all import identities once, not only the first budget-sized prefix") + cold := newStore() + cold.AddBatch(nodes, []*graph.Edge{edge}) + for _, file := range []string{first, second} { + cold.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "b", Language: "go"}) + } + _, err := resolver(cold, &GoModuleDepRetry{}).ResolveAllContext(context.Background()) + require.NoError(t, err) + warmRows, coldRows := g.GetOutEdges("a/main.go"), cold.GetOutEdges("a/main.go") + require.Len(t, coldRows, 1) + require.Len(t, warmRows, 1) + assert.Equal(t, coldRows[0].To, warmRows[0].To) + assert.Equal(t, coldRows[0].CrossRepo, warmRows[0].CrossRepo) + }) + } + } +} + +func TestGoModuleImportOversizedProviderDoesNotStarveNext(t *testing.T) { + for _, route := range []string{"frontier", "identity"} { + t.Run(route, func(t *testing.T) { + g := graph.New() + for _, file := range []string{"a-big/lib.go", "b-small/lib.go", "consumer/main.go"} { + g.AddNode(&graph.Node{ID: file, Name: "lib.go", FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "go"}) + } + const id = "consumer/main.go::import::example.org/small" + g.AddNode(&graph.Node{ID: id, Name: "small", FilePath: "consumer/main.go", StartLine: 2, Kind: graph.KindImport, RepoPrefix: "consumer", Language: "go", Meta: map[string]any{"path": "example.org/small"}}) + cr := NewCrossRepo(g) + modules := map[string]string{"a-big": "example.org/big", "b-small": "example.org/small", "consumer": "example.org/app"} + cr.SetGoModules(modules) + state := cr.goModuleImportState() + state.importsSeeded, state.complete = true, true + state.modules = modules + state.imports["a-big"] = make(map[string]struct{}) + for i := 0; i <= graph.MaxIncomingSourceCandidateRows; i++ { + state.imports["a-big"][fmt.Sprintf("missing/%d", i)] = struct{}{} + } + state.imports["b-small"] = map[string]struct{}{id: {}} + if route == "frontier" { + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"a-big/lib.go", "b-small/lib.go"}) + require.False(t, admission.Refused) + require.NoError(t, admission.Refusal) + require.True(t, state.complete) + } else { + require.NoError(t, cr.recheckGoModuleBindings(t.Context(), nil, &CrossRepoStats{ByRepo: map[string]int{}}, false)) + } + imports := outEdgesOfKind(g, "consumer/main.go", graph.EdgeImports) + require.Len(t, imports, 1, "the small provider must still restore its import") + require.Equal(t, "b-small/lib.go", imports[0].To) + }) + } +} + +func TestGoModuleNestedProviderFullAndWarmParity(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"full", "frontier", "seed"} { + t.Run(backend+"/"+route, func(t *testing.T) { + g := goImportReviewStore(t, backend) + const file = "a/main.go" + const spec = "example.org/root/child/sub" + const id = file + "::import::" + spec + g.AddBatch([]*graph.Node{ + {ID: file, Name: "main.go", FilePath: file, Kind: graph.KindFile, RepoPrefix: "a", Language: "go"}, + {ID: id, Name: "sub", FilePath: file, StartLine: 2, Kind: graph.KindImport, RepoPrefix: "a", Language: "go", Meta: map[string]any{"path": spec}}, + {ID: "b/sub/lib.go", Name: "lib.go", FilePath: "b/sub/lib.go", Kind: graph.KindFile, RepoPrefix: "b", Language: "go"}, + }, []*graph.Edge{{From: file, To: "dep::example.org/root/child", FilePath: file, Line: 2, Kind: graph.EdgeImports}}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"a": "example.org/root", "b": "example.org/root/child"}) + if route == "seed" { + require.NoError(t, cr.seedGoModuleImportRegistry(t.Context())) + require.Contains(t, cr.goModuleRetry.imports["b"], id) + return + } + if route == "full" { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + } else { + cr.ResolveMutationFiles([]string{file}, nil) + } + imports := outEdgesOfKind(g, file, graph.EdgeImports) + require.Len(t, imports, 1) + require.Equal(t, "b/sub/lib.go", imports[0].To) + require.True(t, imports[0].CrossRepo) + }) + } + } +} + +func TestGoModuleImportRecoveryBudgetRefusal(t *testing.T) { + old := newGoModuleImportRecoveryBudget + newGoModuleImportRecoveryBudget = func() *graph.IncomingSourceBudget { + b := &graph.IncomingSourceBudget{} + require.NoError(t, b.Charge(graph.MaxIncomingSourceCandidateRows-1)) + return b + } + t.Cleanup(func() { newGoModuleImportRecoveryBudget = old }) + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"frontier", "identity"} { + t.Run(backend+"/"+route, func(t *testing.T) { + g := &goImportCountingStore{Store: goImportReviewStore(t, backend)} + g.AddNode(&graph.Node{ID: "provider/lib.go", Name: "lib.go", FilePath: "provider/lib.go", Kind: graph.KindFile, RepoPrefix: "provider", Language: "go"}) + ids := map[string]struct{}{} + for i := 0; i < 2; i++ { + file := fmt.Sprintf("consumer/f%d.go", i) + id := file + "::import::example.org/lib" + ids[id] = struct{}{} + g.AddBatch([]*graph.Node{{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "go"}, {ID: id, FilePath: file, StartLine: 1, Kind: graph.KindImport, RepoPrefix: "consumer", Language: "go", Meta: map[string]any{"path": "example.org/lib"}}}, []*graph.Edge{{From: file, FilePath: file, To: "dep::example.org/lib", Kind: graph.EdgeImports, Line: 1}}) + } + cr := NewCrossRepo(g) + modules := map[string]string{"provider": "example.org/lib", "consumer": "example.org/app"} + cr.SetGoModules(modules) + state := cr.goModuleImportState() + state.importsSeeded = true + state.imports["provider"] = ids + state.complete = route == "frontier" + state.modules = modules + changed := "provider/lib.go" + if route == "identity" { + changed = "consumer/f0.go" + } + if route == "frontier" { + for _, file := range []string{changed, changed, "consumer/f0.go"} { + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{file}) + require.False(t, admission.Refused, "a provider skip must not request a full continuation") + require.NoError(t, admission.Refusal) + require.Zero(t, admission.Dropped) + require.True(t, state.complete, "a provider skip must not reopen the revision") + } + require.Zero(t, g.bindingRechecks, "two provider saves and a third Go save must not recheck all repositories") + for i := range 2 { + rows := outEdgesOfKind(g, fmt.Sprintf("consumer/f%d.go", i), graph.EdgeImports) + require.Len(t, rows, 1) + require.Equal(t, "dep::example.org/lib", rows[0].To, "provider skips preserve stale imports until a full pass") + } + } else { + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{changed}) + require.True(t, admission.Refused) + require.Error(t, admission.Refusal) + require.GreaterOrEqual(t, admission.Dropped, 2) + require.False(t, state.complete) + require.False(t, admission.IncomingRefused) + require.Equal(t, 1, g.bindingRechecks) + } + // The explicit full pass is unbounded and repairs the deferred imports. + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + require.True(t, state.complete, "a full pass must finish the deferred revision") + for _, restart := range []bool{false, true} { + next := cr + if restart { + next = NewCrossRepo(g) + next.SetGoModules(modules) + next.SetGoModuleDepRetry(state, cr.goModuleRevision) + } + _, nextAdmission := next.ResolveMutationFrontiersBounded(nil, nil, []string{"consumer/f0.go"}) + require.False(t, nextAdmission.Refused, "completed revision must not reopen its bounded recovery") + } + for i := 0; i < 2; i++ { + rows := outEdgesOfKind(g, fmt.Sprintf("consumer/f%d.go", i), graph.EdgeImports) + require.Len(t, rows, 1) + require.Equal(t, "provider/lib.go", rows[0].To) + } + _, err = cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + require.True(t, state.complete) + }) + } + } +} + +func TestGoModuleImportUnrelatedMutationSkipsRecovery(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, file := range []string{"other/main.py", "other/main.ts", "provider/README.md", "provider/lib_test.go"} { + t.Run(backend+"/"+file, func(t *testing.T) { + g := &goImportCountingStore{Store: goImportReviewStore(t, backend)} + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: graph.RepoPrefixOfID(file)}) + state := &GoModuleDepRetry{} + for i := 0; i < 2; i++ { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"provider": "example.org/lib"}) + cr.SetGoModuleDepRetry(state, 1) + packageView := map[string]graph.FileNodeIdentity{"provider/lib.go": {ID: "provider/lib.go", FilePath: "provider/lib.go", RepoPrefix: "provider"}} + cr.goModuleImportState().packages["provider"] = packageView + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{file, file}) + require.False(t, admission.Refused) + require.Zero(t, g.importScans, "non-production-Go mutations must not seed the import registry") + require.Equal(t, packageView, state.packages["provider"], "unrelated saves must not invalidate provider projections") + } + }) + } + } +} + +func TestGoModuleManifestFrontiersRunRetry(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, file := range []string{"provider/go.mod", "provider/go.sum", "provider/go.work", "provider/go.work.sum", "provider/vendor/modules.txt", "provider/nested/vendor/modules.txt"} { + for route := range 3 { + t.Run(fmt.Sprintf("%s/%s/frontier_%d", backend, file, route), func(t *testing.T) { + g := &goImportCountingStore{Store: goImportReviewStore(t, backend)} + state := &GoModuleDepRetry{} + frontiers := [3][]string{} + frontiers[route] = []string{file, file} + for range 2 { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"provider": "example.org/lib"}) + cr.SetGoModuleDepRetry(state, 1) + _, admission := cr.ResolveMutationFrontiersBounded(frontiers[0], frontiers[1], frontiers[2]) + require.False(t, admission.Refused) + require.True(t, state.complete, "Go build-list input must enter revision retry") + } + require.Equal(t, 1, g.importScans, "manifest retry must seed imports once for revision recovery") + }) + } + } + } +} + +func TestGoModuleImportProviderBudgetOrder(t *testing.T) { + t.Cleanup(SetGoModuleImportRecoveryBudgetForTest(1)) + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + modules := map[string]string{"a-provider": "example.org/a", "b-provider": "example.org/b", "consumer": "example.org/app"} + cr := NewCrossRepo(g) + cr.SetGoModules(modules) + state := cr.goModuleImportState() + state.modules, state.complete, state.importsSeeded = modules, true, true + for i, repo := range []string{"a-provider", "b-provider"} { + file := fmt.Sprintf("consumer/f%d.go", i) + id := file + "::import::" + modules[repo] + g.AddBatch([]*graph.Node{ + {ID: repo + "/lib.go", Name: "lib.go", FilePath: repo + "/lib.go", Kind: graph.KindFile, RepoPrefix: repo, Language: "go"}, + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", Language: "go"}, + {ID: id, FilePath: file, StartLine: 1, Kind: graph.KindImport, RepoPrefix: "consumer", Language: "go", Meta: map[string]any{"path": modules[repo]}}, + }, []*graph.Edge{{From: file, FilePath: file, To: "dep::" + modules[repo], Kind: graph.EdgeImports, Line: 1}}) + state.imports[repo] = map[string]struct{}{id: {}} + } + core, logs := observer.New(zap.WarnLevel) + cr.SetLogger(zap.New(core)) + for range 8 { + for i, repo := range []string{"a-provider", "b-provider"} { + rows := outEdgesOfKind(g, fmt.Sprintf("consumer/f%d.go", i), graph.EdgeImports) + require.Len(t, rows, 1) + oldTo := rows[0].To + rows[0].To, rows[0].CrossRepo, rows[0].Meta = "dep::"+modules[repo], false, nil + g.ReindexEdges([]graph.EdgeReindex{{Edge: rows[0], OldTo: oldTo}}) + } + stats, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{"b-provider/lib.go", "a-provider/lib.go"}) + require.False(t, admission.Refused) + require.True(t, state.complete) + require.Equal(t, 1, stats.Resolved) + require.Equal(t, "a-provider/lib.go", outEdgesOfKind(g, "consumer/f0.go", graph.EdgeImports)[0].To) + require.Equal(t, "dep::example.org/b", outEdgesOfKind(g, "consumer/f1.go", graph.EdgeImports)[0].To) + warnings := logs.TakeAll() + require.Len(t, warnings, 1) + require.Equal(t, "cross-repo resolve: provider consumers stay stale until next full pass", warnings[0].Message) + require.Equal(t, "b-provider", warnings[0].ContextMap()["repo"]) + } + }) + } +} diff --git a/internal/resolver/go_module_imports.go b/internal/resolver/go_module_imports.go new file mode 100644 index 000000000..c2404d7c7 --- /dev/null +++ b/internal/resolver/go_module_imports.go @@ -0,0 +1,501 @@ +package resolver + +import ( + "context" + "iter" + "maps" + "strings" + + "github.com/zzet/gortex/internal/graph" +) + +// resolveGoModuleImports preserves dependency contracts while lifting Go source +// imports onto a tracked package's representative file in a file frontier. +// counted records whether the ordinary frontier pass already counted an import +// as resolved (true) or unresolved (false), before this pass retargets it. +func (cr *CrossRepoResolver) resolveGoModuleImports(ctx context.Context, files []string, stats *CrossRepoStats, counted map[graph.EdgeIdentity]bool) error { + if len(files) == 0 || len(cr.goModules) == 0 && cr.goModuleRetry == nil { + return nil + } + cr.goModuleImportState() + flush := func(paths []string) { + nodes := cr.graph.GetFileNodesByPaths(paths) + outgoing := cr.graph.GetOutEdgesByNodeIDs(paths) + var batch []graph.EdgeReindex + for _, path := range paths { + cr.rememberGoModuleImports(nodes[path]) + var source *graph.Node + for _, node := range nodes[path] { + if node.Kind == graph.KindFile && node.ID == path { + source = node + break + } + } + caller := "" + if source != nil { + caller = source.RepoPrefix + } + for _, e := range outgoing[path] { + if e.Kind != graph.EdgeImports { + continue + } + importPath := goModuleImportPath(e, nodes[path], outgoing[path]) + if importPath == "" || !trackedGoImport(cr.goModulePaths, importPath) && e.Meta["go_import"] == nil { + continue + } + match := cr.goModuleImportCandidate(importPath, caller, candidateWorkspaceID(source)) + if match != nil && match.ID == e.To || match == nil && e.Meta["go_import"] == nil { + continue + } + copy := *e + copy.Meta = maps.Clone(e.Meta) + if copy.Meta == nil { + copy.Meta = make(map[string]any) + } + if copy.Meta["go_import"] == nil { + copy.Meta["go_import"] = importPath + copy.Meta["go_import_fallback"] = e.To + } + if match != nil { + copy.To = match.ID + copy.CrossRepo = caller != "" && match.RepoPrefix != caller + } else { + copy.To, _ = copy.Meta["go_import_fallback"].(string) + if copy.To == "" || graph.IsUnresolvedTarget(copy.To) { + copy.To = "external::" + importPath + } + copy.CrossRepo, copy.Confidence, copy.ConfidenceLabel, copy.Origin, copy.Tier = false, 0, "", "", "" + } + if copy.To == e.To { + continue + } + clearEdgeTerminal(©) + batch = append(batch, graph.EdgeReindex{Edge: ©, OldTo: e.To, RefreshIdentity: true, OldFilePath: e.FilePath, OldLine: e.Line}) + if len(batch) == externalCallMutationChunk { + cr.applyGoModuleImportReindexes(batch, stats, counted) + batch = nil + } + } + } + cr.applyGoModuleImportReindexes(batch, stats, counted) + } + unique := make(map[string]bool) + var goFiles []string + for _, path := range files { + if strings.HasSuffix(path, ".go") && !unique[path] { + unique[path] = true + goFiles = append(goFiles, path) + } + } + files = goFiles + for start := 0; start < len(files); start += externalCallMutationChunk { + if err := ctx.Err(); err != nil { + return err + } + flush(files[start:min(start+externalCallMutationChunk, len(files))]) + } + return ctx.Err() +} + +// A dep contract has lost the package suffix. Recover it from the retained +// import statement, but never guess when several same-line imports fit. +func goModuleImportPath(e *graph.Edge, nodes []*graph.Node, edges []*graph.Edge) string { + if path, ok := e.Meta["go_import"].(string); ok && path != "" { + return path + } + target := graph.UnresolvedName(e.To) + if target == "" { + target = e.To + } + target = strings.TrimPrefix(target, externalCallPrefix) + if path, ok := strings.CutPrefix(target, "import::"); ok { + return path + } + if path, ok := strings.CutPrefix(target, "external::"); ok { + return path + } + module, dep := strings.CutPrefix(target, "dep::") + if !dep || strings.Contains(module, "::") { + return "" + } + // Duplicate aliases share an import node, whose retained line is the last + // occurrence. Containment edges preserve every statement's source line. + contains := make(map[string]bool) + for _, link := range edges { + if link.Kind == graph.EdgeContains && link.From == e.From && link.Line == e.Line { + contains[link.To] = true + } + } + path := "" + for _, n := range nodes { + if n.Kind != graph.KindImport || n.Language != "go" || (n.StartLine != e.Line && !contains[n.ID]) { + continue + } + candidate, _ := n.Meta["path"].(string) + if candidate != module && !strings.HasPrefix(candidate, module+"/") { + continue + } + if path != "" && path != candidate { + return "" + } + path = candidate + } + return path +} + +// Package identities are cached across resolver objects by module revision. +// A provider mutation invalidates only that provider's projection. +func (cr *CrossRepoResolver) goModuleImportCandidate(importPath, caller, workspace string) *graph.Node { + state := cr.goModuleRetry + var candidates []*graph.Node + module := cr.goModulePaths.longest(importPath) + replacement, replaced := goModuleReplacementFor(cr.goModuleReplacements, caller, importPath) + repos := cr.goModuleRepos[module] + if replaced { + if replacement.Directory == "" { + return nil + } + repos = []string{caller} + } + for _, repo := range repos { + packages, cached := state.packages[repo] + if !cached { + packages = make(map[string]graph.FileNodeIdentity) + for file := range graph.FileNodeIdentitiesSeq(cr.graph, []string{repo}) { + if !strings.HasSuffix(file.FilePath, ".go") || strings.HasSuffix(file.FilePath, "_test.go") { + continue + } + dir := filePathDir(file.FilePath) + if prior, ok := packages[dir]; !ok || file.ID < prior.ID { + packages[dir] = file + } + } + state.packages[repo] = packages + } + dir := repo + strings.TrimPrefix(importPath, module) + if replaced { + dir = replacement.Directory + } + if file, ok := packages[dir]; ok { + candidate := &graph.Node{ID: file.ID, Kind: graph.KindFile, FilePath: file.FilePath, RepoPrefix: file.RepoPrefix, WorkspaceID: file.WorkspaceID, Language: "go"} + if replaced { + return candidate + } + candidates = append(candidates, candidate) + } + } + if replaced { + return nil + } + return goModuleCandidate(cr.goModules, importPath, candidates, caller, workspace, cr.crossWorkspaceLookup) +} + +func (cr *CrossRepoResolver) applyGoModuleImportReindexes(batch []graph.EdgeReindex, stats *CrossRepoStats, counted map[graph.EdgeIdentity]bool) { + batch = cr.filterLiveReindex(batch) + if !cr.applyGoModuleReindexes(batch) { + originals := make(map[graph.EdgeIdentity]*graph.Edge) + for _, row := range batch { + original := *row.Edge + original.To = row.OldTo + original.Meta = maps.Clone(row.Edge.Meta) + originals[graph.EdgeIdentityFor(&original)] = &original + } + cr.queueGoModuleEdges(originals) + cr.logger.Warn("Go module import batch dropped; queued for a later module pass") + return + } + if stats == nil { + return + } + targetIDs := make([]string, 0, len(batch)) + seenTargets := make(map[string]bool) + for _, row := range batch { + if row.Edge.CrossRepo && !seenTargets[row.Edge.To] { + seenTargets[row.Edge.To] = true + targetIDs = append(targetIDs, row.Edge.To) + } + } + var targets map[string]*graph.Node + if len(targetIDs) > 0 { + targets = cr.graph.GetNodesByIDs(targetIDs) + } + for _, row := range batch { + original := graph.EdgeIdentityFor(row.Edge) + original.To = row.OldTo + resolved, alreadyCounted := counted[original] + nowResolved := !isSyntheticResolveTarget(row.Edge.To) + if nowResolved { + if !alreadyCounted || !resolved { + stats.Resolved++ + } + if alreadyCounted && !resolved { + stats.Unresolved-- + } + } else { + if !alreadyCounted || resolved { + stats.Unresolved++ + } + if alreadyCounted && resolved { + stats.Resolved-- + } + } + if row.Edge.CrossRepo { + stats.CrossRepoEdges++ + if target := targets[row.Edge.To]; target != nil { + stats.ByRepo[target.RepoPrefix]++ + } + } + } +} + +// Filter identity-only rows before loading any file payload or adjacency. +func (cr *CrossRepoResolver) goModuleImportFileBatches(ctx context.Context, repos []string) iter.Seq2[[]string, error] { + return func(yield func([]string, error) bool) { + seen := make(map[string]bool) + var files []string + for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, graph.KindImport) { + if err := ctx.Err(); err != nil { + yield(nil, err) + return + } + file, path, ok := strings.Cut(n.ID, "::import::") + if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModulePaths, path) || seen[file] { + continue + } + if module := cr.goModules[n.RepoPrefix]; module != "" && cr.goModulePaths.longest(path) == module { + continue + } + seen[file] = true + files = append(files, file) + if len(files) == externalCallMutationChunk { + if !yield(files, nil) { + return + } + files = nil + } + } + if len(files) > 0 { + yield(files, nil) + } + } +} + +// Import indexes share the store's resolve mutex, not the retry queue mutex. +// Retain statement IDs, not edges: consumer edits and removals are revalidated. +func (cr *CrossRepoResolver) goModuleImportState() *GoModuleDepRetry { + if cr.goModuleRetry == nil { + cr.goModuleRetry = &GoModuleDepRetry{} + } + state := cr.goModuleRetry + if state.packageRevision != cr.goModuleRevision { + state.packageRevision = cr.goModuleRevision + state.packages = nil + } + if !maps.Equal(state.packageModules, cr.goModules) { + state.packageModules = maps.Clone(cr.goModules) + state.packageRevision = cr.goModuleRevision + state.packages = nil + state.imports = nil + state.importsSeeded = false + } + if state.packages == nil { + state.packages = make(map[string]map[string]graph.FileNodeIdentity) + } + if state.imports == nil { + state.imports = make(map[string]map[string]struct{}) + } + return state +} + +// Seed import identities without hydrating files or mutating edges. Saves seed +// once; whole-graph passes invalidate the seed. This scan must not consume a +// provider's recovery budget. +func (cr *CrossRepoResolver) seedGoModuleImportRegistry(ctx context.Context) error { + if len(cr.goModules) == 0 { + return ctx.Err() + } + cr.mu.Lock() + defer cr.mu.Unlock() + state := cr.goModuleImportState() + if state.importsSeeded { + return ctx.Err() + } + for n := range graph.RepoNodeIdentitiesSeq(cr.graph, nil, graph.KindImport) { + if err := ctx.Err(); err != nil { + return err + } + file, path, ok := strings.Cut(n.ID, "::import::") + if !ok || !strings.HasSuffix(file, ".go") || !trackedGoImport(cr.goModulePaths, path) { + continue + } + if module := cr.goModules[n.RepoPrefix]; module != "" && cr.goModulePaths.longest(path) == module { + continue + } + cr.rememberGoModuleImports([]*graph.Node{{ID: n.ID, RepoPrefix: n.RepoPrefix, Kind: graph.KindImport, Language: "go"}}) + } + state.importsSeeded = ctx.Err() == nil + return ctx.Err() +} + +// An unchanged warm restart skips global resolution. Public entry points seed +// once before taking the pass lock; each candidate batch takes its own lock. +func (cr *CrossRepoResolver) resolveGoModuleImportSweep(ctx context.Context, stats *CrossRepoStats) error { + if len(cr.goModules) == 0 { + return ctx.Err() + } + cr.mu.Lock() + state := cr.goModuleImportState() + seeded := state.importsSeeded + cr.mu.Unlock() + if seeded { + return ctx.Err() + } + for files, err := range cr.goModuleImportFileBatches(ctx, nil) { + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if err := func() error { + cr.mu.Lock() + defer cr.mu.Unlock() + return cr.resolveGoModuleImports(ctx, files, stats, nil) + }(); err != nil { + return err + } + } + cr.mu.Lock() + defer cr.mu.Unlock() + if ctx.Err() == nil && state.packageRevision == cr.goModuleRevision && maps.Equal(state.packageModules, cr.goModules) { + state.importsSeeded = true + } + return ctx.Err() +} + +func (cr *CrossRepoResolver) rememberGoModuleImports(nodes []*graph.Node) { + state := cr.goModuleRetry + for _, n := range nodes { + if n.Kind != graph.KindImport || n.Language != "go" { + continue + } + _, path, ok := strings.Cut(n.ID, "::import::") + if !ok { + continue + } + for _, repo := range cr.goModuleRepos[cr.goModulePaths.longest(path)] { + if repo == n.RepoPrefix { + continue + } + if state.imports[repo] == nil { + state.imports[repo] = make(map[string]struct{}) + } + state.imports[repo][n.ID] = struct{}{} + } + } +} + +// InvalidatePackageRepo drops only a removed provider's file projection. +// The caller holds the store resolve mutex, which guards package projections; +// the retry queue mutex does not. +func (r *GoModuleDepRetry) InvalidatePackageRepo(repo string) { + if r == nil || repo == "" { + return + } + delete(r.packages, repo) +} + +// Only retained statements registered for this provider are probed on edits. +// No global node scan is needed to reconstruct imports deleted by eviction. +func (cr *CrossRepoResolver) goModuleImportIDs(repo string) []string { + state := cr.goModuleImportState() + delete(state.packages, repo) + var ids []string + for id := range state.imports[repo] { + ids = append(ids, id) + } + return ids +} + +func (cr *CrossRepoResolver) restoreGoModuleImports(ctx context.Context, repo string, stats *CrossRepoStats) error { + if len(cr.graph.RepoPrefixes()) < 2 { + return ctx.Err() + } + ids := cr.goModuleImportIDs(repo) + for start := 0; start < len(ids); start += externalCallMutationChunk { + if err := ctx.Err(); err != nil { + return err + } + if err := cr.restoreGoModuleImportBatch(ctx, repo, ids[start:min(start+externalCallMutationChunk, len(ids))], stats); err != nil { + return err + } + } + return ctx.Err() +} + +func (cr *CrossRepoResolver) restoreGoModuleImportBatch(ctx context.Context, repo string, ids []string, stats *CrossRepoStats) error { + state := cr.goModuleImportState() + nodes := cr.graph.GetNodesByIDs(ids) + paths := make(map[string]bool) + var files []string + for _, n := range nodes { + if n != nil && n.Kind == graph.KindImport && n.Language == "go" && !paths[n.FilePath] { + paths[n.FilePath] = true + files = append(files, n.FilePath) + } + } + outgoing := cr.graph.GetOutEdgesByNodeIDs(files) + var fileNodes map[string][]*graph.Node + for _, id := range ids { + n := nodes[id] + if n == nil || n.Kind != graph.KindImport || n.Language != "go" { + delete(state.imports[repo], id) + continue + } + _, path, ok := strings.Cut(n.ID, "::import::") + module := cr.goModules[repo] + if n.RepoPrefix == repo || !ok || module == "" || (path != module && !strings.HasPrefix(path, module+"/")) { + delete(state.imports[repo], id) + continue + } + lines := map[int]bool{n.StartLine: true} + for _, edge := range outgoing[n.FilePath] { + if edge.Kind == graph.EdgeContains && edge.From == n.FilePath && edge.To == n.ID { + lines[edge.Line] = true + } + } + for line := range lines { + present := false + for _, e := range outgoing[n.FilePath] { + if e.Kind != graph.EdgeImports || e.Line != line { + continue + } + // Any existing concrete Go-file binding owns this statement, + // including per-repo vendored bindings outside the module map. + boundPath, _ := e.Meta["go_import"].(string) + if (boundPath == "" || boundPath == path) && strings.HasSuffix(e.To, ".go") && !graph.IsUnresolvedTarget(e.To) && + !isSyntheticResolveTarget(e.To) && !strings.HasPrefix(e.To, externalCallPrefix) { + present = true + break + } + target := graph.UnresolvedName(e.To) + if target == "" { + target = e.To + } + if strings.HasPrefix(strings.TrimPrefix(target, externalCallPrefix), "dep::") && fileNodes == nil { + fileNodes = cr.graph.GetFileNodesByPaths(files) + } + if goModuleImportPath(e, fileNodes[n.FilePath], outgoing[n.FilePath]) == path { + present = true + break + } + + } + if !present { + cr.graph.AddEdge(&graph.Edge{From: n.FilePath, To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: n.FilePath, Line: line}) + } + } + + } + return cr.resolveGoModuleImports(ctx, files, stats, nil) +} diff --git a/internal/resolver/go_module_imports_test.go b/internal/resolver/go_module_imports_test.go new file mode 100644 index 000000000..dd93fced8 --- /dev/null +++ b/internal/resolver/go_module_imports_test.go @@ -0,0 +1,770 @@ +package resolver + +import ( + "context" + "fmt" + "iter" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestGoModuleImportRetry(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, tc := range []struct { + name, targetWorkspace, declared, want string + modules map[string]string + }{ + {"sub_package", "shared", "", "b/sub/a.go", map[string]string{"b": "example.com/lib"}}, + {"denied_workspace", "other", "", "dep::example.com/lib", map[string]string{"b": "example.com/lib"}}, + {"declared_workspace", "other", "example.com/lib", "b/sub/a.go", map[string]string{"b": "example.com/lib"}}, + {"wrong_declaration", "other", "example.com/lib2", "dep::example.com/lib", map[string]string{"b": "example.com/lib"}}, + {"wrong_module", "shared", "", "dep::example.com/lib", map[string]string{"b": "example.com/lib2"}}, + {"duplicate", "shared", "", "dep::example.com/lib", map[string]string{"b": "example.com/lib", "fork": "example.com/lib"}}, + {"longest", "shared", "", "child/a.go", map[string]string{"b": "example.com/lib", "child": "example.com/lib/sub"}}, + } { + t.Run(backend+"/"+tc.name, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", WorkspaceID: "shared", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib/sub", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib/sub"}}) + addDepNode(t, g, "a", "example.com/lib") + g.AddEdge(&graph.Edge{From: "a/go.mod", To: "dep::example.com/lib", Kind: graph.EdgeConsumes, FilePath: "a/go.mod"}) + for repo := range tc.modules { + dir := repo + "/sub" + if repo == "child" { + dir = repo + } + for _, name := range []string{"z.go", "a.go", "0_test.go"} { + file := dir + "/" + name + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: repo, WorkspaceID: tc.targetWorkspace, Language: "go"}) + } + } + g.AddEdge(&graph.Edge{From: "a/main.go", To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + state := &GoModuleDepRetry{} + resolve := func() { + cr := NewCrossRepo(g) + cr.SetGoModules(tc.modules) + cr.SetGoModuleDepRetry(state, 1) + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { + if tc.declared == "" { + return nil + } + return []CrossWorkspaceDepRule{{Workspace: "other", Modules: []string{tc.declared}}} + }) + require.NoError(t, cr.RetryGoModuleDeps(context.Background())) + } + resolve() + resolve() + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + assert.Equal(t, tc.want, edges[0].To) + assert.Equal(t, tc.want != "dep::example.com/lib", edges[0].CrossRepo) + if edges[0].CrossRepo { + assert.Empty(t, edges[0].Origin) + assert.Zero(t, edges[0].Confidence) + } + require.NotNil(t, g.GetNode("dep::example.com/lib")) + contracts := g.GetOutEdges("a/go.mod") + require.Len(t, contracts, 1) + assert.Equal(t, "dep::example.com/lib", contracts[0].To) + assert.Equal(t, graph.EdgeConsumes, contracts[0].Kind) + }) + } + } +} + +func TestGoModuleImportPathAmbiguousLine(t *testing.T) { + edge := &graph.Edge{To: "dep::example.com/lib", Kind: graph.EdgeImports, Line: 2} + nodes := []*graph.Node{ + {Kind: graph.KindImport, Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib/a"}}, + {Kind: graph.KindImport, Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib/b"}}, + } + assert.Empty(t, goModuleImportPath(edge, nodes, nil), "a legacy module-only target is not enough evidence for either package") +} + +func TestGoModuleImportPathContainment(t *testing.T) { + const file = "a/main.go" + const path = "example.com/lib/sub" + const id = file + "::import::" + path + edge := &graph.Edge{From: file, To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: file, Line: 3} + nodes := []*graph.Node{ + {ID: id, Kind: graph.KindImport, Language: "go", StartLine: 4, Meta: map[string]any{"path": path}}, + } + for _, tc := range []struct { + name string + link *graph.Edge + want string + }{ + {"exact_line", &graph.Edge{From: file, To: id, Kind: graph.EdgeContains, Line: 3}, path}, + {"wrong_owner", &graph.Edge{From: "a/other.go", To: id, Kind: graph.EdgeContains, Line: 3}, ""}, + {"wrong_line", &graph.Edge{From: file, To: id, Kind: graph.EdgeContains, Line: 4}, ""}, + {"wrong_kind", &graph.Edge{From: file, To: id, Kind: graph.EdgeImports, Line: 3}, ""}, + {"wrong_target", &graph.Edge{From: file, To: "other", Kind: graph.EdgeContains, Line: 3}, ""}, + {"missing", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + var links []*graph.Edge + if tc.link != nil { + links = append(links, tc.link) + } + assert.Equal(t, tc.want, goModuleImportPath(edge, nodes, links)) + }) + } + nodes = append(nodes, &graph.Node{ID: "other", Kind: graph.KindImport, Language: "go", StartLine: 4, Meta: map[string]any{"path": "example.com/lib/other"}}) + links := []*graph.Edge{ + {From: file, To: id, Kind: graph.EdgeContains, Line: 3}, + {From: file, To: "other", Kind: graph.EdgeContains, Line: 3}, + } + assert.Empty(t, goModuleImportPath(edge, nodes, links), "containment must not bypass same-line package ambiguity") +} + +func TestGoModuleImportDroppedBatch(t *testing.T) { + g := &goModuleDroppingStore{Store: graph.New(), drop: true} + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib"}}) + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + state := &GoModuleDepRetry{} + for pass := range 2 { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + cr.SetGoModuleDepRetry(state, 1) + require.NoError(t, cr.RetryGoModuleDeps(context.Background())) + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + if pass == 0 { + assert.Equal(t, "dep::example.com/lib", edges[0].To) + assert.Len(t, state.pending, 1) + } else { + assert.Equal(t, "b/lib.go", edges[0].To) + assert.Empty(t, state.pending) + } + } +} + +// Count the expensive projections, not elapsed time. +type goImportCountingStore struct { + graph.Store + fileScans, hydrated int + edgeLoads int + importScans, importRows int + bindingRechecks int + afterBatch func() +} + +func (s *goImportCountingStore) FileNodeIdentitiesSeq(repos []string) iter.Seq[graph.FileNodeIdentity] { + s.fileScans++ + return graph.FileNodeIdentitiesSeq(s.Store, repos) +} +func (s *goImportCountingStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { + if IsGoModuleBindingRecheckForTest(kinds) { + s.bindingRechecks++ + } + return func(yield func(graph.RepoNodeIdentity) bool) { + imports := len(kinds) == 1 && kinds[0] == graph.KindImport && repos == nil + if imports { + s.importScans++ + } + visited := 0 + for row := range graph.RepoNodeIdentitiesSeq(s.Store, repos, kinds...) { + if imports { + s.importRows++ + } + if !yield(row) { + return + } + visited++ + if visited == externalCallMutationChunk && s.afterBatch != nil { + s.afterBatch() + } + } + } +} +func (s *goImportCountingStore) GetFileNodesByPaths(paths []string) map[string][]*graph.Node { + s.hydrated += len(paths) + return s.Store.GetFileNodesByPaths(paths) +} +func (s *goImportCountingStore) GetOutEdgesByNodeIDs(ids []string) map[string][]*graph.Edge { + s.edgeLoads += len(ids) + return s.Store.GetOutEdgesByNodeIDs(ids) +} + +func TestGoModuleImportLookupBudget(t *testing.T) { + for _, mode := range []string{"cache", "filter", "dedup", "frontier_stats", "external_stats", "deleted_target"} { + t.Run(mode, func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib"}}) + e := &graph.Edge{From: "a/main.go", To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2} + g.AddEdge(e) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + switch mode { + case "cache": + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"a/main.go"}, nil, nil)) + g.RemoveEdge("a/main.go", "b/lib.go", graph.EdgeImports) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "dep::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"a/main.go"}, nil, nil)) + assert.Equal(t, 1, g.fileScans) + case "filter": + g.RemoveEdge(e.From, e.To, e.Kind) + g.EvictFile("a/main.go") + g.AddNode(&graph.Node{ID: "a/unrelated.go::import::fmt", Kind: graph.KindImport, FilePath: "a/unrelated.go", RepoPrefix: "a", Language: "go"}) + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("b")) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + assert.Zero(t, g.hydrated) + case "dedup": + stats := &CrossRepoStats{ByRepo: map[string]int{}} + cr.resolveGoModuleEdgesLocked([]*graph.Edge{e, e}, stats) + assert.Equal(t, 1, stats.Resolved) + assert.Equal(t, 1, stats.CrossRepoEdges) + case "frontier_stats", "external_stats": + g.RemoveEdge(e.From, e.To, e.Kind) + if mode == "frontier_stats" { + addDepNode(t, g, "a", "example.com/lib") + } + e.To = "unresolved::import::example.com/lib" + g.AddEdge(e) + stats := cr.ResolveMutationFrontiers([]string{"a/main.go"}, nil, nil) + assert.Equal(t, 1, stats.Resolved) + assert.Equal(t, 1, stats.CrossRepoEdges) + assert.Zero(t, stats.Unresolved) + case "deleted_target": + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"a/main.go"}, nil, nil)) + g.EvictFile("b/lib.go") + g.AddEdge(&graph.Edge{From: e.From, To: "dep::example.com/lib", Kind: e.Kind, FilePath: e.FilePath, Line: e.Line}) + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"a/main.go"}, nil, nil)) + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + assert.Equal(t, "dep::example.com/lib", edges[0].To, "a stale cached representative must not become a dangling target") + } + }) + } +} + +func TestGoModuleImportWarmProviderEdit(t *testing.T) { + for _, route := range []string{"mutation", "post_publish"} { + t.Run(route, func(t *testing.T) { + db := filepath.Join(t.TempDir(), "graph.sqlite") + s, err := store_sqlite.Open(db) + require.NoError(t, err) + s.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + s.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib"}}) + s.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + // Persist an already-bound graph, with no retry state carried over. + s.AddEdge(&graph.Edge{From: "a/main.go", To: "b/lib.go", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2, CrossRepo: true}) + require.NoError(t, s.Close()) + s, err = store_sqlite.Open(db) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g := &goImportCountingStore{Store: s} + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + // The unchanged warm-restart fast path skips global resolution. + // Include unrelated imports to expose the initial scan's exact cost. + for _, path := range []string{"fmt", "strings", "example.com/untracked"} { + s.AddNode(&graph.Node{ID: "a/other.go::import::" + path, Kind: graph.KindImport, FilePath: "a/other.go", RepoPrefix: "a", Language: "go"}) + } + s.EvictFile("b/lib.go") + s.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + if route == "mutation" { + cr.ResolveMutationFrontiers(nil, nil, []string{"b/lib.go"}) + } else { + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("b")) + } + edges := s.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + assert.Equal(t, "b/lib.go", edges[0].To) + assert.True(t, edges[0].CrossRepo) + assert.Equal(t, 1, g.importScans) + assert.Equal(t, 4, g.importRows, "one tracked and three unrelated import identities") + s.EvictFile("b/lib.go") + s.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + if route == "mutation" { + cr.ResolveMutationFrontiers(nil, nil, []string{"b/lib.go"}) + } else { + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("b")) + } + require.Len(t, s.GetOutEdges("a/main.go"), 1) + assert.Equal(t, 1, g.importScans, "later edits must not repeat the global identity scan") + assert.Equal(t, 4, g.importRows) + }) + } +} + +type goImportPanicStore struct{ graph.Store } + +func (s *goImportPanicStore) GetFileNodesByPaths([]string) map[string][]*graph.Node { + panic("import-store-panic") +} + +func TestGoModuleImportPanicUnlock(t *testing.T) { + if os.Getenv("GORTEX_TEST_IMPORT_PANIC") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=^TestGoModuleImportPanicUnlock$", "-test.v") + cmd.Env = append(os.Environ(), "GORTEX_TEST_IMPORT_PANIC=1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", output) + return + } + g := &goImportPanicStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + assert.PanicsWithValue(t, "import-store-panic", func() { _ = cr.RetryGoModuleDeps(t.Context()) }) + require.True(t, cr.mu.TryLock(), "the store resolve mutex must be released after panic") + cr.mu.Unlock() + require.True(t, cr.goModuleRetry.mu.TryLock(), "the retry queue mutex must remain usable") + cr.goModuleRetry.mu.Unlock() +} + +func TestGoModuleImportBatchUnlock(t *testing.T) { + for _, route := range []string{"retry", "post_publish"} { + t.Run(route, func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + for i := 0; i <= externalCallMutationChunk; i++ { + file := fmt.Sprintf("a/f%d.go", i) + g.AddNode(&graph.Node{ID: file + "::import::example.com/lib", Kind: graph.KindImport, FilePath: file, RepoPrefix: "a", Language: "go"}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"a": "example.com/consumer", "b": "example.com/lib"}) + probes := 0 + g.afterBatch = func() { + probes++ + available := cr.mu.TryLock() + assert.True(t, available, "the next identity page must not hold the resolve mutex") + if available { + cr.mu.Unlock() + } + } + if route == "retry" { + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + } else { + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("a")) + } + assert.Positive(t, probes) + }) + } +} + +func TestGoModuleImportSameRepoSave(t *testing.T) { + for _, otherRepo := range []bool{false, true} { + t.Run(fmt.Sprint(otherRepo), func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + modules := map[string]string{"a": "example.com/self"} + if otherRepo { + modules["b"] = "example.com/other" + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + } + for i := range 4 { + file := fmt.Sprintf("a/f%d.go", i) + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: file + "::import::example.com/self", Kind: graph.KindImport, FilePath: file, RepoPrefix: "a", Language: "go", StartLine: 2}) + g.AddEdge(&graph.Edge{From: file, To: "a/f0.go", Kind: graph.EdgeImports, FilePath: file, Line: 2}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(modules) + for range 2 { // cold seed, then an ordinary same-repo save + g.hydrated, g.edgeLoads = 0, 0 + cr.ResolveMutationFrontiers(nil, nil, []string{"a/f0.go"}) + assert.Equal(t, 1, g.hydrated, "only the exact definition file may be hydrated") + assert.Zero(t, g.edgeLoads, "same-repo imports must not trigger restore adjacency loads") + assert.Empty(t, cr.goModuleRetry.imports["a"]) + } + }) + } +} + +func TestGoModuleImportFallbackProviderDeletion(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2}) + for _, file := range []string{"b/a.go", "b/z.go"} { + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "b", Language: "go"}) + } + resetImport := func() { + for _, e := range g.GetOutEdges("a/main.go") { + g.RemoveEdge(e.From, e.To, e.Kind) + } + g.AddEdge(&graph.Edge{From: "a/main.go", To: "unresolved::import::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + resetImport() + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + require.Equal(t, "b/a.go", g.GetOutEdges("a/main.go")[0].To) + g.EvictFile("b/a.go") + _, err := cr.ResolveAllContext(t.Context()) // incomplete receipt and watcher fallback + require.NoError(t, err) + var targets []string + for _, e := range g.GetOutEdges("a/main.go") { + if e.Kind == graph.EdgeImports { + targets = append(targets, e.To) + } + } + assert.Equal(t, []string{"b/z.go"}, targets, "fallback must restore the evicted import onto a live representative") + resetImport() + cr.ResolveMutationFrontiers([]string{"a/main.go"}, nil, nil) + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1) + assert.Equal(t, "b/z.go", edges[0].To, "consumer edit must not reuse the deleted package-cache entry") + require.NotNil(t, g.GetNode(edges[0].To)) + }) + } +} + +func TestGoModuleImportIDsDoesNotSeed(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + cr.mu.Lock() + defer cr.mu.Unlock() + cr.goModuleImportIDs("b") + assert.Zero(t, g.importScans, "the ID getter must not run a lock-yielding seed") + assert.Zero(t, g.hydrated) +} + +type goImportCancelStore struct { + graph.Store + cancel context.CancelFunc + batches int +} + +func (s *goImportCancelStore) GetNodesByIDs(ids []string) map[string]*graph.Node { + if s.cancel != nil && len(ids) > 0 && strings.Contains(ids[0], "::import::") { + s.batches++ + s.cancel() + } + return s.Store.GetNodesByIDs(ids) +} + +func TestGoModuleImportRestoreCancellation(t *testing.T) { + g := &goImportCancelStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + for i := range 2*externalCallMutationChunk + 1 { + file := fmt.Sprintf("a/f%d.go", i) + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: file + "::import::example.com/lib", Kind: graph.KindImport, FilePath: file, RepoPrefix: "a", Language: "go", StartLine: 2}) + g.AddEdge(&graph.Edge{From: file, To: "unresolved::import::example.com/lib", Kind: graph.EdgeImports, FilePath: file, Line: 2}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + g.EvictFile("b/lib.go") + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + g.cancel = cancel // cancel when restoration starts its first payload batch + stats, err := cr.ResolveAllContext(ctx) + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, g.batches, "cancellation must stop before the next restoration batch") + assert.Zero(t, stats.Resolved, "the cancelled context must reach the import rebind") +} + +func TestGoModuleImportRestoreSameLine(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, route := range []string{"mutation", "fallback", "post_publish"} { + t.Run(backend+"/"+route, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + // import ( b "example.com/lib/sub"; c "example.net/other/pkg" ) + for _, path := range []string{"example.com/lib/sub", "example.net/other/pkg"} { + g.AddNode(&graph.Node{ID: "a/main.go::import::" + path, Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": path}}) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + } + for _, file := range []string{"b/sub/lib.go", "c/pkg/other.go"} { + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: graph.RepoPrefixOfID(file), Language: "go"}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib", "c": "example.net/other"}) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + require.Len(t, g.GetOutEdges("a/main.go"), 2) + g.EvictFile("b/sub/lib.go") + remaining := g.GetOutEdges("a/main.go") + require.Len(t, remaining, 1) + require.Equal(t, "c/pkg/other.go", remaining[0].To) + g.AddNode(&graph.Node{ID: "b/sub/lib.go", Kind: graph.KindFile, FilePath: "b/sub/lib.go", RepoPrefix: "b", Language: "go"}) + resolve := func() *CrossRepoStats { + switch route { + case "mutation": + return cr.ResolveMutationFrontiers(nil, nil, []string{"b/sub/lib.go"}) + case "fallback": + stats, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + return stats + default: + require.NoError(t, cr.ResolveGoModuleIncomingForRepo("b")) + return nil + } + } + for pass := range 2 { // recovery, then an unchanged provider pass + if stats := resolve(); stats != nil { + assert.Equal(t, 1-pass, stats.Resolved, "only the missing import may be rebound") + } + var targets []string + for _, e := range g.GetOutEdges("a/main.go") { + if e.Kind == graph.EdgeImports { + targets = append(targets, e.To) + assert.True(t, e.CrossRepo) + assert.Equal(t, 2, e.Line) + } + } + assert.ElementsMatch(t, []string{"b/sub/lib.go", "c/pkg/other.go"}, targets, "both same-line imports must survive provider recovery without duplicates") + } + }) + } + } +} + +func TestGoModuleImportRestoreNestedProvider(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, module := range []string{"example.com/lib/v2", "example.com/lib/nested"} { + t.Run(backend+"/"+module, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + g = s + } + path := module + "/pkg" + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::" + path, Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": path}}) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "unresolved::import::" + path, Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + bFile := "b" + strings.TrimPrefix(path, "example.com/lib") + "/lib.go" + for _, file := range []string{bFile, "d/pkg/lib.go"} { + g.AddNode(&graph.Node{ID: file, Kind: graph.KindFile, FilePath: file, RepoPrefix: graph.RepoPrefixOfID(file), Language: "go"}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib", "d": module}) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + check := func(stats *CrossRepoStats) { + t.Helper() + edges := g.GetOutEdges("a/main.go") + require.Len(t, edges, 1, "an import bound to the longer module must not be restored again") + assert.Equal(t, "d/pkg/lib.go", edges[0].To) + assert.Equal(t, graph.EdgeImports, edges[0].Kind) + assert.Equal(t, 2, edges[0].Line) + assert.True(t, edges[0].CrossRepo) + if stats != nil { + assert.Zero(t, stats.Resolved) + assert.Zero(t, stats.CrossRepoEdges) + assert.Empty(t, stats.ByRepo) + } + } + check(nil) + for range 2 { + stats, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + check(stats) + g.EvictFile(bFile) + g.AddNode(&graph.Node{ID: bFile, Kind: graph.KindFile, FilePath: bFile, RepoPrefix: "b", Language: "go"}) + check(cr.ResolveMutationFrontiers(nil, nil, []string{bFile})) + } + }) + } + } +} + +func TestGoModuleImportRestoreLookupBudget(t *testing.T) { + for _, tc := range []struct { + name, target string + line, want int + }{ + {"bound", "b/lib.go", 2, 1}, + {"missing", "", 2, 1}, + {"unresolved", "unresolved::import::example.com/lib", 2, 1}, + {"external", "external::example.com/lib", 2, 1}, + {"dep", "dep::example.com/lib", 2, 2}, + {"unresolved_dep", "unresolved::dep::example.com/lib", 2, 2}, + {"external_call_dep", "external-call::dep::example.com/lib", 2, 2}, + {"other_line_dep", "dep::example.com/lib", 3, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + g.AddNode(&graph.Node{ID: "a/main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"}) + g.AddNode(&graph.Node{ID: "a/main.go::import::example.com/lib", Kind: graph.KindImport, FilePath: "a/main.go", RepoPrefix: "a", Language: "go", StartLine: 2, Meta: map[string]any{"path": "example.com/lib"}}) + g.AddNode(&graph.Node{ID: "b/lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"}) + g.AddEdge(&graph.Edge{From: "a/main.go", To: "unresolved::import::example.com/lib", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.com/lib"}) + require.NoError(t, cr.RetryGoModuleDeps(t.Context())) + g.RemoveEdge("a/main.go", "b/lib.go", graph.EdgeImports) + if tc.target != "" { + g.AddEdge(&graph.Edge{From: "a/main.go", To: tc.target, Kind: graph.EdgeImports, FilePath: "a/main.go", Line: tc.line}) + } + g.hydrated = 0 + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + assert.LessOrEqual(t, g.hydrated, tc.want, "only a same-line dep terminal may need restore payloads in addition to the rebind's single load") + }) + } +} + +// Given import rows above the shared admission ceiling, the incoming leg +// refuses before either parked-edge binding or legacy recovery changes rows. +func TestGoModuleImportIncomingAdmissionRefused(t *testing.T) { + for _, recovery := range []bool{false, true} { + t.Run(fmt.Sprint(recovery), func(t *testing.T) { + source := &graph.Node{ID: "a/main.go", Name: "main.go", Kind: graph.KindFile, FilePath: "a/main.go", RepoPrefix: "a", Language: "go"} + provider := &graph.Node{ID: "b/lib.go", Name: "lib.go", Kind: graph.KindFile, FilePath: "b/lib.go", RepoPrefix: "b", Language: "go"} + edge := &graph.Edge{From: source.ID, To: "unresolved::import::example.org/lib", Kind: graph.EdgeImports, FilePath: source.FilePath, Meta: map[string]any{"go_import": "example.org/lib"}} + g := &goModuleAdmissionStore{Store: graph.New(), edge: edge} + g.AddBatch([]*graph.Node{source, provider}, []*graph.Edge{edge}) + modules := map[string]string{"a": "example.org/app", "b": "example.org/lib"} + state := &GoModuleDepRetry{modules: modules, complete: true, revision: 1, packageModules: modules, packageRevision: 1, importsSeeded: true} + if recovery { + state.imports = map[string]map[string]struct{}{"b": {}} + for i := 0; i <= graph.MaxIncomingSourceCandidateRows; i++ { + state.imports["b"][fmt.Sprint(i)] = struct{}{} + } + } + cr := NewCrossRepo(g) + cr.SetGoModules(modules) + cr.SetGoModuleDepRetry(state, 1) + _, admission := cr.ResolveMutationFrontiersBounded(nil, nil, []string{provider.FilePath}) + require.True(t, admission.Refused) + var limit *graph.BoundedLocalizationLimitError + require.ErrorAs(t, admission.Refusal, &limit) + assert.Equal(t, "unresolved::import::example.org/lib", g.GetOutEdges(source.ID)[0].To) + }) + } +} + +func TestGoModuleImportFallbackStatsAreUnresolved(t *testing.T) { + for _, target := range []string{"dep::example.org/lib", "external::example.org/lib"} { + for _, priorCount := range []string{"none", "unresolved", "resolved"} { + t.Run(target+"/"+priorCount, func(t *testing.T) { + g := graph.New() + old := &graph.Edge{From: "a/main.go", To: "b/lib.go", Kind: graph.EdgeImports, FilePath: "a/main.go", Line: 2} + g.AddEdge(old) + fresh := *old + fresh.To = target + stats := &CrossRepoStats{ByRepo: map[string]int{}} + counted := map[graph.EdgeIdentity]bool{} + if priorCount != "none" { + resolved := priorCount == "resolved" + counted[graph.EdgeIdentityFor(old)] = resolved + if resolved { + stats.Resolved = 1 + } else { + stats.Unresolved = 1 + } + } + NewCrossRepo(g).applyGoModuleImportReindexes([]graph.EdgeReindex{{Edge: &fresh, OldTo: old.To}}, stats, counted) + require.Zero(t, stats.Resolved) + require.Equal(t, 1, stats.Unresolved) + require.Zero(t, stats.CrossRepoEdges) + }) + } + } +} + +func TestInvalidatePackageRepoKeepsOtherProviders(t *testing.T) { + // Given one shared retry object with independent provider projections. + state := &GoModuleDepRetry{complete: true, packages: map[string]map[string]graph.FileNodeIdentity{ + "b": {"b": {ID: "b/lib.go", FilePath: "b/lib.go", RepoPrefix: "b"}}, + "c": {"c": {ID: "c/lib.go", FilePath: "c/lib.go", RepoPrefix: "c"}}, + }} + // When only C's projection is invalidated. + state.InvalidatePackageRepo("c") + // Then B's cached identity and the shared retry state are preserved. + require.NotContains(t, state.packages, "c") + require.Equal(t, graph.FileNodeIdentity{ID: "b/lib.go", FilePath: "b/lib.go", RepoPrefix: "b"}, state.packages["b"]["b"]) + require.True(t, state.complete, "package invalidation must not replace the terminal-sweep policy") +} + +func TestGoModuleImportRestorePreservesVendoredBinding(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := goImportReviewStore(t, backend) + for _, file := range []string{"consumer/main.go", "consumer/vendor/example.org/lib/lib.go", "provider/lib.go"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), Language: "go"}) + } + g.AddNode(&graph.Node{ID: "consumer/main.go::import::example.org/lib", FilePath: "consumer/main.go", StartLine: 2, Kind: graph.KindImport, RepoPrefix: "consumer", Language: "go", Meta: map[string]any{"path": "example.org/lib"}}) + g.AddEdge(&graph.Edge{From: "consumer/main.go", FilePath: "consumer/main.go", To: "consumer/vendor/example.org/lib/lib.go", Kind: graph.EdgeImports, Line: 2, Origin: graph.OriginASTResolved}) + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"provider": "example.org/lib"}) + for pass := 0; pass < 2; pass++ { + _, err := cr.ResolveAllContext(t.Context()) + require.NoError(t, err) + rows := outEdgesOfKind(g, "consumer/main.go", graph.EdgeImports) + require.Len(t, rows, 1) + require.Equal(t, "consumer/vendor/example.org/lib/lib.go", rows[0].To) + require.False(t, rows[0].CrossRepo) + } + }) + } +} + +func TestGoModuleImportUsesBatchedFileAndStatsNodes(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &resolverBatchCountingStore{Store: goImportReviewStore(t, backend)} + g.AddBatch([]*graph.Node{{ID: "consumer/main.go", FilePath: "consumer/main.go", Kind: graph.KindFile, RepoPrefix: "consumer", Language: "go"}, {ID: "provider/lib.go", FilePath: "provider/lib.go", Kind: graph.KindFile, RepoPrefix: "provider", Language: "go"}}, nil) + const count = 64 + for i := 0; i < count; i++ { + g.AddEdge(&graph.Edge{From: "consumer/main.go", FilePath: "consumer/main.go", To: "external::example.org/lib", Kind: graph.EdgeImports, Line: i + 1}) + } + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"provider": "example.org/lib"}) + for pass := 0; pass < 2; pass++ { + g.getNodeCalls, g.getFileNodesByPathsCalls = 0, 0 + stats := &CrossRepoStats{ByRepo: map[string]int{}} + require.NoError(t, cr.resolveGoModuleImports(t.Context(), []string{"consumer/main.go"}, stats, nil)) + require.Zero(t, g.getNodeCalls, "reuse loaded caller files and batch statistics targets") + require.Equal(t, 1, g.getFileNodesByPathsCalls) + rows := outEdgesOfKind(g, "consumer/main.go", graph.EdgeImports) + require.Len(t, rows, count) + for _, e := range rows { + require.Equal(t, "provider/lib.go", e.To) + } + if pass == 0 { + require.Equal(t, count, stats.ByRepo["provider"]) + } else { + require.Zero(t, stats.Resolved) + require.Empty(t, stats.ByRepo) + } + } + }) + } +} + +func TestGoModuleImportCounterExactBindingKinds(t *testing.T) { + g := &goImportCountingStore{Store: graph.New()} + for range g.RepoNodeIdentitiesSeq(nil, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { + } + require.Equal(t, 1, g.bindingRechecks) + g.bindingRechecks = 0 + for range g.RepoNodeIdentitiesSeq(nil, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindImport) { + } + require.Zero(t, g.bindingRechecks, "four unrelated kinds must not count as the binding recheck") +} diff --git a/internal/resolver/go_module_pending_drain_test.go b/internal/resolver/go_module_pending_drain_test.go new file mode 100644 index 000000000..fdc6e330f --- /dev/null +++ b/internal/resolver/go_module_pending_drain_test.go @@ -0,0 +1,61 @@ +package resolver + +import ( + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "iter" + "testing" +) + +type goPendingDrainStore struct { + *goModuleDroppingStore + identitySweeps int +} + +func (s *goPendingDrainStore) RepoNodeIdentitiesSeq(repos []string, kinds ...graph.NodeKind) iter.Seq[graph.RepoNodeIdentity] { + s.identitySweeps++ + return graph.RepoNodeIdentitiesSeq(s.Store, repos, kinds...) +} + +func TestGoModulePendingDrainsOnNonGoMutation(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &goPendingDrainStore{goModuleDroppingStore: &goModuleDroppingStore{Store: goImportReviewStore(t, backend), drop: true}} + g.AddNode(&graph.Node{ID: "a/main.go::Caller", FilePath: "a/main.go", Kind: graph.KindFunction, Name: "Caller", Language: "go", RepoPrefix: "a", WorkspaceID: "shared"}) + g.AddNode(&graph.Node{ID: "b/lib.go::Fn", FilePath: "b/lib.go", Kind: graph.KindFunction, Name: "Fn", Language: "go", RepoPrefix: "b", WorkspaceID: "shared"}) + edge := &graph.Edge{From: "a/main.go::Caller", FilePath: "a/main.go", To: "dep::example.org/lib::Fn", Kind: graph.EdgeCalls, Line: 1} + g.AddEdge(edge) + state := &GoModuleDepRetry{} + makeResolver := func() *CrossRepoResolver { + cr := NewCrossRepo(g) + cr.SetGoModules(map[string]string{"b": "example.org/lib"}) + cr.SetGoModuleDepRetry(state, 1) + return cr + } + require.NoError(t, makeResolver().ResolveDeferredGoModuleEdges([]*graph.Edge{edge})) + require.Len(t, state.pending, 1) + g.drop = true // The next drain must preserve a second refusal. + g.identitySweeps = 0 + _, admission := makeResolver().ResolveMutationFrontiersBounded(nil, nil, []string{"a/README.md"}) + require.False(t, admission.Refused) + require.Len(t, state.pending, 1) + require.False(t, g.drop, "the queued batch must have been attempted") + require.Equal(t, "dep::example.org/lib::Fn", g.GetOutEdges(edge.From)[0].To) + _, admission = makeResolver().ResolveMutationFrontiersBounded(nil, nil, []string{"a/app.py"}) + require.False(t, admission.Refused) + require.Empty(t, state.pending) + var calls []*graph.Edge + for _, row := range g.GetOutEdges(edge.From) { + if row.Kind == graph.EdgeCalls { + calls = append(calls, row) + } + } + require.Len(t, calls, 1) + require.Equal(t, "b/lib.go::Fn", calls[0].To) + require.True(t, calls[0].CrossRepo) + require.False(t, state.complete, "a queue drain must not certify the legacy sweep") + makeResolver().ResolveMutationFrontiersBounded(nil, nil, []string{"a/app.ts"}) + require.Zero(t, g.identitySweeps, "non-Go saves may drain, not reseed or recheck all repositories") + }) + } +} diff --git a/internal/resolver/go_modules.go b/internal/resolver/go_modules.go index f34424afd..37ded3b3a 100644 --- a/internal/resolver/go_modules.go +++ b/internal/resolver/go_modules.go @@ -158,7 +158,7 @@ func goModuleCandidate(modules map[string]string, importPath string, candidates ambiguous := false priority := -1 for _, c := range candidates { - if (c.Language != "" && c.Language != "go") || (c.Kind != graph.KindFunction && c.Kind != graph.KindType && c.Kind != graph.KindInterface) { + if (c.Language != "" && c.Language != "go") || (c.Kind != graph.KindFunction && c.Kind != graph.KindType && c.Kind != graph.KindInterface && c.Kind != graph.KindFile) { continue // a package selector does not name a receiver method } module := modules[c.RepoPrefix] diff --git a/internal/resolver/go_modules_cross_repo.go b/internal/resolver/go_modules_cross_repo.go index a9955d8b0..0aeac2542 100644 --- a/internal/resolver/go_modules_cross_repo.go +++ b/internal/resolver/go_modules_cross_repo.go @@ -7,12 +7,18 @@ import ( "strings" "sync" + "go.uber.org/zap" + "github.com/zzet/gortex/internal/graph" ) func (cr *CrossRepoResolver) SetGoModules(modules map[string]string) { cr.goModules = maps.Clone(modules) cr.goModulePaths = indexGoModulePaths(modules) + cr.goModuleRepos = make(map[string][]string) + for repo, module := range modules { + cr.goModuleRepos[module] = append(cr.goModuleRepos[module], repo) + } } func (cr *CrossRepoResolver) resolveGoExtern(e *graph.Edge, spec string, stats *CrossRepoStats) { @@ -43,6 +49,10 @@ func (cr *CrossRepoResolver) resolveGoExtern(e *graph.Edge, spec string, stats * } } +var newGoModuleImportRecoveryBudget = func() *graph.IncomingSourceBudget { + return &graph.IncomingSourceBudget{} +} + // GoModuleDepRetry remembers the last completed legacy-terminal sweep for a // store's module/config revision. MultiIndexer shares it across pass objects. type GoModuleDepRetry struct { @@ -52,6 +62,12 @@ type GoModuleDepRetry struct { revision uint64 complete bool pending map[graph.EdgeIdentity]*graph.Edge + // These compact import indexes are protected by the store resolve mutex. + packageModules map[string]string + packageRevision uint64 + packages map[string]map[string]graph.FileNodeIdentity + imports map[string]map[string]struct{} + importsSeeded bool } func (cr *CrossRepoResolver) SetGoModuleDepRetry(state *GoModuleDepRetry, revision uint64) { @@ -60,7 +76,7 @@ func (cr *CrossRepoResolver) SetGoModuleDepRetry(state *GoModuleDepRetry, revisi // retryGoModuleDeps visits only external terminals, filters by tracked module // identity, and bounds mutation batches. Symbol lookup is memoized per batch. -func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *CrossRepoStats) error { +func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *CrossRepoStats, importsRestored bool) error { if err := ctx.Err(); err != nil { return err } @@ -89,31 +105,21 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross if seedBindings || changedReplacements { previous = nil } - if err := cr.recheckGoModuleBindings(ctx, previous, stats); err != nil { + if err := cr.recheckGoModuleBindings(ctx, previous, stats, importsRestored); err != nil { return err } } + cr.retryPendingGoModuleEdges(stats) state.mu.Lock() - pending := state.pending - state.pending = nil - state.mu.Unlock() - var deferred []*graph.Edge - for _, edge := range pending { - deferred = append(deferred, edge) - if len(deferred) == externalCallMutationChunk { - cr.resolveGoModuleEdgesLocked(deferred, stats) - deferred = deferred[:0] - } - } - cr.resolveGoModuleEdgesLocked(deferred, stats) - state.mu.Lock() - defer state.mu.Unlock() if len(cr.goModules) == 0 { - state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, true + state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, cr.goModuleImportsDeferred == 0 state.replacements = maps.Clone(cr.goModuleReplacements) + state.mu.Unlock() return nil } - if state.complete && state.revision == cr.goModuleRevision && maps.Equal(state.modules, cr.goModules) && !changedReplacements { + complete := state.complete && state.revision == cr.goModuleRevision && maps.Equal(state.modules, cr.goModules) && !changedReplacements + state.mu.Unlock() + if complete { return nil } var batch []graph.EdgeReindex @@ -126,10 +132,7 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross if len(batch) > 0 { if !cr.applyGoModuleReindexes(batch) { cr.logger.Warn("Go module terminal batch dropped; queued for a later module pass") - if state.pending == nil { - state.pending = make(map[graph.EdgeIdentity]*graph.Edge) - } - maps.Copy(state.pending, originals) + cr.queueGoModuleEdges(originals) } batch = nil clear(originals) @@ -212,11 +215,35 @@ func (cr *CrossRepoResolver) retryGoModuleDeps(ctx context.Context, stats *Cross if err := ctx.Err(); err != nil { return err } - state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, true + state.mu.Lock() + defer state.mu.Unlock() + state.modules, state.revision, state.complete = maps.Clone(cr.goModules), cr.goModuleRevision, cr.goModuleImportsDeferred == 0 state.replacements = maps.Clone(cr.goModuleReplacements) return nil } +// retryPendingGoModuleEdges drains exact dropped batches without opening the +// revision-wide registry seed or legacy binding sweep. The resolve mutex is held. +func (cr *CrossRepoResolver) retryPendingGoModuleEdges(stats *CrossRepoStats) { + state := cr.goModuleRetry + if state == nil { + return + } + state.mu.Lock() + pending := state.pending + state.pending = nil + state.mu.Unlock() + var deferred []*graph.Edge + for _, edge := range pending { + deferred = append(deferred, edge) + if len(deferred) == externalCallMutationChunk { + cr.resolveGoModuleEdgesLocked(deferred, stats) + deferred = deferred[:0] + } + } + cr.resolveGoModuleEdgesLocked(deferred, stats) +} + // goModuleIncomingEdges probes only package-qualified placeholders owned by the // changed Go definitions. Bare-name frontier keys cannot find these imports. func (cr *CrossRepoResolver) goModuleIncomingEdges(nodesByFile map[string][]*graph.Node, definitionFiles []string) []*graph.Edge { @@ -234,7 +261,7 @@ func (cr *CrossRepoResolver) goModuleIncomingEdgesBounded(nodesByFile map[string synthetic := make(map[string][]*graph.Node) for _, file := range definitionFiles { for _, n := range nodesByFile[file] { - if n == nil || n.Language != "go" || n.Name == "" || (n.Kind != graph.KindFunction && n.Kind != graph.KindType && n.Kind != graph.KindInterface) { + if n == nil || n.Language != "go" || n.Name == "" || (n.Kind != graph.KindFunction && n.Kind != graph.KindType && n.Kind != graph.KindInterface && n.Kind != graph.KindFile) { continue } module := cr.goModules[n.RepoPrefix] @@ -243,6 +270,12 @@ func (cr *CrossRepoResolver) goModuleIncomingEdgesBounded(nodesByFile map[string continue } spec := module + strings.TrimPrefix(dir, n.RepoPrefix) + "::" + n.Name + if n.Kind == graph.KindFile { + spec = module + strings.TrimPrefix(dir, n.RepoPrefix) + providers[spec] = append(providers[spec], n) + keys[graph.UnresolvedMarker+"import::"+spec] = struct{}{} + continue + } providers[spec] = append(providers[spec], n) keys[graph.UnresolvedMarker+"extern::"+spec] = struct{}{} keys["dep::"+spec] = struct{}{} @@ -292,12 +325,15 @@ func (cr *CrossRepoResolver) goModuleIncomingEdgesBounded(nodesByFile map[string } spec := goModuleEdgeSpec(e) sep := strings.LastIndex(spec, "::") - if sep < 0 { + importPath := spec + if sep >= 0 { + importPath = spec[:sep] + } else if e.Kind != graph.EdgeImports { continue } source := sourceOf(e.From) for _, provider := range providers[spec] { - if crossWorkspaceEligible(cr.crossWorkspaceLookup, candidateWorkspaceID(source), candidateWorkspaceID(provider), spec[:sep]) { + if crossWorkspaceEligible(cr.crossWorkspaceLookup, candidateWorkspaceID(source), candidateWorkspaceID(provider), importPath) { edges = append(edges, e) break } @@ -395,6 +431,17 @@ func goModuleEdgeSpec(e *graph.Edge) string { } func (cr *CrossRepoResolver) resolveGoModuleEdgesLocked(edges []*graph.Edge, stats *CrossRepoStats) { + var importFiles []string + seenImports := make(map[string]bool) + for _, e := range edges { + if e != nil && e.Kind == graph.EdgeImports && !seenImports[e.FilePath] { + seenImports[e.FilePath] = true + importFiles = append(importFiles, e.FilePath) + } + } + if len(importFiles) > 0 { + _ = cr.resolveGoModuleImports(context.Background(), importFiles, stats, nil) + } var batch []graph.EdgeReindex names := make(map[[2]string][]*graph.Node) sources := make(map[string]*graph.Node) @@ -459,15 +506,7 @@ func (cr *CrossRepoResolver) resolveGoModuleEdgesLocked(edges []*graph.Edge, sta batch = cr.filterLiveReindex(batch) if !cr.applyGoModuleReindexes(batch) { cr.logger.Warn("Go module rebind batch dropped; queued for a later module pass") - if cr.goModuleRetry == nil { - cr.goModuleRetry = &GoModuleDepRetry{} - } - cr.goModuleRetry.mu.Lock() - if cr.goModuleRetry.pending == nil { - cr.goModuleRetry.pending = make(map[graph.EdgeIdentity]*graph.Edge) - } - maps.Copy(cr.goModuleRetry.pending, originals) - cr.goModuleRetry.mu.Unlock() + cr.queueGoModuleEdges(originals) return } if stats == nil { @@ -488,15 +527,55 @@ func (cr *CrossRepoResolver) resolveGoModuleEdgesLocked(edges []*graph.Edge, sta // RetryGoModuleDeps is the revision-keyed migration used after publishing a module. func (cr *CrossRepoResolver) RetryGoModuleDeps(ctx context.Context) error { - cr.mu.Lock() - defer cr.mu.Unlock() - return cr.retryGoModuleDeps(ctx, &CrossRepoStats{ByRepo: make(map[string]int)}) + stats := &CrossRepoStats{ByRepo: make(map[string]int)} + if err := func() error { + cr.mu.Lock() + defer cr.mu.Unlock() + cr.goModuleImportsDeferred = 0 + return cr.retryGoModuleDeps(ctx, stats, false) + }(); err != nil { + return err + } + return cr.resolveGoModuleImportSweep(ctx, stats) } -// ResolveGoModuleIncomingForRepo probes exact targets for the provider's definitions. -// Reindexing has no changed-symbol receipt, so stream only its definition IDs and -// hydrate at most one batch; never load the repository's nodes or edges as a set. +// ResolveGoModuleIncomingForRepo probes the provider's definitions and retained +// consumer import statements. Stream identities and hydrate at most one batch; +// never load the repository's nodes or edges as a set. func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { + if len(cr.goModules) == 0 { + return nil + } + if err := cr.resolveGoModuleImportSweep(context.Background(), nil); err != nil { + return err + } + if cr.goModules[repo] != "" && len(cr.graph.RepoPrefixes()) >= 2 { + var importIDs []string + func() { + cr.mu.Lock() + defer cr.mu.Unlock() + importIDs = cr.goModuleImportIDs(repo) + }() + for start := 0; start < len(importIDs); start += externalCallMutationChunk { + if err := func() error { + cr.mu.Lock() + defer cr.mu.Unlock() + return cr.restoreGoModuleImportBatch(context.Background(), repo, importIDs[start:min(start+externalCallMutationChunk, len(importIDs))], nil) + }(); err != nil { + return err + } + } + } + for files, err := range cr.goModuleImportFileBatches(context.Background(), []string{repo}) { + if err != nil { + return err + } + func() { + cr.mu.Lock() + defer cr.mu.Unlock() + _ = cr.resolveGoModuleImports(context.Background(), files, nil, nil) + }() + } if cr.goModules[repo] == "" { return nil } @@ -518,7 +597,7 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { ids = ids[:0] return cr.ResolveDeferredGoModuleEdges(edges) } - for n := range graph.RepoNodeIdentitiesSeq(cr.graph, []string{repo}, graph.KindFunction, graph.KindType, graph.KindInterface) { + for n := range graph.RepoNodeIdentitiesSeq(cr.graph, []string{repo}, graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile) { ids = append(ids, n.ID) if len(ids) == externalCallMutationChunk { if err := flush(); err != nil { @@ -532,9 +611,11 @@ func (cr *CrossRepoResolver) ResolveGoModuleIncomingForRepo(repo string) error { return nil } +var goModuleBindingRecheckKinds = []graph.NodeKind{graph.KindFunction, graph.KindType, graph.KindInterface, graph.KindFile} + // Re-check only marked incoming calls into providers affected by identity edits. // The compact definition stream avoids loading a whole repo's nodes or edges. -func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previous map[string]string, stats *CrossRepoStats) error { +func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previous map[string]string, stats *CrossRepoStats, importsRestored bool) error { changed := make(map[string]bool) for repo, module := range previous { if cr.goModules[repo] != module { @@ -581,7 +662,7 @@ func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previo cr.resolveGoModuleEdgesLocked(edges, stats) ids = ids[:0] } - for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, graph.KindFunction, graph.KindType, graph.KindInterface) { + for n := range graph.RepoNodeIdentitiesSeq(cr.graph, repos, goModuleBindingRecheckKinds...) { if err := ctx.Err(); err != nil { return err } @@ -593,6 +674,33 @@ func (cr *CrossRepoResolver) recheckGoModuleBindings(ctx context.Context, previo if len(ids) > 0 { flush() } + if importsRestored { + return ctx.Err() + } + // A changed module can make another provider's fallback imports bindable, + // even when that provider is outside the saved-file frontier. + importBudget := newGoModuleImportRecoveryBudget() + for _, repo := range repos { + if cr.goModules[repo] == "" { + continue + } + importIDs := cr.goModuleImportIDs(repo) + if len(importIDs) > importBudget.Remaining() { + cr.goModuleImportsDeferred += len(importIDs) + cr.logger.Warn("Go module import identity recheck deferred to full pass", zap.Int("imports", len(importIDs)), zap.Int("remaining", importBudget.Remaining())) + continue + } + if err := importBudget.Charge(len(importIDs)); err != nil { + cr.goModuleImportsDeferred += len(importIDs) + cr.logger.Warn("Go module import identity recheck deferred to full pass", zap.Error(err)) + continue + } + for start := 0; start < len(importIDs); start += externalCallMutationChunk { + if err := cr.restoreGoModuleImportBatch(ctx, repo, importIDs[start:min(start+externalCallMutationChunk, len(importIDs))], stats); err != nil { + return err + } + } + } return ctx.Err() } @@ -867,3 +975,16 @@ func (state *GoModuleDepRetry) Invalidate() { defer state.mu.Unlock() state.complete = false } + +func (cr *CrossRepoResolver) queueGoModuleEdges(originals map[graph.EdgeIdentity]*graph.Edge) { + if cr.goModuleRetry == nil { + cr.goModuleRetry = &GoModuleDepRetry{} + } + state := cr.goModuleRetry + state.mu.Lock() + defer state.mu.Unlock() + if state.pending == nil { + state.pending = make(map[graph.EdgeIdentity]*graph.Edge) + } + maps.Copy(state.pending, originals) +} diff --git a/internal/resolver/incremental_attribution_cache.go b/internal/resolver/incremental_attribution_cache.go index 7ea55d990..aa83cc4fe 100644 --- a/internal/resolver/incremental_attribution_cache.go +++ b/internal/resolver/incremental_attribution_cache.go @@ -107,6 +107,8 @@ func (r *Resolver) runFileAttributionPassesForFilesLocked(frontier incrementalFi // before the builtin and external materialisation passes inspect them. r.flushIncrementalAttributionReindexes() legs.lap("flush") + r.attributePythonModuleImportsForFiles(frontier.paths) + legs.lap("python_modules") if !r.graphHasLanguage("go") { return diff --git a/internal/resolver/module_attribution.go b/internal/resolver/module_attribution.go index b9c7f79e1..741a9afcc 100644 --- a/internal/resolver/module_attribution.go +++ b/internal/resolver/module_attribution.go @@ -1,6 +1,8 @@ package resolver import ( + "iter" + "slices" "strings" "github.com/zzet/gortex/internal/graph" @@ -40,18 +42,45 @@ func (r *Resolver) attributeNonGoModuleImports() { if !r.graphHasLanguage("python") && !r.graphHasLanguage("dart") { return } - fileLang := r.collectFileLanguages() + r.attributeNonGoModuleImportEdges(r.graph.EdgesByKind(graph.EdgeImports), graph.EdgesLightSeq(r.graph, graph.EdgeDependsOnModule), r.collectFileLanguages()) +} + +// Python saves collect their cached frontier before flushing attribution writes. +func (r *Resolver) attributePythonModuleImportsForFiles(files []string) { + languages := make(map[string]string) + var edges []*graph.Edge + for _, file := range files { + if !isPythonSourcePath(file) { + continue + } + for _, node := range r.incrementalFileNodes(file) { + if node.Kind == graph.KindFile && node.Language == "python" { + languages[node.ID] = node.Language + edges = append(edges, r.fileOutEdges(file)...) + break + } + } + } + if len(languages) == 0 { + return + } + r.attributeNonGoModuleImportEdges(slices.Values(edges), slices.Values(edges), languages) +} + +func (r *Resolver) attributeNonGoModuleImportEdges(imports, dependencies iter.Seq[*graph.Edge], fileLang map[string]string) { type pendingEdge struct { - edge *graph.Edge - oldTo string - moduleID string + edge *graph.Edge + oldTo string + moduleID string + targetIdentity graph.EdgeIdentity } var rewrites []pendingEdge + targetSites := make(map[graph.EdgeIdentity]int) moduleSeeds := map[string]moduleSeed{} dependsSeen := map[string]map[string]struct{}{} // fileID → set of moduleIDs - for e := range r.graph.EdgesByKind(graph.EdgeImports) { - if !strings.HasPrefix(e.To, "external::") { + for e := range imports { + if e.Kind != graph.EdgeImports || !strings.HasPrefix(e.To, "external::") { continue } // Scoped warm pass: an unchanged repo's external imports were already @@ -68,10 +97,15 @@ func (r *Resolver) attributeNonGoModuleImports() { if !ok { continue } + projected := *e + projected.To = moduleID + identity := graph.EdgeIdentityFor(&projected) + targetSites[identity]++ rewrites = append(rewrites, pendingEdge{ - edge: e, - oldTo: e.To, - moduleID: moduleID, + edge: e, + oldTo: e.To, + moduleID: moduleID, + targetIdentity: identity, }) moduleSeeds[moduleID] = moduleSeed{ id: moduleID, @@ -120,7 +154,10 @@ func (r *Resolver) attributeNonGoModuleImports() { // read on every backend, plus a Go-side map build that turns // the per-rewrite check into a constant-time lookup. existingDepends := make(map[string]map[string]struct{}) - for e := range graph.EdgesLightSeq(r.graph, graph.EdgeDependsOnModule) { + for e := range dependencies { + if e.Kind != graph.EdgeDependsOnModule { + continue + } // Only the rewrites' own callers (in scope) are queried below, so a // scoped pass needs the dedupe index for just those repos. if !r.edgeFromInScope(e.From) { @@ -144,7 +181,17 @@ func (r *Resolver) attributeNonGoModuleImports() { } dependsBatch := make([]*graph.Edge, 0, dependsBatchCapacity) for _, p := range rewrites { - p.edge.To = p.moduleID + if strings.HasPrefix(p.moduleID, "module::pypi:") { + if p.edge.Meta == nil { + p.edge.Meta = map[string]any{} + } + p.edge.Meta["python_import"] = strings.TrimPrefix(p.oldTo, "external::") + } + // Distinct Python spellings on one line must survive until tracked retry. + // Keep their external identities, while still recording module evidence. + if !strings.HasPrefix(p.moduleID, "module::pypi:") || targetSites[p.targetIdentity] == 1 { + p.edge.To = p.moduleID + } p.edge.Origin = graph.OriginASTResolved reindexBatch = append(reindexBatch, graph.EdgeReindex{Edge: p.edge, OldTo: p.oldTo}) diff --git a/internal/resolver/module_attribution_test.go b/internal/resolver/module_attribution_test.go index e214f9713..3f9290e67 100644 --- a/internal/resolver/module_attribution_test.go +++ b/internal/resolver/module_attribution_test.go @@ -267,6 +267,12 @@ func TestAttributeNonGo_BatchedWritesPreserveDedupeAndOrdering(t *testing.T) { require.NotNil(t, base.GetNode("module::pypi:requests")) for i, edge := range imports { want := "module::pypi:numpy" + if i == 0 { + want = "external::numpy" + } + if i == 1 { + want = "external::numpy.linalg" + } if i == len(imports)-1 { want = "module::pypi:requests" } @@ -279,9 +285,8 @@ func TestAttributeNonGo_BatchedWritesPreserveDedupeAndOrdering(t *testing.T) { assert.Len(t, outEdgesOfKind(base, "app/c.py", graph.EdgeDependsOnModule), 1, "existingDepends must suppress an already persisted dependency") assert.Len(t, outEdgesOfKind(base, "app/d.py", graph.EdgeDependsOnModule), 1) - // Both app/a.py imports share line 1, so reindexing them to the same - // module converges them onto one edge identity. - assert.Equal(t, 8, base.EdgeCount(), "four import identities plus four dependency identities") + // Same-line spellings keep their distinct identities before tracked retry. + assert.Equal(t, 9, base.EdgeCount(), "five import identities plus four dependency identities") } func TestAttributeNonGo_BoundsModuleAndDependencyWriteBatches(t *testing.T) { diff --git a/internal/resolver/python_absolute_imports.go b/internal/resolver/python_absolute_imports.go index 2767aa2da..09a6c94b1 100644 --- a/internal/resolver/python_absolute_imports.go +++ b/internal/resolver/python_absolute_imports.go @@ -58,6 +58,15 @@ func pythonModuleFileMatches(filePath, module string) bool { return false } +// IsPythonTestPackageFile identifies the supported root/src test packages, +// relative to the provider rather than its repository alias. +func IsPythonTestPackageFile(filePath, repoPrefix string) bool { + relative := strings.TrimPrefix(filePath, repoPrefix+"/") + relative = strings.TrimPrefix(relative, "src/") + top, _, nested := strings.Cut(relative, "/") + return nested && (top == "tests" || top == "test") +} + // resolvePythonModuleExtern resolves a Python caller's extern edge onto the // module-level function or class it imports. It tries the two readings of a // dotted import path in turn — the symbol defined in the module importPath @@ -66,6 +75,12 @@ func pythonModuleFileMatches(filePath, module string) bool { // ambiguous layout keeps the previous behaviour. Methods never match: an // import binds module attributes, not class members. func (r *Resolver) resolvePythonModuleExtern(e *graph.Edge, importPath, symbol, callerRepo string, stats *ResolveStats) bool { + return resolvePythonModuleExtern(e, importPath, symbol, callerRepo, stats, func(name string) ([]*graph.Node, error) { + return r.cachedFindExternNodesByName(name, e) + }) +} + +func resolvePythonModuleExtern(e *graph.Edge, importPath, symbol, callerRepo string, stats *ResolveStats, find func(string) ([]*graph.Node, error)) bool { if !isPythonSourcePath(e.FilePath) || !isDottedPythonModule(importPath) { return false } @@ -75,13 +90,16 @@ func (r *Resolver) resolvePythonModuleExtern(e *graph.Edge, importPath, symbol, readings = append(readings, reading{importPath[:len(importPath)-len(name)-1], name}) } for _, rd := range readings { - candidates, err := r.cachedFindExternNodesByName(rd.name, e) + candidates, err := find(rd.name) if err != nil { return false } var match *graph.Node ambiguous := false for _, c := range candidates { + if c.RepoPrefix != callerRepo && IsPythonTestPackageFile(c.FilePath, c.RepoPrefix) { + continue + } if c.Kind != graph.KindFunction && c.Kind != graph.KindType { continue } diff --git a/internal/resolver/python_import_call_retry_test.go b/internal/resolver/python_import_call_retry_test.go new file mode 100644 index 000000000..5ca8ac059 --- /dev/null +++ b/internal/resolver/python_import_call_retry_test.go @@ -0,0 +1,207 @@ +package resolver + +import ( + "context" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestTrackedPythonImportReopensOnlyNewlyLinkedCalls(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file := "consumer/app.py" + for _, path := range []string{file, "consumer/other.py", "outsider/app.py"} { + g.AddNode(&graph.Node{ID: path, Kind: graph.KindFile, FilePath: path}) + } + eligible := "external-call::dep::PIL.image.f" + for i, tc := range []struct { + file, target string + kind graph.EdgeKind + }{ + {file, eligible, graph.EdgeCalls}, + {file, "external-call::dep::other.f", graph.EdgeCalls}, + {file, "external-call::dep::PIL.image_extra.f", graph.EdgeCalls}, + {file, "provider/src/PIL/image.py::f", graph.EdgeCalls}, + {file, eligible, graph.EdgeReferences}, + {"consumer/other.py", eligible, graph.EdgeCalls}, + {"outsider/app.py", eligible, graph.EdgeCalls}, + {file, "dep::PIL.image::f", graph.EdgeCalls}, + {file, "dep::other::f", graph.EdgeCalls}, + {file, "dep::PIL.image_extra::f", graph.EdgeCalls}, + {file, "dep::PIL.image::f", graph.EdgeReferences}, + } { + g.AddEdge(&graph.Edge{From: tc.file, FilePath: tc.file, To: tc.target, Kind: tc.kind, Line: i + 1, Meta: map[string]any{"external_call": true, PythonCallStubMeta: "unresolved::extern::PIL.image::f"}}) + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: eligible, Kind: graph.EdgeCalls, Line: 12}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: "dep::PIL.image::f", Kind: graph.EdgeCalls, Line: 13}) + linked := &graph.Edge{From: file, FilePath: file, To: "provider/src/PIL/image.py", Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{PythonImportStubMeta: "module::pypi:PIL", "python_import": "PIL.image"}} + unmarked := &graph.Edge{From: "consumer/other.py", FilePath: "consumer/other.py", To: linked.To, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{"python_import": "PIL.image"}} + unlinked := &graph.Edge{From: "outsider/app.py", FilePath: "outsider/app.py", To: "module::pypi:PIL", Kind: graph.EdgeImports, + Meta: linked.Meta} + cr := NewCrossRepo(g) + reopened := cr.restubCallsForLinkedPythonImports([]graph.EdgeReindex{{Edge: linked}, {Edge: unmarked}, {Edge: unlinked}}) + require.Len(t, reopened, 2) + for _, call := range reopened { + require.Equal(t, file, call.FilePath) + require.Equal(t, "unresolved::extern::PIL.image::f", call.To) + require.NotContains(t, call.Meta, "external_call") + } + for _, edges := range g.GetOutEdgesByNodeIDs([]string{file, "consumer/other.py", "outsider/app.py"}) { + for _, edge := range edges { + if edge.Line != 1 && edge.Line != 8 { + require.NotEqual(t, "unresolved::extern::PIL.image::f", edge.To) + } + } + } + }) + } +} + +func TestTrackedPythonImportFullPassStreamsReopenedCalls(t *testing.T) { + g := graph.New() + for _, file := range []string{"consumer/app.py", "provider/PIL/__init__.py", "provider/PIL/image.py"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), WorkspaceID: "shared"}) + } + importEdge := &graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeImports, To: "external::PIL.image", Line: 1} + g.AddEdge(importEdge) + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeCalls, To: "external-call::dep::PIL.image.f", Line: 2, Meta: map[string]any{PythonCallStubMeta: "unresolved::extern::PIL.image::f"}}) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + return &PythonPackageLookup{HasTop: func(top string) bool { return top == "PIL" }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", RepoPrefix: "provider", WorkspaceID: "shared"}}} + }} + }) + calls, err := cr.resolveTrackedPythonImports(context.Background(), slices.Values([]*graph.Edge{importEdge}), &CrossRepoStats{ByRepo: map[string]int{}}, false) + require.NoError(t, err) + require.Empty(t, calls, "the full pass must not retain call metadata across import pages") + out := outEdgesOfKind(g, "consumer/app.py", graph.EdgeCalls) + require.Len(t, out, 1) + require.Equal(t, "unresolved::extern::PIL.image::f", out[0].To, "the full pass's pending stream will admit this reopened call") +} + +func TestTrackedPythonModuleCallKeepsImportAndWorkspaceEvidence(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, scenario := range []string{"module", "from_alias", "unreachable", "foreign_workspace", "wrong_module", "method", "other_provider"} { + t.Run(backend+"/"+scenario, func(t *testing.T) { + var g graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + g = db + } + file, providerFile := "consumer/app.py", "provider/PIL/image.py" + workspace := "shared" + if scenario == "foreign_workspace" { + workspace = "foreign" + } + for _, node := range []*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared", Language: "python"}, + {ID: file + "::run", FilePath: file, Name: "run", Kind: graph.KindFunction, RepoPrefix: "consumer", WorkspaceID: "shared", Language: "python"}, + {ID: providerFile, FilePath: providerFile, Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: workspace, Language: "python"}, + } { + g.AddNode(node) + } + if scenario != "unreachable" { + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeImports, To: providerFile, CrossRepo: true, Line: 1, + Meta: map[string]any{PythonImportStubMeta: "external::PIL.image", "python_import": "PIL.image"}}) + } + candidate := &graph.Node{ID: providerFile + "::f", FilePath: providerFile, Name: "f", Kind: graph.KindFunction, RepoPrefix: "provider", WorkspaceID: workspace, Language: "python"} + switch scenario { + case "wrong_module": + candidate.FilePath = "provider/PIL/unrelated.py" + candidate.ID = candidate.FilePath + "::f" + case "method": + candidate.Kind = graph.KindMethod + case "other_provider": + candidate.FilePath, candidate.RepoPrefix = "other/PIL/image.py", "other" + candidate.ID = candidate.FilePath + "::f" + g.AddNode(&graph.Node{ID: "other/unrelated.py", FilePath: "other/unrelated.py", Kind: graph.KindFile, RepoPrefix: "other", WorkspaceID: "shared"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeImports, To: "other/unrelated.py", Line: 2, CrossRepo: true}) + } + g.AddNode(candidate) + original := "unresolved::extern::PIL.image::f" + if scenario == "from_alias" { + original = "unresolved::extern::PIL.image.f::g" + } + g.AddEdge(&graph.Edge{From: file + "::run", FilePath: file, Kind: graph.EdgeCalls, To: original, Line: 3, + Meta: map[string]any{PythonCallStubMeta: original}}) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { return nil }) + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { return nil }) + cr.ResolveForFile("consumer", "app.py") + calls := outEdgesOfKind(g, file+"::run", graph.EdgeCalls) + require.Len(t, calls, 1) + if scenario == "module" || scenario == "from_alias" { + require.Equal(t, candidate.ID, calls[0].To) + require.True(t, calls[0].CrossRepo) + } else { + require.Equal(t, original, calls[0].To) + require.False(t, calls[0].CrossRepo) + } + }) + } + } +} + +func TestTrackedPythonScopedRetryVisitsEachCallOnce(t *testing.T) { + g := graph.New() + for _, file := range []string{"consumer/app.py", "provider/PIL/__init__.py", "provider/PIL/image.py"} { + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: graph.RepoPrefixOfID(file), WorkspaceID: "shared"}) + } + g.AddNode(&graph.Node{ID: "provider/PIL/image.py::f", FilePath: "provider/PIL/image.py", Name: "f", Kind: graph.KindFunction, RepoPrefix: "provider", WorkspaceID: "shared"}) + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeImports, To: "external::PIL.image", Line: 1}) + g.AddEdge(&graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", Kind: graph.EdgeCalls, To: "external-call::dep::PIL.image.f", Line: 2, + Meta: map[string]any{PythonCallStubMeta: "unresolved::extern::PIL.image::f"}}) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + return &PythonPackageLookup{HasTop: func(top string) bool { return top == "PIL" }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", RepoPrefix: "provider", WorkspaceID: "shared"}}} + }} + }) + stats := cr.ResolveForFile("consumer", "app.py") + require.Equal(t, 2, stats.Resolved) + require.Zero(t, stats.Unresolved) +} + +func TestTrackedPythonTopLevelCallReopensStdlibTerminal(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + for _, terminal := range []string{"stdlib::mylib::f", "external-call::stdlib::mylib"} { + t.Run(backend+"/"+terminal, func(t *testing.T) { + g := goImportReviewStore(t, backend) + const file = "consumer/app.py" + const stub = "unresolved::extern::mylib::f" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, To: terminal, Line: 2, Meta: map[string]any{PythonCallStubMeta: stub, "external_call": true}}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, To: "external-call::stdlib::other", Line: 3, Meta: map[string]any{PythonCallStubMeta: "unresolved::extern::other::f"}}) + imp := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeImports, To: "provider/mylib.py", CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::mylib", "python_import": "mylib"}} + cr := NewCrossRepo(g) + reopened := cr.restubCallsForLinkedPythonImports([]graph.EdgeReindex{{Edge: imp}}) + require.Len(t, reopened, 1) + require.Equal(t, stub, reopened[0].To) + require.NotContains(t, reopened[0].Meta, "external_call") + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 2) + for _, call := range calls { + if call.Line == 3 { + require.Equal(t, "external-call::stdlib::other", call.To) + } + } + }) + } + } +} diff --git a/internal/resolver/python_module_attribution_batch_test.go b/internal/resolver/python_module_attribution_batch_test.go new file mode 100644 index 000000000..6ba4df813 --- /dev/null +++ b/internal/resolver/python_module_attribution_batch_test.go @@ -0,0 +1,73 @@ +package resolver + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" +) + +func TestPythonModuleAttributionBatchWritesArePerPass(t *testing.T) { + for _, count := range []int{3, 7} { + t.Run(strconv.Itoa(count), func(t *testing.T) { + base := graph.New() + frontier := incrementalFileFrontier{nodesByFile: map[string][]*graph.Node{}, outByNode: map[string][]*graph.Edge{}} + var imports []*graph.Edge + for i := 0; i < count; i++ { + path := "app/f" + strconv.Itoa(i) + ".py" + seedFile(base, path, "python") + edge := seedExternalImport(base, path, "package_"+strconv.Itoa(i)+".mod") + imports = append(imports, edge) + frontier.paths = append(frontier.paths, path) + frontier.nodesByFile[path] = base.GetFileNodes(path) + frontier.outByNode[path] = []*graph.Edge{edge} + } + seedFile(base, "outside/main.py", "python") + outside := seedExternalImport(base, "outside/main.py", "outside_package") + store := &moduleAttributionBatchRecordingStore{Store: base} + r := New(store) + r.prepareIncrementalAttributionCache(frontier) + defer r.clearIncrementalAttributionCache() + // Given cached Python imports, when the save batch attributes them, + // then writes cover the entire pass rather than each file. + r.runFileAttributionPassesForFilesLocked(frontier) + assert.Equal(t, []moduleAttributionBatchCall{{nodes: count}, {edges: count}}, store.batchCalls) + assert.Equal(t, []int{count}, store.reindexSizes) + for i, edge := range imports { + spec := "package_" + strconv.Itoa(i) + assert.Equal(t, "module::pypi:"+spec, edge.To) + assert.Equal(t, spec+".mod", edge.Meta["python_import"]) + require.NotNil(t, base.GetNode(edge.To)) + assert.Len(t, outEdgesOfKind(base, edge.From, graph.EdgeDependsOnModule), 1) + } + assert.Equal(t, "external::outside_package", outside.To, "a save must not sweep files outside its frontier") + }) + } +} + +func TestPythonModuleAttributionScopedResolveAllWritesOnce(t *testing.T) { + base := graph.New() + const count = 3 + var imports []*graph.Edge + for i := 0; i < count; i++ { + path := "consumer/f" + strconv.Itoa(i) + ".py" + base.AddNode(&graph.Node{ID: path, Kind: graph.KindFile, Name: path, FilePath: path, Language: "python", RepoPrefix: "consumer"}) + imports = append(imports, seedExternalImport(base, path, "package_"+strconv.Itoa(i)+".mod")) + } + // A pending reference makes ResolveAll reach its post-resolution tail. + base.AddEdge(&graph.Edge{From: "consumer/f0.py", To: "unresolved::Missing", Kind: graph.EdgeCalls, FilePath: "consumer/f0.py", Line: 2}) + store := &moduleAttributionBatchRecordingStore{Store: base} + r := New(store) + r.SetScope(map[string]struct{}{"consumer": {}}) + // Given a small scoped warm restart, when ResolveAll runs its file tail, + // then module attribution remains one later whole-scope pass. + r.ResolveAll() + assert.Equal(t, []moduleAttributionBatchCall{{nodes: count}, {edges: count}}, store.batchCalls) + assert.Equal(t, []int{count}, store.reindexSizes) + for i, edge := range imports { + assert.Equal(t, "module::pypi:package_"+strconv.Itoa(i), edge.To) + assert.Equal(t, "package_"+strconv.Itoa(i)+".mod", edge.Meta["python_import"]) + } +} diff --git a/internal/resolver/python_package_reexport.go b/internal/resolver/python_package_reexport.go new file mode 100644 index 000000000..4ab70f363 --- /dev/null +++ b/internal/resolver/python_package_reexport.go @@ -0,0 +1,84 @@ +package resolver + +import ( + "strings" + + "github.com/zzet/gortex/internal/graph" +) + +// PythonReExportFileMeta retains the package whose binding selected the call. +const PythonReExportFileMeta = "python_reexport_file" + +// resolvePythonPackageReExport follows one exact relative from-import in a +// linked package __init__.py. A directory import alone is not symbol evidence. +func (cr *CrossRepoResolver) resolvePythonPackageReExport(e *graph.Edge, module, symbol string) bool { + callerWS := cr.callerWorkspaceID(e) + var match *graph.Node + packageFile := "" + for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { + if imp.Kind != graph.EdgeImports || !imp.CrossRepo || !strings.HasSuffix(imp.To, "/__init__.py") { + continue + } + stub, _ := imp.Meta[PythonImportStubMeta].(string) + spec, _ := imp.Meta["python_import"].(string) + if stub == "" { + continue + } + pkg := cr.cachedGetNode(imp.To) + if pkg == nil || !cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(pkg), spec) { + continue + } + name := symbol + if module != spec { + name = pythonImportedName(module) + if name == "" || module != spec+"."+name { + continue + } + } + if !pythonModuleFileMatches(pkg.FilePath, spec) { + continue + } + for _, reexport := range cr.pythonReexportsByFile[pkg.ID] { + if reexport.Kind != graph.EdgeImports { + continue + } + exports, _ := reexport.Meta["python_reexports"].(map[string]any) + name, _ := exports[name].(string) + if name == "" { + continue + } + // The extractor's stem is repo-relative even when the per-repo pass + // has parked the relative import before multi-repo prefixing. + stem, _ := reexport.Meta["python_reexport_stem"].(string) + if stem == "" { + return false + } + found := false + for _, suffix := range []string{".py", "/__init__.py"} { + candidate := cr.cachedGetNode(pkg.RepoPrefix + "/" + stem + suffix + "::" + name) + if candidate == nil { + continue + } + if candidate.RepoPrefix != pkg.RepoPrefix || + (candidate.Kind != graph.KindFunction && candidate.Kind != graph.KindType) || + !cr.crossWorkspaceEligible(callerWS, candidateWorkspaceID(candidate), spec) { + return false + } + if match != nil && match.ID != candidate.ID { + return false + } + match, found = candidate, true + packageFile = pkg.ID + } + if !found { + return false + } + } + } + if match == nil { + return false + } + e.To, e.CrossRepo = match.ID, true + e.Meta[PythonReExportFileMeta] = packageFile + return true +} diff --git a/internal/resolver/python_package_reexport_test.go b/internal/resolver/python_package_reexport_test.go new file mode 100644 index 000000000..061e1f2e8 --- /dev/null +++ b/internal/resolver/python_package_reexport_test.go @@ -0,0 +1,64 @@ +package resolver + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" +) + +func TestTrackedPythonPackageReExportEvidence(t *testing.T) { + for _, scenario := range []string{"module_call", "from_alias", "no_import", "unmarked_import", "wrong_package", "package_workspace", "symbol_workspace", "missing_symbol", "ambiguous_layout"} { + t.Run(scenario, func(t *testing.T) { + g := graph.New() + file, pkg, impl := "consumer/app.py", "provider/src/pkg/__init__.py", "provider/src/pkg/impl.py" + for _, node := range []*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared"}, + {ID: pkg, FilePath: pkg, Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: "shared"}, + {ID: impl + "::Original", FilePath: impl, Name: "Original", Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}, + } { + if scenario == "package_workspace" && node.ID == pkg || scenario == "symbol_workspace" && node.ID == impl+"::Original" { + node.WorkspaceID = "foreign" + } + if scenario != "missing_symbol" || node.ID != impl+"::Original" { + g.AddNode(node) + } + } + imp := &graph.Edge{From: file, FilePath: file, To: pkg, Kind: graph.EdgeImports, CrossRepo: true, + Meta: map[string]any{PythonImportStubMeta: "external::pkg", "python_import": "pkg"}} + if scenario == "unmarked_import" { + delete(imp.Meta, PythonImportStubMeta) + } + if scenario == "wrong_package" { + imp.Meta["python_import"] = "other" + } + if scenario != "no_import" { + g.AddEdge(imp) + } + g.AddEdge(&graph.Edge{From: pkg, FilePath: pkg, To: impl, Kind: graph.EdgeImports, + Meta: map[string]any{"python_reexports": map[string]any{"Public": "Original"}, "python_reexport_stem": "src/pkg/impl"}}) + if scenario == "ambiguous_layout" { + path := "provider/src/pkg/impl/__init__.py" + g.AddNode(&graph.Node{ID: path + "::Original", FilePath: path, Name: "Original", Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}) + } + stub := "unresolved::extern::pkg::Public" + if scenario == "from_alias" { + stub = "unresolved::extern::pkg.Public::Alias" + } + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: stub, Kind: graph.EdgeCalls, Line: 2, Meta: map[string]any{PythonCallStubMeta: stub}}) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { return nil }) + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { return nil }) + cr.ResolveForFile("consumer", "app.py") + calls := outEdgesOfKind(g, file, graph.EdgeCalls) + require.Len(t, calls, 1) + if scenario == "module_call" || scenario == "from_alias" { + require.Equal(t, impl+"::Original", calls[0].To) + require.True(t, calls[0].CrossRepo) + } else { + require.Equal(t, stub, calls[0].To) + require.False(t, calls[0].CrossRepo) + } + }) + } +} diff --git a/internal/resolver/python_reexport_work_test.go b/internal/resolver/python_reexport_work_test.go new file mode 100644 index 000000000..53bb6f4b7 --- /dev/null +++ b/internal/resolver/python_reexport_work_test.go @@ -0,0 +1,152 @@ +package resolver + +import ( + "fmt" + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "strings" + "sync" + "testing" +) + +type pythonReexportCountStore struct { + graph.Store + mu sync.Mutex + reads map[string]int + batches [][]string + batchCalls int +} + +func (s *pythonReexportCountStore) GetOutEdges(id string) []*graph.Edge { + s.mu.Lock() + s.reads[id]++ + s.mu.Unlock() + return s.Store.GetOutEdges(id) +} + +func (s *pythonReexportCountStore) GetOutEdgesByNodeIDs(ids []string) map[string][]*graph.Edge { + s.mu.Lock() + s.batchCalls++ + s.mu.Unlock() + var packages []string + for _, id := range ids { + if strings.HasSuffix(id, "/__init__.py") { + packages = append(packages, id) + } + } + if len(packages) > 0 { + s.mu.Lock() + s.batches = append(s.batches, packages) + s.mu.Unlock() + } + return s.Store.GetOutEdgesByNodeIDs(ids) +} + +func TestPythonPackageReExportReadsOncePerPass(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &pythonReexportCountStore{Store: goImportReviewStore(t, backend), reads: map[string]int{}} + file, pkg, impl := "consumer/app.py", "provider/pkg/__init__.py", "provider/pkg/impl.py" + for _, node := range []*graph.Node{ + {ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared"}, + {ID: pkg, FilePath: pkg, Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: "shared"}, + {ID: impl + "::Original", Name: "Original", FilePath: impl, Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}, + {ID: impl + "::Alternate", Name: "Alternate", FilePath: impl, Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}, + } { + g.AddNode(node) + } + g.AddEdge(&graph.Edge{From: file, To: pkg, FilePath: file, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::pkg", "python_import": "pkg"}}) + g.AddEdge(&graph.Edge{From: pkg, To: impl, FilePath: pkg, Kind: graph.EdgeImports, Meta: map[string]any{"python_reexports": map[string]any{"Public": "Original"}, "python_reexport_stem": "pkg/impl"}}) + cr := NewCrossRepo(g) + g.reads = map[string]int{} + cr.buildReachableReposIndex(nil) + require.Len(t, cr.pythonReexportsByFile[pkg], 1, "publish reexports before workers start") + require.Zero(t, g.reads[pkg], "prefill must not issue package point reads") + require.Equal(t, [][]string{{pkg}}, g.batches) + g.reads = map[string]int{} + for range 4 { + edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{}} + require.True(t, cr.resolvePythonPackageReExport(edge, "pkg", "Public")) + require.Equal(t, impl+"::Original", edge.To) + } + require.Zero(t, g.reads[file], "caller bridge metadata is already loaded") + require.Zero(t, g.reads[pkg], "workers only read the prefilled package index") + cr.clearReachableReposIndex() + row := g.GetOutEdges(pkg)[0] + row.Meta["python_reexports"] = map[string]any{"Public": "Alternate"} + g.ReindexEdges([]graph.EdgeReindex{{Edge: row, OldTo: row.To}}) + g.reads = map[string]int{} + g.batches = nil + cr.buildReachableReposIndex(nil) + require.Len(t, cr.pythonReexportsByFile[pkg], 1, "publish reexports before workers start") + require.Zero(t, g.reads[pkg], "prefill must not issue package point reads") + require.Equal(t, [][]string{{pkg}}, g.batches) + g.reads = map[string]int{} + edge := &graph.Edge{From: file, FilePath: file, Kind: graph.EdgeCalls, Meta: map[string]any{}} + require.True(t, cr.resolvePythonPackageReExport(edge, "pkg", "Public")) + require.Equal(t, impl+"::Alternate", edge.To) + require.Zero(t, g.reads[file]) + require.Zero(t, g.reads[pkg]) + }) + } +} + +// The full pass must publish package exports before its resolveEdge fan-out. +// Run this result assertion with -race in fork CI as well. +func TestPythonPackageReExportFullPassWorkers(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &pythonReexportCountStore{Store: goImportReviewStore(t, backend), reads: map[string]int{}} + var packages []string + for i := range 4 { + pkg := fmt.Sprintf("provider/pkg%d/__init__.py", i) + packages = append(packages, pkg) + impl := fmt.Sprintf("provider/pkg%d/impl.py", i) + g.AddNode(&graph.Node{ID: pkg, FilePath: pkg, Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: "shared"}) + g.AddNode(&graph.Node{ID: impl + "::Original", Name: "Original", FilePath: impl, Kind: graph.KindType, RepoPrefix: "provider", WorkspaceID: "shared"}) + g.AddEdge(&graph.Edge{From: pkg, To: impl, FilePath: pkg, Kind: graph.EdgeImports, Meta: map[string]any{"python_reexports": map[string]any{"Public": "Original"}, "python_reexport_stem": fmt.Sprintf("pkg%d/impl", i)}}) + } + for i := range 64 { + file := fmt.Sprintf("consumer/app%d.py", i) + spec := fmt.Sprintf("pkg%d", i%4) + pkg := "provider/" + spec + "/__init__.py" + stub := "unresolved::extern::" + spec + "::Public" + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared"}) + g.AddEdge(&graph.Edge{From: file, To: pkg, FilePath: file, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::" + spec, "python_import": spec}}) + g.AddEdge(&graph.Edge{From: file, To: stub, FilePath: file, Kind: graph.EdgeCalls, Line: 2, Meta: map[string]any{PythonCallStubMeta: stub}}) + } + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { return nil }) + cr.SetCrossWorkspaceDepLookup(func(string) []CrossWorkspaceDepRule { return nil }) + cr.ResolveAll() + require.Len(t, g.batches, 1, "one package batch must precede all workers") + require.ElementsMatch(t, packages, g.batches[0], "duplicate caller imports must not duplicate package IDs") + for _, pkg := range packages { + require.Zero(t, g.reads[pkg], "prefill and workers must not issue package point reads") + } + for i := range 64 { + calls := outEdgesOfKind(g, fmt.Sprintf("consumer/app%d.py", i), graph.EdgeCalls) + require.Len(t, calls, 1) + require.Equal(t, fmt.Sprintf("provider/pkg%d/impl.py::Original", i%4), calls[0].To) + require.True(t, calls[0].CrossRepo) + } + }) + } +} + +func TestPythonPackageReExportNonPackageSkipsBatch(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + g := &pythonReexportCountStore{Store: goImportReviewStore(t, backend), reads: map[string]int{}} + file, target := "consumer/app.py", "provider/module.py" + g.AddNode(&graph.Node{ID: target, FilePath: target, Kind: graph.KindFile, RepoPrefix: "provider"}) + g.AddEdge(&graph.Edge{From: file, FilePath: file, To: target, Kind: graph.EdgeImports, CrossRepo: true, Meta: map[string]any{PythonImportStubMeta: "external::module"}}) + cr := NewCrossRepo(g) + cr.buildReachableReposIndex(nil) + require.Empty(t, g.batches) + require.Zero(t, g.batchCalls, "no package IDs means no batch query, even an empty one") + require.Empty(t, g.reads) + require.Empty(t, cr.pythonReexportsByFile) + }) + } +} diff --git a/internal/resolver/python_tracked_import.go b/internal/resolver/python_tracked_import.go new file mode 100644 index 000000000..a58c52e45 --- /dev/null +++ b/internal/resolver/python_tracked_import.go @@ -0,0 +1,348 @@ +package resolver + +import ( + "context" + "iter" + "slices" + "strings" + + "github.com/zzet/gortex/internal/graph" +) + +// PythonImportStubMeta preserves the import's exact target before a tracked bind. +const PythonImportStubMeta = "python_import_stub" + +// PythonCallStubMeta retains the exact function, method or module-qualified call target. +const PythonCallStubMeta = "python_call_stub" + +// PythonPackageLookup exposes only indexed tops from declared tracked +// distributions. The indexer owns manifest I/O and consumer-local checks. +type PythonPackageLookup struct { + HasTop func(top string) bool + Providers func(callerRepo, top string) map[string][]graph.FileNodeIdentity +} + +// SetPythonPackageLookup defers manifest reads until a non-stdlib Python import +// needs a retry. A nil result means no declared tracked providers exist. +func (cr *CrossRepoResolver) SetPythonPackageLookup(build func() *PythonPackageLookup) { + cr.pythonPackages = build +} + +func trackedPythonImportSpec(e *graph.Edge) string { + if e == nil || e.Kind != graph.EdgeImports || !isPythonSourcePath(e.FilePath) { + return "" + } + var spec string + switch { + case strings.HasPrefix(e.To, "module::python:"): + return "" + case strings.HasPrefix(e.To, "unresolved::import::"): + spec = strings.TrimPrefix(e.To, "unresolved::import::") + case strings.HasPrefix(e.To, "external::"): + spec = strings.TrimPrefix(e.To, "external::") + case strings.HasPrefix(e.To, "module::pypi:"): + spec, _ = e.Meta["python_import"].(string) + } + top, _, _ := strings.Cut(spec, ".") + if !isDottedPythonModule(spec) || isPythonStdlibTop(top) { + return "" + } + return spec +} + +// Attributed targets retain the top even in a metadata-free edge projection. +// The complete dotted path must still be recovered from the full edge later. +func trackedPythonImportTop(e *graph.Edge) string { + if e != nil && e.Kind == graph.EdgeImports && isPythonSourcePath(e.FilePath) && strings.HasPrefix(e.To, "module::pypi:") { + top := strings.TrimPrefix(e.To, "module::pypi:") + if isDottedPythonModule(top) && !isPythonStdlibTop(top) { + return top + } + return "" + } + top, _, _ := strings.Cut(trackedPythonImportSpec(e), ".") + return top +} + +func (cr *CrossRepoResolver) trackedPythonImport(caller *graph.Node, spec string, lookup *PythonPackageLookup, nodes map[string]*graph.Node) *graph.Node { + top, _, _ := strings.Cut(spec, ".") + var files []graph.FileNodeIdentity + providers := 0 + for _, candidates := range lookup.Providers(caller.RepoPrefix, top) { + if len(candidates) == 0 || !cr.crossWorkspaceEligible(candidateWorkspaceID(caller), fileCandidateWorkspaceID(candidates[0]), spec) { + continue + } + providers++ + files = candidates + } + if providers != 1 { + return nil + } + var match *graph.Node + for _, file := range files { + base := strings.TrimSuffix(strings.TrimSuffix(file.FilePath, "/__init__.py"), ".py") + stem := strings.TrimSuffix(base, top) + strings.ReplaceAll(spec, ".", "/") + for _, suffix := range []string{".py", "/__init__.py"} { + id := stem + suffix + node, seen := nodes[id] + if !seen { + node = cr.cachedGetNode(id) + nodes[id] = node + } + if node == nil || !pythonModuleFileMatches(node.FilePath, spec) { + continue + } + if match != nil && match.ID != node.ID { + return nil + } + match = node + } + } + return match +} + +// SQLite's projection scanner closes each page cursor before yielding, so the +// retry may hydrate and reindex edges without retaining the complete light scan. +// Other stores drain the light scan first: EdgesLightSeq can hold a live cursor +// that must be closed before the retry reads or writes the graph. +func (cr *CrossRepoResolver) trackedPythonImportEdges() iter.Seq[*graph.Edge] { + if scanner, ok := cr.graph.(interface { + ScanTestEdgeProjections([]graph.EdgeKind, int, func([]graph.TestEdgeProjection) bool) + }); ok { + return func(yield func(*graph.Edge) bool) { + scanner.ScanTestEdgeProjections([]graph.EdgeKind{graph.EdgeImports}, resolvePendingPageRows, func(page []graph.TestEdgeProjection) bool { + for _, e := range page { + if !yield(&graph.Edge{From: e.From, To: e.To, Kind: e.Kind, FilePath: e.FilePath, Line: e.Line}) { + return false + } + } + return true + }) + } + } + var edges []*graph.Edge + for e := range graph.EdgesLightSeq(cr.graph, graph.EdgeImports) { + if trackedPythonImportTop(e) != "" { + edges = append(edges, e) + } + } + return slices.Values(edges) +} + +// Retry only import/file edges; never reconsider working links. Full passes +// stream reopened calls later; scoped passes collect them for their pending list. +func (cr *CrossRepoResolver) resolveTrackedPythonImports(ctx context.Context, edges iter.Seq[*graph.Edge], stats *CrossRepoStats, captureCalls bool) (reopened []*graph.Edge, err error) { + if cr.pythonPackages == nil { + return reopened, ctx.Err() + } + next, stop := iter.Pull(edges) + defer stop() + var lookup *PythonPackageLookup + for { + page := make([]*graph.Edge, 0, resolvePendingPageRows) + for len(page) < resolvePendingPageRows { + if err := ctx.Err(); err != nil { + return reopened, err + } + e, ok := next() + if !ok { + break + } + top := trackedPythonImportTop(e) + if top == "" { + continue + } + if lookup == nil { + lookup = cr.pythonPackages() + if lookup == nil { + return reopened, ctx.Err() + } + } + if lookup.HasTop(top) { + page = append(page, e) + } + } + if err := ctx.Err(); err != nil { + return reopened, err + } + if len(page) == 0 { + return reopened, nil + } + nodes := map[string]*graph.Node{} + sites := make([]graph.EdgeSite, 0, len(page)) + ids := make([]string, 0, len(page)) + seen := map[string]bool{} + for _, e := range page { + sites = append(sites, graph.EdgeSite{From: e.From, Line: e.Line, Kind: graph.EdgeImports}) + if !seen[e.From] { + ids = append(ids, e.From) + seen[e.From] = true + } + } + full := cr.graph.GetEdgeCandidates(nil, sites) + callers := cr.hotCachedNodesByIDs(ids) + var batch []graph.EdgeReindex + for _, projected := range page { + caller := callers[projected.From] + if caller == nil { + continue + } + for _, e := range full.Site(projected.From, projected.Line, graph.EdgeImports) { + if e.To != projected.To { + continue + } + spec := trackedPythonImportSpec(e) + if spec == "" { + continue + } + match := cr.trackedPythonImport(caller, spec, lookup, nodes) + if match == nil { + continue + } + old := e.To + if e.Meta == nil { + e.Meta = make(map[string]any) + } + e.Meta[PythonImportStubMeta], e.Meta["python_import"] = old, spec + e.To, e.CrossRepo = match.ID, true + batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: old}) + projected.To, projected.CrossRepo = e.To, e.CrossRepo + stats.Resolved++ + stats.CrossRepoEdges++ + stats.ByRepo[match.RepoPrefix]++ + } + } + if len(batch) > 0 { + cr.graph.ReindexEdges(batch) + calls := cr.restubCallsForLinkedPythonImports(batch) + if captureCalls { + reopened = append(reopened, calls...) + } + } + } +} + +// Reopen only synthesized calls backed by an import this bridge just linked. +func (cr *CrossRepoResolver) restubCallsForLinkedPythonImports(changes []graph.EdgeReindex) []*graph.Edge { + specs := make(map[string][]string) + var files []string + for _, change := range changes { + spec, _ := change.Edge.Meta["python_import"].(string) + if stub, _ := change.Edge.Meta[PythonImportStubMeta].(string); stub == "" || spec == "" || !change.Edge.CrossRepo { + continue + } + file := change.Edge.FilePath + if _, seen := specs[file]; !seen { + files = append(files, file) + } + specs[file] = append(specs[file], spec) + } + if len(files) == 0 { + return nil + } + var reopened []*graph.Edge + var batch []graph.EdgeReindex + for row := range graph.EdgesInScopeSeq(cr.graph, nil, files, graph.EdgeCalls) { + call := row.Edge + if call == nil { + continue + } + for _, spec := range specs[call.FilePath] { + original, _ := call.Meta[PythonCallStubMeta].(string) + // Cold master resolution leaves dep:: terminals until externalization; + // incremental tracking has already synthesized external-call:: nodes. + callTarget := call.To + if ecosystem, module, ok := parseExternalCallTarget(callTarget); ok && (ecosystem == "dep" || ecosystem == "stdlib") { + callTarget = externalCallNodeID("dep", module) + } + callTarget = strings.Replace(callTarget, externalCallPrefix+"stdlib::", externalCallPrefix+"dep::", 1) + target := externalCallPrefix + "dep::" + spec + if original == "" || (callTarget != target && !strings.HasPrefix(callTarget, target+".")) { + continue + } + old := call.To + graph.StashRestubProvenance(call) + delete(call.Meta, "external_call") + call.To, call.CrossRepo, call.ConfidenceLabel = original, false, "" + reopened = append(reopened, call) + batch = append(batch, graph.EdgeReindex{Edge: call, OldTo: old}) + break + } + } + if len(batch) > 0 { + cr.graph.ReindexEdges(batch) + } + return reopened +} + +// Qualified Python calls use the exact linked import, not repo-wide reachability. +func (cr *CrossRepoResolver) linkedPythonCallCandidate(e *graph.Edge, node *graph.Node, module string) bool { + for _, imp := range cr.bridgeImportsByFile[cr.callerFileID(e)] { + spec, _ := imp.Meta["python_import"].(string) + if imp.Meta[PythonImportStubMeta] == nil || spec == "" || (module != spec && !strings.HasPrefix(module, spec+".")) { + continue + } + if node.FilePath == imp.To && cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(node), spec) { + return true + } + } + return false +} + +func (cr *CrossRepoResolver) resolvePythonImportedMethod(e *graph.Edge, target string) bool { + if e.Kind != graph.EdgeCalls || !isPythonSourcePath(e.FilePath) || !strings.HasPrefix(target, "*.") { + return false + } + receiver := edgeReceiverType(e) + if receiver == "" { + return false + } + file := cr.callerFileID(e) + var imports []*graph.Edge + for _, imp := range cr.bridgeImportsByFile[file] { + spec, _ := imp.Meta["python_import"].(string) + bindings, _ := imp.Meta["python_bindings"].(map[string]any) + imported, _ := bindings[receiver].(string) + if imported != "" && imp.Meta[PythonImportStubMeta] != nil && pythonModuleFileMatches(imp.To, spec) { + imports = append(imports, imp) + } + } + if len(imports) == 0 { + return false + } + if cr.pythonLocalTypesByFile[file][receiver] { + return false + } + candidates := make(map[string][]*graph.Node) + name := strings.TrimPrefix(target, "*.") + for _, node := range cr.bridgeCandidates(e, map[string]bool{name: true}, false)[name] { + candidates[node.FilePath] = append(candidates[node.FilePath], node) + } + var match *graph.Node + for _, imp := range imports { + spec, _ := imp.Meta["python_import"].(string) + if imp.Meta[PythonImportStubMeta] == nil || !pythonModuleFileMatches(imp.To, spec) { + continue + } + bindings, _ := imp.Meta["python_bindings"].(map[string]any) + imported, _ := bindings[receiver].(string) + if imported == "" { + continue + } + for _, candidate := range candidates[imp.To] { + if candidate.Kind != graph.KindMethod || candidate.FilePath != imp.To || nodeReceiverType(candidate) != imported || !cr.crossWorkspaceEligible(cr.callerWorkspaceID(e), candidateWorkspaceID(candidate), spec) { + continue + } + if match != nil && match.ID != candidate.ID { + return false + } + match = candidate + } + } + if match == nil { + return false + } + e.To, e.CrossRepo = match.ID, true + e.Confidence, e.Origin = 0.85, graph.OriginASTResolved + return true +} diff --git a/internal/resolver/python_tracked_import_test.go b/internal/resolver/python_tracked_import_test.go new file mode 100644 index 000000000..82c33c07e --- /dev/null +++ b/internal/resolver/python_tracked_import_test.go @@ -0,0 +1,195 @@ +package resolver + +import ( + "context" + "iter" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestTrackedPythonImportRetry(t *testing.T) { + for _, target := range []string{"unresolved::import::PIL.image", "external::PIL.image", "module::pypi:PIL", "local.py"} { + t.Run(target, func(t *testing.T) { + g := graph.New() + for _, file := range []string{"consumer/app.py", "provider/src/PIL/__init__.py", "provider/src/PIL/image.py", "local.py"} { + repo := "provider" + if file == "consumer/app.py" { + repo = "consumer" + } + g.AddNode(&graph.Node{ID: file, FilePath: file, Kind: graph.KindFile, Language: "python", RepoPrefix: repo, WorkspaceID: "shared"}) + } + e := &graph.Edge{From: "consumer/app.py", FilePath: "consumer/app.py", To: target, Kind: graph.EdgeImports, Line: 1, Meta: map[string]any{"python_import": "PIL.image", "keep": "metadata"}} + g.AddEdge(e) + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + return &PythonPackageLookup{HasTop: func(top string) bool { return top == "PIL" }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + require.Equal(t, "consumer", caller) + require.Equal(t, "PIL", top) + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/src/PIL/__init__.py", FilePath: "provider/src/PIL/__init__.py", RepoPrefix: "provider", WorkspaceID: "shared"}}} + }} + }) + cr.ResolveMutationFiles([]string{"consumer/app.py"}, nil) + imports := outEdgesOfKind(g, "consumer/app.py", graph.EdgeImports) + require.Len(t, imports, 1) + if target == "local.py" { + require.Equal(t, target, imports[0].To) + require.NotContains(t, imports[0].Meta, PythonImportStubMeta) + return + } + require.Equal(t, "provider/src/PIL/image.py", imports[0].To) + require.True(t, imports[0].CrossRepo) + require.Equal(t, "metadata", imports[0].Meta["keep"]) + require.Equal(t, target, imports[0].Meta[PythonImportStubMeta]) + require.Equal(t, "PIL.image", imports[0].Meta["python_import"]) + }) + } +} + +func TestTrackedPythonImportSkipsStdlibBeforeLookup(t *testing.T) { + for _, target := range []string{"module::python:stdlib::json", "external::json", "unresolved::import::json.tool"} { + t.Run(target, func(t *testing.T) { + g := graph.New() + g.AddNode(&graph.Node{ID: "app/a.py", FilePath: "app/a.py", Kind: graph.KindFile, RepoPrefix: "app"}) + g.AddEdge(&graph.Edge{From: "app/a.py", FilePath: "app/a.py", To: target, Kind: graph.EdgeImports, Meta: map[string]any{"python_import": "json"}}) + lookups := 0 + cr := NewCrossRepo(g) + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + lookups++ + return nil + }) + cr.ResolveMutationFiles([]string{"app/a.py"}, nil) + require.Zero(t, lookups) + }) + } +} + +type pythonImportPageStore struct { + graph.Store + fullScans, callerReads int + hydrated, callerBatches []int + scanned, processed, peakPending int +} + +func (s *pythonImportPageStore) EdgesByKind(kind graph.EdgeKind) iter.Seq[*graph.Edge] { + if kind == graph.EdgeImports { + s.fullScans++ + } + return s.Store.EdgesByKind(kind) +} + +func (s *pythonImportPageStore) EdgesLightSeq(kinds ...graph.EdgeKind) iter.Seq[*graph.Edge] { + return func(yield func(*graph.Edge) bool) { + for e := range graph.EdgesLightSeq(s.Store, kinds...) { + if trackedPythonImportTop(e) == "PIL" { + s.scanned++ + } + if !yield(e) { + return + } + } + } +} + +type pythonImportProjectionStore struct { + *pythonImportPageStore +} + +func (s *pythonImportProjectionStore) ScanTestEdgeProjections(kinds []graph.EdgeKind, pageSize int, yield func([]graph.TestEdgeProjection) bool) { + s.Store.(graph.TestProjectionScanner).ScanTestEdgeProjections(kinds, pageSize, func(page []graph.TestEdgeProjection) bool { + for _, e := range page { + if e.To == "module::pypi:PIL" { + s.scanned++ + } + if !yield([]graph.TestEdgeProjection{e}) { + return false + } + } + return true + }) +} + +func (s *pythonImportPageStore) GetNode(id string) *graph.Node { + if id == "consumer/a.py" { + s.callerReads++ + } + return s.Store.GetNode(id) +} + +func (s *pythonImportPageStore) GetNodesByIDs(ids []string) map[string]*graph.Node { + if len(ids) > 0 { // Empty scoped-call projections perform no node hydration. + s.callerBatches = append(s.callerBatches, len(ids)) + } + return s.Store.GetNodesByIDs(ids) +} + +func (s *pythonImportPageStore) GetEdgeCandidates(endpoints []graph.EdgeEndpoint, sites []graph.EdgeSite) graph.EdgeCandidateSet { + s.hydrated = append(s.hydrated, len(sites)) + s.peakPending = max(s.peakPending, s.scanned-s.processed) + s.processed += len(sites) + return s.Store.GetEdgeCandidates(endpoints, sites) +} + +func TestTrackedPythonImportPagedFullPass(t *testing.T) { + for _, backend := range []string{"memory", "sqlite"} { + t.Run(backend, func(t *testing.T) { + var store graph.Store = graph.New() + if backend == "sqlite" { + db, err := store_sqlite.Open(filepath.Join(t.TempDir(), "graph.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + store = db + } + g := &pythonImportPageStore{Store: store} + g.AddNode(&graph.Node{ID: "consumer/a.py", FilePath: "consumer/a.py", Kind: graph.KindFile, RepoPrefix: "consumer", WorkspaceID: "shared"}) + g.AddNode(&graph.Node{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", Kind: graph.KindFile, RepoPrefix: "provider", WorkspaceID: "shared"}) + for line := 1; line <= resolvePendingPageRows+1; line++ { + g.AddEdge(&graph.Edge{From: "consumer/a.py", FilePath: "consumer/a.py", To: "module::pypi:PIL", Kind: graph.EdgeImports, Line: line, Meta: map[string]any{"python_import": "PIL", "keep": "metadata"}}) + } + g.AddEdge(&graph.Edge{From: "consumer/a.py", FilePath: "consumer/a.py", To: "module::pypi:untracked", Kind: graph.EdgeImports, Line: resolvePendingPageRows + 2, Meta: map[string]any{"python_import": "untracked"}}) + var retryStore graph.Store = g + if backend == "sqlite" { + retryStore = &pythonImportProjectionStore{g} + } + cr := NewCrossRepo(retryStore) + lookups := 0 + cr.SetPythonPackageLookup(func() *PythonPackageLookup { + lookups++ + return &PythonPackageLookup{HasTop: func(top string) bool { return top == "PIL" }, Providers: func(caller, top string) map[string][]graph.FileNodeIdentity { + if top != "PIL" { + return nil + } + return map[string][]graph.FileNodeIdentity{"provider": {{ID: "provider/PIL/__init__.py", FilePath: "provider/PIL/__init__.py", RepoPrefix: "provider", WorkspaceID: "shared"}}} + }} + }) + stats, err := cr.ResolveAllContext(context.Background()) + require.NoError(t, err) + require.Equal(t, resolvePendingPageRows+1, stats.Resolved) + require.Equal(t, 1, lookups, "build the lookup once across all pages") + require.Zero(t, g.fullScans, "full import scans must use metadata-free projections") + require.Zero(t, g.callerReads, "caller nodes must be loaded in batches") + if backend == "sqlite" { + // Stores without the projection scanner drain the light scan + // first so no live cursor stays open during the retry. + require.LessOrEqual(t, g.peakPending, resolvePendingPageRows, "process each bounded page before retaining more tracked imports") + } + require.Equal(t, []int{resolvePendingPageRows, 1}, g.hydrated) + require.Equal(t, []int{1, 1}, g.callerBatches) + imports := g.GetOutEdges("consumer/a.py") + require.Len(t, imports, resolvePendingPageRows+2) + for _, e := range imports { + if e.Kind == graph.EdgeImports && e.Line <= resolvePendingPageRows+1 { + require.Equal(t, "provider/PIL/__init__.py", e.To) + require.Equal(t, "metadata", e.Meta["keep"]) + } else if e.Kind == graph.EdgeImports { + require.Equal(t, "module::pypi:untracked", e.To) + require.False(t, e.CrossRepo) + } + } + }) + } +} diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 7a34f7aba..6bc617547 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -2749,6 +2749,7 @@ func (r *Resolver) ResolveFileAndIncoming(filePath string) *ResolveStats { r.resolveFileEdgesLocked(filePath, stats) forwardDuration := time.Since(forwardStarted) attributionStarted := time.Now() + r.attributePythonModuleImportsForFiles([]string{filePath}) r.runFileAttributionPassesForFileLocked(filePath) attributionDuration := time.Since(attributionStarted) incomingStarted := time.Now() @@ -3329,6 +3330,7 @@ func (r *Resolver) startIncrementalPhase(logger *zap.Logger, phase string) func( // has built the per-pass indexes. func (r *Resolver) resolveFileLocked(filePath string, stats *ResolveStats) { r.resolveFileEdgesLocked(filePath, stats) + r.attributePythonModuleImportsForFiles([]string{filePath}) r.runFileAttributionPassesForFileLocked(filePath) } @@ -4112,6 +4114,19 @@ func (r *Resolver) resolveEdge(e *graph.Edge, stats *ResolveStats) (oldTo string } } + // External terminals need their spelling before a later tracked import retry. + if e.To != oldTo && e.Kind == graph.EdgeCalls && isPythonSourcePath(e.FilePath) && (e.CrossRepo || isExternalCandidateTarget(e.To)) { + if e.Meta == nil { + e.Meta = make(map[string]any) + } + if original, _ := e.Meta[PythonCallStubMeta].(string); original == "" { + e.Meta[PythonCallStubMeta] = oldTo + } + } + if e.To != oldTo { + graph.RestoreRestubProvenance(e) + clearEdgeTerminal(e) + } return oldTo, e.To != oldTo } diff --git a/internal/resolver/terminal.go b/internal/resolver/terminal.go index c91c99140..0d611d80e 100644 --- a/internal/resolver/terminal.go +++ b/internal/resolver/terminal.go @@ -372,3 +372,59 @@ func (r *Resolver) reconcileTerminalStampsBulk(pending []*graph.Edge, counter gr } return changed, stamped, unstamped } + +// ReconcileBridgeCallerTerminals finishes a completed tracked-package mutation +// using the full-pass policy for its affected marked calls. Callers must finish +// bridge resolution first; refused/incomplete passes leave the calls retryable. +func (r *Resolver) ReconcileBridgeCallerTerminals(files []string) { + if len(files) == 0 { + return + } + r.mu.Lock() + defer r.mu.Unlock() + _, counted := r.graph.(graph.NodeNameClassCounter) + _, memory := r.graph.(*graph.Graph) + if !counted && !memory { + return + } // Unknown adapters cannot certify absence. + var pending []*graph.Edge + for row := range graph.EdgesInScopeSeq(r.graph, nil, files, graph.EdgeCalls) { + edge := row.Edge + if edge == nil || !graph.IsUnresolvedTarget(edge.To) || (edge.Meta[CargoCallStubMeta] == nil && edge.Meta[PythonCallStubMeta] == nil) { + continue + } + if r.lspHelper != nil && r.lspHelper.SupportsPath(edge.FilePath) { + continue + } + pending = append(pending, edge) + } + var changed []*graph.Edge + if counter, ok := r.graph.(graph.NodeNameClassCounter); ok { + changed, _, _ = r.reconcileTerminalStampsBulk(pending, counter) + } else { + for _, edge := range pending { + reason, terminal := r.classifyTerminal(edge) + if terminal && !edgeTerminalFlag(edge) { + setEdgeTerminal(edge, reason) + changed = append(changed, edge) + } else if !terminal && edgeTerminalFlag(edge) { + clearEdgeTerminal(edge) + changed = append(changed, edge) + } + } + } + if len(changed) == 0 { + return + } + if p, ok := r.graph.(graph.EdgeTerminalStampPersister); ok { + p.PersistEdgeTerminalStamps(changed) + } else if p, ok := r.graph.(graph.EdgeMetaBatchPersister); ok { + p.PersistEdgeAttributesBatch(changed) + } else { + batch := make([]graph.EdgeReindex, 0, len(changed)) + for _, edge := range changed { + batch = append(batch, graph.EdgeReindex{Edge: edge, OldTo: edge.To}) + } + r.graph.ReindexEdges(batch) + } +} diff --git a/internal/resolver/version.go b/internal/resolver/version.go index 568677412..1a1c81b31 100644 --- a/internal/resolver/version.go +++ b/internal/resolver/version.go @@ -40,7 +40,9 @@ import ( // Convergence winner payloads and surviving outgoing order change stored edges // for an unchanged corpus. // Go module binding changes candidate order, cross-repo provenance and import identity. -const ResolutionSemanticsVersion = 5 +// Python import and call stubs add stored metadata required by tracked-package linking. +// Same-line Python submodule imports retain distinct stored identities. +const ResolutionSemanticsVersion = 6 // resolverVersionDomain versions the ENCODING below, not the contract. Two // fingerprints that disagree about how the preimage is rendered must not be diff --git a/internal/resolver/version_test.go b/internal/resolver/version_test.go index f90af9239..a31bbbcbb 100644 --- a/internal/resolver/version_test.go +++ b/internal/resolver/version_test.go @@ -22,7 +22,7 @@ import ( // and say which in the commit message. A diff that changes this line and // nothing else in internal/resolver is the signal that something re-keyed // every cached generation by accident. -const resolverVersionGolden = "resolver-5:7c5e80643f2ba32b450b7a56735b7e8a" +const resolverVersionGolden = "resolver-6:1d041ee489712f11544c6b6f7ce62982" func TestResolverVersionIsPinned(t *testing.T) { if got := Version(); got != resolverVersionGolden { @@ -38,7 +38,7 @@ func TestResolverVersionIsStable(t *testing.T) { if first != second { t.Fatalf("two reads of the resolution contract disagree: %q then %q", first, second) } - if !strings.HasPrefix(first, "resolver-5:") { + if !strings.HasPrefix(first, "resolver-6:") { t.Fatalf("the version %q does not name the semantics version it was produced under", first) } }