From 91d602f434eb1975feb84e84680fd9dcd4e5f4c9 Mon Sep 17 00:00:00 2001 From: rostislav Date: Thu, 20 Aug 2026 18:01:12 +0300 Subject: [PATCH 1/2] New framework plugins and additional VCS methods --- analysis-plugins/README.md | 21 + .../contracts/fixtures/plugin-globs.json | 10 + .../rostilos/codecrow/plugins/PluginGlob.java | 27 +- .../codecrow/plugins/ProjectSelector.java | 107 +- .../codecrow/plugins/PluginGlobTest.java | 31 + .../codecrow/plugins/ProjectSelectorTest.java | 94 ++ .../contracts/manifest/plugin.schema.json | 6 +- .../python/codecrow_plugins/facts.py | 280 +++-- .../python/codecrow_plugins/plugin_glob.py | 29 + .../python/codecrow_plugins/runtime.py | 215 +++- .../python/codecrow_plugins/selection.py | 52 +- .../python/tests/test_builtin_plugins.py | 169 ++- .../tests/test_django_framework_plugin.py | 446 ++++++++ .../test_javascript_framework_plugins.py | 780 ++++++++++++++ .../python/tests/test_plugin_glob.py | 21 + .../python/tests/test_quarkus_plugin.py | 390 +++++++ .../tests/test_rails_framework_plugin.py | 397 +++++++ .../python/tests/test_repository_facts.py | 246 +++++ .../tests/test_runtime_graph_fact_limits.py | 166 +++ .../frameworks/django/java/pom.xml | 15 + .../codecrow/plugins/django/DjangoPlugin.java | 23 + ...g.rostilos.codecrow.plugins.CodeCrowPlugin | 1 + .../plugins/django/DjangoPluginTest.java | 20 + .../frameworks/django/plugin.json | 52 + .../python/codecrow_plugin_django/__init__.py | 973 ++++++++++++++++++ .../frameworks/ember/java/pom.xml | 15 + .../codecrow/plugins/ember/EmberPlugin.java | 23 + ...g.rostilos.codecrow.plugins.CodeCrowPlugin | 1 + .../plugins/ember/EmberPluginTest.java | 20 + analysis-plugins/frameworks/ember/plugin.json | 32 + .../python/codecrow_plugin_ember/__init__.py | 683 ++++++++++++ .../frameworks/express/java/pom.xml | 15 + .../plugins/express/ExpressPlugin.java | 23 + ...g.rostilos.codecrow.plugins.CodeCrowPlugin | 1 + .../plugins/express/ExpressPluginTest.java | 20 + .../frameworks/express/plugin.json | 25 + .../codecrow_plugin_express/__init__.py | 684 ++++++++++++ .../frameworks/nextjs/java/pom.xml | 15 + .../codecrow/plugins/nextjs/NextJsPlugin.java | 23 + ...g.rostilos.codecrow.plugins.CodeCrowPlugin | 1 + .../plugins/nextjs/NextJsPluginTest.java | 20 + .../frameworks/nextjs/plugin.json | 25 + .../python/codecrow_plugin_nextjs/__init__.py | 787 ++++++++++++++ analysis-plugins/frameworks/pom.xml | 6 + .../frameworks/quarkus/java/pom.xml | 15 + .../plugins/quarkus/QuarkusPlugin.java | 25 + ...g.rostilos.codecrow.plugins.CodeCrowPlugin | 1 + .../plugins/quarkus/QuarkusPluginTest.java | 23 + .../frameworks/quarkus/plugin.json | 50 + .../codecrow_plugin_quarkus/__init__.py | 957 +++++++++++++++++ .../frameworks/rails/java/pom.xml | 15 + .../codecrow/plugins/rails/RailsPlugin.java | 23 + ...g.rostilos.codecrow.plugins.CodeCrowPlugin | 1 + .../plugins/rails/RailsPluginTest.java | 77 ++ analysis-plugins/frameworks/rails/plugin.json | 62 ++ .../python/codecrow_plugin_rails/__init__.py | 774 ++++++++++++++ .../codecrow/vcsclient/VcsClient.java | 26 + .../bitbucket/cloud/BitbucketCloudClient.java | 107 ++ .../vcsclient/github/GitHubClient.java | 182 ++++ .../vcsclient/gitlab/GitLabClient.java | 11 + .../gitlab/api/GitLabRepositoryApi.java | 95 ++ ...tbucketCloudRepositoryFileListingTest.java | 89 ++ .../GitHubRepositoryFileListingTest.java | 99 ++ .../gitlab/api/GitLabRepositoryApiTest.java | 131 +++ .../ProjectCapabilitySelectionService.java | 624 ++++++++++- ...ProjectCapabilitySelectionServiceTest.java | 634 +++++++++++- .../inference-orchestrator/src/Dockerfile | 2 +- .../src/Dockerfile.observable | 2 +- .../src/requirements.txt | 2 + .../tests/test_plugin_context.py | 10 +- .../test_incremental_repository_overlay.py | 101 ++ 71 files changed, 10958 insertions(+), 170 deletions(-) create mode 100644 analysis-plugins/contracts/fixtures/plugin-globs.json create mode 100644 analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/PluginGlobTest.java create mode 100644 analysis-plugins/contracts/python/codecrow_plugins/plugin_glob.py create mode 100644 analysis-plugins/contracts/python/tests/test_django_framework_plugin.py create mode 100644 analysis-plugins/contracts/python/tests/test_javascript_framework_plugins.py create mode 100644 analysis-plugins/contracts/python/tests/test_plugin_glob.py create mode 100644 analysis-plugins/contracts/python/tests/test_quarkus_plugin.py create mode 100644 analysis-plugins/contracts/python/tests/test_rails_framework_plugin.py create mode 100644 analysis-plugins/contracts/python/tests/test_runtime_graph_fact_limits.py create mode 100644 analysis-plugins/frameworks/django/java/pom.xml create mode 100644 analysis-plugins/frameworks/django/java/src/main/java/org/rostilos/codecrow/plugins/django/DjangoPlugin.java create mode 100644 analysis-plugins/frameworks/django/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin create mode 100644 analysis-plugins/frameworks/django/java/src/test/java/org/rostilos/codecrow/plugins/django/DjangoPluginTest.java create mode 100644 analysis-plugins/frameworks/django/plugin.json create mode 100644 analysis-plugins/frameworks/django/python/codecrow_plugin_django/__init__.py create mode 100644 analysis-plugins/frameworks/ember/java/pom.xml create mode 100644 analysis-plugins/frameworks/ember/java/src/main/java/org/rostilos/codecrow/plugins/ember/EmberPlugin.java create mode 100644 analysis-plugins/frameworks/ember/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin create mode 100644 analysis-plugins/frameworks/ember/java/src/test/java/org/rostilos/codecrow/plugins/ember/EmberPluginTest.java create mode 100644 analysis-plugins/frameworks/ember/plugin.json create mode 100644 analysis-plugins/frameworks/ember/python/codecrow_plugin_ember/__init__.py create mode 100644 analysis-plugins/frameworks/express/java/pom.xml create mode 100644 analysis-plugins/frameworks/express/java/src/main/java/org/rostilos/codecrow/plugins/express/ExpressPlugin.java create mode 100644 analysis-plugins/frameworks/express/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin create mode 100644 analysis-plugins/frameworks/express/java/src/test/java/org/rostilos/codecrow/plugins/express/ExpressPluginTest.java create mode 100644 analysis-plugins/frameworks/express/plugin.json create mode 100644 analysis-plugins/frameworks/express/python/codecrow_plugin_express/__init__.py create mode 100644 analysis-plugins/frameworks/nextjs/java/pom.xml create mode 100644 analysis-plugins/frameworks/nextjs/java/src/main/java/org/rostilos/codecrow/plugins/nextjs/NextJsPlugin.java create mode 100644 analysis-plugins/frameworks/nextjs/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin create mode 100644 analysis-plugins/frameworks/nextjs/java/src/test/java/org/rostilos/codecrow/plugins/nextjs/NextJsPluginTest.java create mode 100644 analysis-plugins/frameworks/nextjs/plugin.json create mode 100644 analysis-plugins/frameworks/nextjs/python/codecrow_plugin_nextjs/__init__.py create mode 100644 analysis-plugins/frameworks/quarkus/java/pom.xml create mode 100644 analysis-plugins/frameworks/quarkus/java/src/main/java/org/rostilos/codecrow/plugins/quarkus/QuarkusPlugin.java create mode 100644 analysis-plugins/frameworks/quarkus/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin create mode 100644 analysis-plugins/frameworks/quarkus/java/src/test/java/org/rostilos/codecrow/plugins/quarkus/QuarkusPluginTest.java create mode 100644 analysis-plugins/frameworks/quarkus/plugin.json create mode 100644 analysis-plugins/frameworks/quarkus/python/codecrow_plugin_quarkus/__init__.py create mode 100644 analysis-plugins/frameworks/rails/java/pom.xml create mode 100644 analysis-plugins/frameworks/rails/java/src/main/java/org/rostilos/codecrow/plugins/rails/RailsPlugin.java create mode 100644 analysis-plugins/frameworks/rails/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin create mode 100644 analysis-plugins/frameworks/rails/java/src/test/java/org/rostilos/codecrow/plugins/rails/RailsPluginTest.java create mode 100644 analysis-plugins/frameworks/rails/plugin.json create mode 100644 analysis-plugins/frameworks/rails/python/codecrow_plugin_rails/__init__.py create mode 100644 java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudRepositoryFileListingTest.java create mode 100644 java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/GitHubRepositoryFileListingTest.java diff --git a/analysis-plugins/README.md b/analysis-plugins/README.md index 6919a3cd..96630a71 100644 --- a/analysis-plugins/README.md +++ b/analysis-plugins/README.md @@ -55,6 +55,27 @@ approval. This behavior lives entirely under `languages/javascript`; generic RAG and inference hosts only consume neutral snapshots, packets, and validation results. +Framework-owned indexing is also packaged inside this boundary. Django extracts +AppConfig, installed apps, middleware and root URL configuration, URL paths and +includes, views, models/relations, middleware hooks, and signal receivers from +Python. Quarkus extracts CDI beans/injection, JAX-RS resources/routes, +configuration-property uses and application-property key/profile metadata, +schedules, reactive channels, and Panache topology from Java. Ember.js extracts +router maps, route/controller/component/service/model roles, service injection, +Ember Data relationships, and conservative `.hbs` ownership/invocations; +Express.js extracts routing and +middleware topology, and Next.js extracts file-system routes, boundaries, data +loaders, and Server Actions from JavaScript-family source; Rails extracts routes, +mounts, controllers/actions, models, associations, callbacks, and Active Job +queue/perform/retry/discard declarations from Ruby. These plugins use bounded repository +markers and exact source constructs, abstain when a construct is dynamic or +ambiguous, and add no model or embedding call. Their structural facts can refute +a matching absence claim, but topology alone is not positive proof of a defect. +Full selection may under-detect when optional marker acquisition is bounded or +unavailable. Incremental selection retains the last reliable content for a +changed marker that cannot be inspected; a successfully read marker that stops +matching remains an authoritative plugin-set change and requires a full reindex. + Graph-fact attributes whose keys start with `retrievalIdentifier:` are a neutral exact-retrieval hint. The value nominates an identifier inside a fact's already-proven related paths; generic RAG may diff --git a/analysis-plugins/contracts/fixtures/plugin-globs.json b/analysis-plugins/contracts/fixtures/plugin-globs.json new file mode 100644 index 00000000..6162c834 --- /dev/null +++ b/analysis-plugins/contracts/fixtures/plugin-globs.json @@ -0,0 +1,10 @@ +[ + {"glob": "*.gemspec", "path": "blog.gemspec", "matches": true}, + {"glob": "*.gemspec", "path": "nested/blog.gemspec", "matches": false}, + {"glob": "lib/**/engine.rb", "path": "lib/blog/engine.rb", "matches": true}, + {"glob": "lib/**/engine.rb", "path": "vendor/lib/blog/engine.rb", "matches": false}, + {"glob": "**/*.java", "path": "src/main/App.java", "matches": true}, + {"glob": "**/*.java", "path": "App.java", "matches": false}, + {"glob": "app/?/page.tsx", "path": "app/a/page.tsx", "matches": true}, + {"glob": "app/?/page.tsx", "path": "app/ab/page.tsx", "matches": false} +] diff --git a/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/PluginGlob.java b/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/PluginGlob.java index 963ef237..241bbe19 100644 --- a/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/PluginGlob.java +++ b/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/PluginGlob.java @@ -1,12 +1,37 @@ package org.rostilos.codecrow.plugins; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.regex.Pattern; public final class PluginGlob { + private static final int CACHE_LIMIT = 512; + private static final Map COMPILED = new LinkedHashMap<>( + CACHE_LIMIT, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > CACHE_LIMIT; + } + }; + private PluginGlob() { } public static boolean matches(String glob, String path) { + return compiled(glob).matcher(path).matches(); + } + + private static Pattern compiled(String glob) { + synchronized (COMPILED) { + Pattern cached = COMPILED.get(glob); + if (cached != null) return cached; + Pattern created = compile(glob); + COMPILED.put(glob, created); + return created; + } + } + + private static Pattern compile(String glob) { StringBuilder regex = new StringBuilder("^"); for (int index = 0; index < glob.length(); index++) { char character = glob.charAt(index); @@ -24,6 +49,6 @@ public static boolean matches(String glob, String path) { regex.append(Pattern.quote(String.valueOf(character))); } } - return Pattern.compile(regex.append('$').toString()).matcher(path).matches(); + return Pattern.compile(regex.append('$').toString()); } } diff --git a/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/ProjectSelector.java b/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/ProjectSelector.java index c4bd1fde..57fe22fa 100644 --- a/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/ProjectSelector.java +++ b/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/ProjectSelector.java @@ -6,6 +6,8 @@ import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.ArrayList; +import java.util.Collections; +import java.util.Collection; import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -28,11 +30,32 @@ public ProjectSelector(PluginRegistry registry) { } public ProjectCapabilities select(RepositoryFacts facts) { - if (facts.projectType() != null) return selectExplicit(facts); + return select(facts, sourcePaths(facts)); + } + + /** + * Select from complete repository facts while limiting per-file language + * ownership to the files a host will actually analyze. + * + *

PR hosts need unchanged marker and path-pattern evidence to select a + * framework, but must not turn that repository inventory into an + * enrichment request for every file.

+ */ + public ProjectCapabilities select( + RepositoryFacts facts, + Collection fileAssignmentPaths) { + if (facts == null) throw new IllegalArgumentException("repository facts are required"); + if (fileAssignmentPaths == null) { + throw new IllegalArgumentException("file assignment paths are required"); + } + List assignmentPaths = sourcePaths( + List.copyOf(fileAssignmentPaths), facts.sourceRoot()); + if (facts.projectType() != null) return selectExplicit(facts, assignmentPaths); List selected = new ArrayList<>(); Map> evidence = new TreeMap<>(); + Set repositoryPathSet = Set.copyOf(facts.paths()); for (PluginDescriptor descriptor : registry.descriptors()) { - List matched = match(descriptor, facts); + List matched = match(descriptor, facts, repositoryPathSet); if (matched == null) continue; if (!selected.containsAll(descriptor.requires())) continue; selected.add(descriptor.id()); @@ -44,7 +67,7 @@ public ProjectCapabilities select(RepositoryFacts facts) { .map(registry::descriptor) .filter(descriptor -> descriptor.kind() == PluginKind.LANGUAGE) .toList(); - for (String path : sourcePaths(facts)) { + for (String path : assignmentPaths) { String extension = extension(path); List matches = languages.stream() .filter(descriptor -> descriptor.detection().extensions().contains(extension)) @@ -63,13 +86,15 @@ public ProjectCapabilities select(RepositoryFacts facts) { registry.fingerprintFor(selected)); } - private ProjectCapabilities selectExplicit(RepositoryFacts facts) { + private ProjectCapabilities selectExplicit( + RepositoryFacts facts, + List assignmentPaths) { PluginDescriptor requested = registry.descriptor(facts.projectType()); TreeSet requestedIds = new TreeSet<>(); requestedIds.add(requested.id()); for (PluginDescriptor descriptor : registry.descriptors()) { if (descriptor.kind() != PluginKind.LANGUAGE) continue; - if (sourcePaths(facts).stream().anyMatch(path -> + if (assignmentPaths.stream().anyMatch(path -> descriptor.detection().extensions().contains(extension(path)))) { requestedIds.add(descriptor.id()); } @@ -89,7 +114,7 @@ private ProjectCapabilities selectExplicit(RepositoryFacts facts) { List languages = resolved.stream() .filter(descriptor -> descriptor.kind() == PluginKind.LANGUAGE) .toList(); - for (String path : sourcePaths(facts)) { + for (String path : assignmentPaths) { List matches = languages.stream() .filter(descriptor -> descriptor.detection().extensions().contains(extension(path))) .map(PluginDescriptor::id) @@ -105,10 +130,14 @@ private ProjectCapabilities selectExplicit(RepositoryFacts facts) { registry.fingerprintFor(selected)); } - private List match(PluginDescriptor descriptor, RepositoryFacts facts) { + private List match( + PluginDescriptor descriptor, + RepositoryFacts facts, + Set repositoryPaths) { DetectionRules rules = descriptor.detection(); List extensionHits = sourcePaths(facts).stream() .filter(path -> rules.extensions().contains(extension(path))) + .limit(MAX_EVIDENCE_PER_PLUGIN) .toList(); List groups = new ArrayList<>(); if (!rules.filesAll().isEmpty() || !rules.filesAny().isEmpty() @@ -121,7 +150,7 @@ private List match(PluginDescriptor descriptor, RepositoryFacts facts) { extensionHits.forEach(path -> evidence.add("extension:" + path)); boolean groupMatched = false; for (DetectionAlternative group : groups) { - List matched = matchGroup(group, facts); + List matched = matchGroup(group, facts, repositoryPaths); if (matched != null) { groupMatched = true; evidence.addAll(matched); @@ -131,19 +160,28 @@ private List match(PluginDescriptor descriptor, RepositoryFacts facts) { if (descriptor.kind() == PluginKind.LANGUAGE) { if (extensionHits.isEmpty() && !groupMatched) return null; } else if (!groupMatched) return null; - return evidence.stream().limit(MAX_EVIDENCE_PER_PLUGIN).toList(); + return boundedEvidence(evidence); } private static List sourcePaths(RepositoryFacts facts) { if (facts.sourceRoot() == null) return facts.paths(); - String prefix = facts.sourceRoot() + "/"; - return facts.paths().stream() - .filter(path -> path.equals(facts.sourceRoot()) || path.startsWith(prefix)) + return sourcePaths(facts.paths(), facts.sourceRoot()); + } + + private static List sourcePaths( + Collection paths, + String sourceRoot) { + if (sourceRoot == null) return List.copyOf(paths); + String prefix = sourceRoot + "/"; + return paths.stream() + .filter(path -> path.equals(sourceRoot) || path.startsWith(prefix)) .toList(); } - private List matchGroup(DetectionAlternative group, RepositoryFacts facts) { - Set paths = Set.copyOf(facts.paths()); + private List matchGroup( + DetectionAlternative group, + RepositoryFacts facts, + Set paths) { List> rootSets = new ArrayList<>(); group.filesAll().forEach(relative -> rootSets.add(suffixRoots(facts.paths(), relative))); group.contentMarkers().forEach(marker -> rootSets.add(facts.markerContents().entrySet().stream() @@ -163,6 +201,8 @@ private List matchGroup(DetectionAlternative group, RepositoryFacts fact candidateRoots.retainAll(Set.of(facts.sourceRoot())); } + TreeSet matchedEvidence = new TreeSet<>(); + int matchedRoots = 0; for (String root : candidateRoots) { List filesAll = group.filesAll().stream() .map(relative -> rooted(root, relative)).toList(); @@ -171,9 +211,12 @@ private List matchGroup(DetectionAlternative group, RepositoryFacts fact .map(relative -> rooted(root, relative)).filter(paths::contains).toList(); if (!group.filesAny().isEmpty() && filesAny.isEmpty()) continue; - Map> allPatternHits = patternHits(group.pathPatternsAll(), facts.paths(), root); + List rootedPaths = pathsUnderRoot(facts.paths(), root); + Map> allPatternHits = patternHits( + group.pathPatternsAll(), rootedPaths, root); if (allPatternHits.values().stream().anyMatch(List::isEmpty)) continue; - Map> anyPatternHits = patternHits(group.pathPatternsAny(), facts.paths(), root); + Map> anyPatternHits = patternHits( + group.pathPatternsAny(), rootedPaths, root); if (!group.pathPatternsAny().isEmpty() && anyPatternHits.values().stream().allMatch(List::isEmpty)) continue; @@ -208,9 +251,11 @@ private List matchGroup(DetectionAlternative group, RepositoryFacts fact evidence.add("content:" + path + ":" + marker.contains())); patternMarkerHits.forEach((marker, hits) -> hits.forEach(path -> evidence.add( "content-pattern:" + marker.pathPattern() + ":" + path + ":" + marker.contains()))); - return List.copyOf(evidence); + matchedEvidence.addAll(evidence); + matchedRoots++; + if (matchedRoots == MAX_EVIDENCE_PER_PLUGIN) break; } - return null; + return matchedEvidence.isEmpty() ? null : boundedEvidence(matchedEvidence); } private static Set suffixRoots(List paths, String relative) { @@ -241,11 +286,37 @@ private static Map> patternHits( result.put(pattern, paths.stream() .filter(path -> relativeToRoot(path, root) != null) .filter(path -> PluginGlob.matches(pattern, relativeToRoot(path, root))) + .limit(MAX_EVIDENCE_PER_PLUGIN) .toList()); } return result; } + private static List pathsUnderRoot(List paths, String root) { + if (root.isEmpty()) return paths; + String prefix = root + "/"; + int start = Collections.binarySearch(paths, prefix); + if (start < 0) start = -start - 1; + int end = start; + while (end < paths.size() && paths.get(end).startsWith(prefix)) end++; + return paths.subList(start, end); + } + + private static List boundedEvidence(Collection evidence) { + TreeSet ordered = new TreeSet<>(evidence); + List roots = ordered.stream() + .filter(item -> item.startsWith("root:")) + .limit(MAX_EVIDENCE_PER_PLUGIN) + .toList(); + int remaining = MAX_EVIDENCE_PER_PLUGIN - roots.size(); + TreeSet retained = new TreeSet<>(roots); + ordered.stream() + .filter(item -> !item.startsWith("root:")) + .limit(remaining) + .forEach(retained::add); + return List.copyOf(retained); + } + private String fingerprint( String revision, List selected, diff --git a/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/PluginGlobTest.java b/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/PluginGlobTest.java new file mode 100644 index 00000000..ea311ff3 --- /dev/null +++ b/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/PluginGlobTest.java @@ -0,0 +1,31 @@ +package org.rostilos.codecrow.plugins; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class PluginGlobTest { + private static final Path FIXTURE = Path.of( + System.getProperty("codecrow.plugin.fixtures"), "plugin-globs.json"); + + @Test + void matchesTheSharedAnchoredProjection() throws Exception { + List cases = new ObjectMapper().readValue( + Files.readString(FIXTURE), + new TypeReference<>() {}); + + assertThat(cases.stream() + .map(value -> PluginGlob.matches(value.glob(), value.path())) + .toList()) + .containsExactlyElementsOf(cases.stream().map(GlobCase::matches).toList()); + } + + private record GlobCase(String glob, String path, boolean matches) { + } +} diff --git a/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/ProjectSelectorTest.java b/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/ProjectSelectorTest.java index c67d6297..4a5e829c 100644 --- a/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/ProjectSelectorTest.java +++ b/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/ProjectSelectorTest.java @@ -5,6 +5,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; @@ -109,4 +110,97 @@ void source_root_is_canonicalized_like_the_python_contract() { assertThat(facts.sourceRoot()).isEqualTo("app/code"); } + + @Test + void automatic_detection_keeps_framework_root_when_evidence_exceeds_cap() { + PluginDescriptor language = new PluginDescriptor( + "fixture-language", + PluginKind.LANGUAGE, + List.of(), + List.of(PluginCapability.SYNTAX), + new DetectionRules( + List.of(".fixture"), List.of(), List.of(), List.of(), List.of()), + Map.of()); + PluginDescriptor framework = new PluginDescriptor( + "fixture-framework", + PluginKind.FRAMEWORK, + List.of("fixture-language"), + List.of(PluginCapability.GRAPH), + new DetectionRules( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new DetectionAlternative( + List.of("framework.marker"), + List.of(), + List.of(), + List.of("**/*.fixture"), + List.of()))), + Map.of()); + PluginRegistry registry = new PluginRegistry(List.of(language, framework)); + List paths = new java.util.ArrayList<>(IntStream.range(0, 70) + .mapToObj(index -> "services/shop/src/Thing%02d.fixture".formatted(index)) + .toList()); + paths.add("services/shop/framework.marker"); + paths.add("tools/Outside.fixture"); + paths.sort(String::compareTo); + + ProjectCapabilities selected = new ProjectSelector(registry).select( + new RepositoryFacts("abc1234", paths, Map.of())); + + assertThat(selected.repositoryPlugins()) + .containsExactly("fixture-language", "fixture-framework"); + assertThat(selected.detectionEvidence().get("fixture-framework")) + .hasSize(64) + .contains("root:services/shop") + .doesNotContain("root:."); + } + + @Test + void automatic_detection_retains_every_matching_framework_root() { + PluginDescriptor language = new PluginDescriptor( + "fixture-language", + PluginKind.LANGUAGE, + List.of(), + List.of(PluginCapability.SYNTAX), + new DetectionRules( + List.of(".fixture"), List.of(), List.of(), List.of(), List.of()), + Map.of()); + PluginDescriptor framework = new PluginDescriptor( + "fixture-framework", + PluginKind.FRAMEWORK, + List.of("fixture-language"), + List.of(PluginCapability.GRAPH), + new DetectionRules( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new DetectionAlternative( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new ContentMarker("package.json", "fixture-framework"))))), + Map.of()); + PluginRegistry registry = new PluginRegistry(List.of(language, framework)); + + ProjectCapabilities selected = new ProjectSelector(registry).select( + new RepositoryFacts( + "abc1234", + List.of( + "apps/admin/package.json", + "apps/admin/source.fixture", + "apps/store/package.json", + "apps/store/source.fixture"), + Map.of( + "apps/admin/package.json", "fixture-framework", + "apps/store/package.json", "fixture-framework"))); + + assertThat(selected.repositoryPlugins()) + .containsExactly("fixture-language", "fixture-framework"); + assertThat(selected.detectionEvidence().get("fixture-framework")) + .contains("root:apps/admin", "root:apps/store"); + } } diff --git a/analysis-plugins/contracts/manifest/plugin.schema.json b/analysis-plugins/contracts/manifest/plugin.schema.json index 6b365af1..fd6bb046 100644 --- a/analysis-plugins/contracts/manifest/plugin.schema.json +++ b/analysis-plugins/contracts/manifest/plugin.schema.json @@ -139,7 +139,11 @@ {"properties": {"filesAny": {"minItems": 1}}}, {"properties": {"pathPatternsAll": {"minItems": 1}}}, {"properties": {"pathPatternsAny": {"minItems": 1}}}, - {"properties": {"contentMarkers": {"minItems": 1}}} + {"properties": {"contentMarkers": {"minItems": 1}}}, + { + "required": ["contentPatternMarkers"], + "properties": {"contentPatternMarkers": {"minItems": 1}} + } ] } } diff --git a/analysis-plugins/contracts/python/codecrow_plugins/facts.py b/analysis-plugins/contracts/python/codecrow_plugins/facts.py index c2fd37f0..5096c4f5 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/facts.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/facts.py @@ -2,10 +2,10 @@ import logging from pathlib import Path -from pathlib import PurePosixPath from typing import Iterable from .api import RepositoryFacts, normalize_path +from .plugin_glob import plugin_glob_matches from .registry import PluginRegistry logger = logging.getLogger(__name__) @@ -24,13 +24,13 @@ def _declared_markers(registry: PluginRegistry): ), ) })) - patterns = tuple( + pattern_markers = tuple(sorted({ marker for descriptor in registry.descriptors for alternative in descriptor.detection.alternatives for marker in alternative.content_pattern_markers - ) - return exact, patterns + })) + return exact, pattern_markers def _under_source_root(path: str, source_root: str | None) -> bool: @@ -41,6 +41,77 @@ def _under_source_root(path: str, source_root: str | None) -> bool: ) +def _potential_root_relative_paths(path: str, source_root: str | None): + if source_root is not None: + if path == source_root: + yield "" + elif path.startswith(source_root + "/"): + yield path[len(source_root) + 1:] + return + yield path + offset = path.find("/") + while offset >= 0: + yield path[offset + 1:] + offset = path.find("/", offset + 1) + + +def _applicable_pattern_markers( + path: str, + pattern_markers, + source_root: str | None, +): + relative_paths = tuple(_potential_root_relative_paths(path, source_root)) + return { + marker + for marker in pattern_markers + if any( + plugin_glob_matches(marker.path_pattern, relative) + for relative in relative_paths + ) + } + + +def _fair_candidate_paths( + paths: tuple[str, ...], + exact_marker_paths: tuple[str, ...], + pattern_markers, + source_root: str | None, +): + def exact_lane(marker_path): + return ( + path for path in paths + if path == marker_path or path.endswith("/" + marker_path) + ) + + def pattern_lane(marker): + return ( + path for path in paths + if marker in _applicable_pattern_markers( + path, (marker,), source_root, + ) + ) + + lanes = [ + exact_lane(marker_path) + for marker_path in exact_marker_paths + ] + lanes.extend( + pattern_lane(marker) + for marker in pattern_markers + ) + seen: set[str] = set() + progressed = True + while progressed: + progressed = False + for lane in lanes: + path = next((candidate for candidate in lane if candidate not in seen), None) + if path is None: + continue + seen.add(path) + progressed = True + yield path + + def _matching_markers(path, content, exact_markers, pattern_markers): matching_exact = { marker @@ -51,8 +122,7 @@ def _matching_markers(path, content, exact_markers, pattern_markers): matching_patterns = { marker for marker in pattern_markers - if PurePosixPath(path).match(marker.path_pattern) - and marker.contains in content + if marker.contains in content } return matching_exact, matching_patterns @@ -64,6 +134,7 @@ def build_repository_facts( registry: PluginRegistry, *, max_marker_bytes: int = 262_144, + max_marker_files: int = 4_096, project_type: str | None = None, source_root: str | None = None, ) -> RepositoryFacts: @@ -85,20 +156,20 @@ def build_repository_facts( marker_contents: dict[str, str] = {} consumed_bytes = 0 - matched_pattern_markers = set() - pattern_candidates = tuple( - path for path in normalized_paths - if _under_source_root(path, source_root) - and ( - any(PurePosixPath(path).match(marker.path_pattern) for marker in declared_pattern_markers) - or any( - path == marker_path or path.endswith("/" + marker_path) - for marker_path in declared_marker_paths - ) - ) + marker_candidates = _fair_candidate_paths( + tuple( + path for path in normalized_paths + if _under_source_root(path, source_root) + ), + declared_marker_paths, + declared_pattern_markers, + source_root, ) skipped_for_bytes = 0 - for marker_path in tuple(dict.fromkeys((*declared_marker_paths, *pattern_candidates))): + skipped_for_files = 0 + skipped_unreadable = 0 + inspected_files = 0 + for marker_path in marker_candidates: if ( marker_path not in available or not _under_source_root(marker_path, source_root) @@ -109,34 +180,44 @@ def build_repository_facts( for marker in declared_markers if marker_path == marker.path or marker_path.endswith("/" + marker.path) } - applicable_pattern_markers = { - marker for marker in declared_pattern_markers - if PurePosixPath(marker_path).match(marker.path_pattern) - } - if ( - not applicable_exact_markers - and applicable_pattern_markers.issubset(matched_pattern_markers) - ): + applicable_pattern_markers = _applicable_pattern_markers( + marker_path, + declared_pattern_markers, + source_root, + ) + if not applicable_exact_markers and not applicable_pattern_markers: + continue + if inspected_files >= max_marker_files: + skipped_for_files = 1 + break + inspected_files += 1 + try: + full_path = (root / marker_path).resolve(strict=True) + if root not in full_path.parents: + skipped_unreadable += 1 + continue + size = full_path.stat().st_size + except (OSError, RuntimeError): + skipped_unreadable += 1 continue - full_path = (root / marker_path).resolve(strict=True) - if root not in full_path.parents: - raise ValueError("plugin marker escaped the repository root") - size = full_path.stat().st_size if consumed_bytes + size > max_marker_bytes: skipped_for_bytes += 1 continue - content = full_path.read_text(encoding="utf-8") + consumed_bytes += size + try: + content = full_path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + skipped_unreadable += 1 + continue matching_exact_markers, matching_pattern_markers = _matching_markers( marker_path, content, applicable_exact_markers, - applicable_pattern_markers - matched_pattern_markers, + applicable_pattern_markers, ) if not matching_exact_markers and not matching_pattern_markers: continue - consumed_bytes += len(content.encode("utf-8")) marker_contents[marker_path] = content - matched_pattern_markers.update(matching_pattern_markers) if skipped_for_bytes: logger.warning( @@ -146,6 +227,21 @@ def build_repository_facts( skipped_for_bytes, max_marker_bytes, ) + if skipped_for_files: + logger.warning( + "Skipped %s plugin marker candidate(s) after reaching the %s-file " + "inspection budget; repository indexing will continue with reduced " + "automatic plugin-detection evidence", + skipped_for_files, + max_marker_files, + ) + if skipped_unreadable: + logger.warning( + "Skipped %s unavailable, unsafe, or non-UTF-8 plugin marker " + "candidate(s); repository indexing will continue with reduced " + "automatic plugin-detection evidence", + skipped_unreadable, + ) return RepositoryFacts( revision=revision, @@ -165,6 +261,7 @@ def overlay_repository_facts( registry: PluginRegistry, *, max_marker_bytes: int = 262_144, + max_marker_files: int = 4_096, ) -> RepositoryFacts: """Apply one exact commit change set to persisted neutral detection facts. @@ -214,10 +311,23 @@ def overlay_repository_facts( path, content, declared_markers, - declared_pattern_markers, + _applicable_pattern_markers( + path, + declared_pattern_markers, + baseline.source_root, + ), )) } + retained_marker_bytes = sum( + len(content.encode("utf-8")) + for content in marker_contents.values() + ) + inspected_bytes = 0 + inspected_files = 0 + skipped_inspection_bytes = 0 + skipped_inspection_files = 0 + skipped_unreadable = 0 for marker_path in updated: if not _under_source_root(marker_path, baseline.source_root): marker_contents.pop(marker_path, None) @@ -227,50 +337,96 @@ def overlay_repository_facts( for marker in declared_markers if marker_path == marker.path or marker_path.endswith("/" + marker.path) ) - applicable_patterns = tuple( - marker - for marker in declared_pattern_markers - if PurePosixPath(marker_path).match(marker.path_pattern) - ) + applicable_patterns = tuple(_applicable_pattern_markers( + marker_path, + declared_pattern_markers, + baseline.source_root, + )) if not applicable_exact_markers and not applicable_patterns: continue - full_path = (root / marker_path).resolve(strict=True) - if root not in full_path.parents: - raise ValueError("plugin marker escaped the repository root") - content = full_path.read_text(encoding="utf-8") + if inspected_files >= max_marker_files: + # This is reduced evidence, not proof that an already persisted + # marker stopped matching. Keep the last reliable content so an + # incremental update cannot deactivate a plugin merely because an + # optional inspection budget was exhausted. + skipped_inspection_files += 1 + continue + inspected_files += 1 + try: + full_path = (root / marker_path).resolve(strict=True) + if root not in full_path.parents: + skipped_unreadable += 1 + continue + size = full_path.stat().st_size + except (OSError, RuntimeError): + skipped_unreadable += 1 + continue + if inspected_bytes + size > max_marker_bytes: + skipped_inspection_bytes += 1 + continue + inspected_bytes += size + try: + content = full_path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + skipped_unreadable += 1 + continue if any(_matching_markers( marker_path, content, applicable_exact_markers, applicable_patterns, )): - marker_contents[marker_path] = content + previous = marker_contents.get(marker_path) + candidate_bytes = len(content.encode("utf-8")) + previous_bytes = ( + len(previous.encode("utf-8")) + if previous is not None + else 0 + ) + candidate_total = ( + retained_marker_bytes - previous_bytes + candidate_bytes + ) + if candidate_total > max_marker_bytes: + # A matching update that cannot fit the persisted evidence + # budget is also inconclusive. Preserve its previous proven + # marker, if any, and omit a newly introduced marker. + skipped_inspection_bytes += 1 + else: + marker_contents[marker_path] = content + retained_marker_bytes = candidate_total else: - marker_contents.pop(marker_path, None) + removed = marker_contents.pop(marker_path, None) + if removed is not None: + retained_marker_bytes -= len(removed.encode("utf-8")) - bounded_marker_contents: dict[str, str] = {} - consumed_bytes = 0 - skipped_for_bytes = 0 - for path, content in sorted(marker_contents.items()): - size = len(content.encode("utf-8")) - if consumed_bytes + size > max_marker_bytes: - skipped_for_bytes += 1 - continue - bounded_marker_contents[path] = content - consumed_bytes += size - if skipped_for_bytes: + if skipped_inspection_bytes: logger.warning( - "Skipped %s persisted plugin marker file(s) after reaching the " - "%s-byte content budget during incremental update; indexing will " - "continue with reduced automatic plugin-detection evidence", - skipped_for_bytes, + "Skipped %s updated plugin marker candidate(s) after reaching the " + "%s-byte inspection budget; incremental indexing will continue with " + "reduced automatic plugin-detection evidence", + skipped_inspection_bytes, max_marker_bytes, ) + if skipped_inspection_files: + logger.warning( + "Skipped %s updated plugin marker candidate(s) after reaching the " + "%s-file inspection budget; incremental indexing will continue with " + "reduced automatic plugin-detection evidence", + skipped_inspection_files, + max_marker_files, + ) + if skipped_unreadable: + logger.warning( + "Skipped %s unavailable, unsafe, or non-UTF-8 updated plugin " + "marker candidate(s); incremental indexing will continue with " + "reduced automatic plugin-detection evidence", + skipped_unreadable, + ) return RepositoryFacts( revision=revision, paths=tuple(sorted(paths)), - marker_contents=bounded_marker_contents, + marker_contents=marker_contents, project_type=baseline.project_type, source_root=baseline.source_root, ) diff --git a/analysis-plugins/contracts/python/codecrow_plugins/plugin_glob.py b/analysis-plugins/contracts/python/codecrow_plugins/plugin_glob.py new file mode 100644 index 00000000..3edbe2fa --- /dev/null +++ b/analysis-plugins/contracts/python/codecrow_plugins/plugin_glob.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import re +from functools import lru_cache + + +@lru_cache(maxsize=512) +def _compiled(glob: str) -> re.Pattern[str]: + parts = ["^"] + index = 0 + while index < len(glob): + character = glob[index] + if character == "*": + recursive = index + 1 < len(glob) and glob[index + 1] == "*" + parts.append(".*" if recursive else "[^/]*") + index += 2 if recursive else 1 + continue + if character == "?": + parts.append("[^/]") + else: + parts.append(re.escape(character)) + index += 1 + parts.append("$") + return re.compile("".join(parts)) + + +def plugin_glob_matches(glob: str, path: str) -> bool: + """Match one normalized path with the Java contract's anchored glob rules.""" + return _compiled(glob).fullmatch(path) is not None diff --git a/analysis-plugins/contracts/python/codecrow_plugins/runtime.py b/analysis-plugins/contracts/python/codecrow_plugins/runtime.py index 6e104048..0ad5b7fa 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/runtime.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/runtime.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json + from .api import ( ArchitecturePacket, CandidateClaim, @@ -26,6 +28,8 @@ class PluginRuntime: """Host-side composition. Implementations return data; the host owns policy.""" MAX_FACTS_PER_FILE = 200 + MAX_GRAPH_FACT_STRING_LENGTH = 4_096 + MAX_GRAPH_FACT_BYTES_PER_ARTIFACT = 262_144 MAX_RULES = 40 MAX_EVIDENCE_REQUESTS = 80 MAX_REPOSITORY_SYMBOLS = 250_000 @@ -126,13 +130,21 @@ def file_disposition( disposition = FileDisposition.FULL for plugin_id in capabilities.repository_plugins: descriptor = self.catalog.registry.descriptor(plugin_id) + plugin_root = self._plugin_root_for_path( + descriptor.kind, + plugin_id, + path, + capabilities, + ) + if plugin_root is None: + continue if Capability.FILE_POLICY not in descriptor.capabilities: continue implementation = self.catalog.implementation(plugin_id) contributor = getattr(implementation, "file_disposition", None) if contributor is None: continue - outcome = contributor(path) + outcome = contributor(self._relative_to_root(path, plugin_root)) if outcome.status is OutcomeStatus.FAILED: raise RuntimeError( f"plugin file policy failed for {path}: {outcome.diagnostic.code}" @@ -157,16 +169,34 @@ def graph_facts( ) -> tuple[tuple[GraphFact, ...], tuple[PluginDiagnostic, ...]]: contributions: list[tuple[PluginKind, str, tuple[GraphFact, ...]]] = [] diagnostics: list[PluginDiagnostic] = [] + rejected: dict[str, list[int]] = {} for plugin_id in capabilities.repository_plugins: descriptor = self.catalog.registry.descriptor(plugin_id) + plugin_root = self._plugin_root_for_path( + descriptor.kind, + plugin_id, + artifact.path, + capabilities, + ) + if plugin_root is None: + continue if not ({Capability.INDEX, Capability.GRAPH} & set(descriptor.capabilities)): continue implementation = self.catalog.implementation(plugin_id) contributor = getattr(implementation, "index_file", None) if contributor is None: continue + plugin_artifact = ( + artifact + if not plugin_root + else FileArtifact( + self._relative_to_root(artifact.path, plugin_root), + artifact.content, + artifact.deleted, + ) + ) try: - outcome = contributor(artifact) + outcome = contributor(plugin_artifact) except Exception as exception: diagnostics.append( PluginDiagnostic( @@ -177,27 +207,100 @@ def graph_facts( ) continue if outcome.status is OutcomeStatus.FAILED: - diagnostics.append(outcome.diagnostic) + diagnostics.append(self._rebase_diagnostic( + outcome.diagnostic, + plugin_root, + )) elif outcome.status is OutcomeStatus.HANDLED: + valid_facts = [] + overlong_count = 0 + for raw_fact in tuple(outcome.value): + fact = self._rebase_fact(raw_fact, plugin_root) + if self._fact_has_overlong_string(fact): + overlong_count += 1 + else: + valid_facts.append(fact) + if overlong_count: + rejected.setdefault(plugin_id, [0, 0])[0] += overlong_count contributions.append(( descriptor.kind, plugin_id, - self._balanced_facts(tuple(outcome.value), self.MAX_FACTS_PER_FILE), + self._balanced_facts( + tuple(valid_facts), + self.MAX_FACTS_PER_FILE, + ), )) facts: set[GraphFact] = set() - for _, _, contribution in sorted( + serialized_bytes = 2 # The opening and closing brackets of the JSON array. + for _, plugin_id, contribution in sorted( contributions, key=lambda item: ( 1 if item[0] is PluginKind.LANGUAGE else 0, item[1], ), ): - remaining = self.MAX_FACTS_PER_FILE - len(facts) - if remaining <= 0: + if len(facts) >= self.MAX_FACTS_PER_FILE: break - facts.update(contribution[:remaining]) + for fact in contribution: + if len(facts) >= self.MAX_FACTS_PER_FILE: + break + if fact in facts: + continue + fact_bytes = self._serialized_fact_bytes(fact) + added_bytes = fact_bytes + (1 if facts else 0) + if ( + serialized_bytes + added_bytes + > self.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT + ): + rejected.setdefault(plugin_id, [0, 0])[1] += 1 + continue + facts.add(fact) + serialized_bytes += added_bytes + for plugin_id, (overlong_count, byte_count) in sorted(rejected.items()): + reasons = [] + if overlong_count: + reasons.append( + f"{overlong_count} fact(s) containing a string longer than " + f"{self.MAX_GRAPH_FACT_STRING_LENGTH} characters" + ) + if byte_count: + reasons.append( + f"{byte_count} fact(s) exceeding the " + f"{self.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT}-byte artifact budget" + ) + diagnostics.append(PluginDiagnostic( + code="plugin-index-output-limit", + message="graph output rejected " + " and ".join(reasons), + plugin_id=plugin_id, + path=artifact.path, + recoverable=True, + )) return tuple(sorted(facts)), tuple(diagnostics) + def _fact_has_overlong_string(self, fact: GraphFact) -> bool: + strings = ( + fact.kind, + fact.source, + fact.relation, + fact.target, + fact.path, + *(value for attribute in fact.attributes for value in attribute), + *fact.related_paths, + ) + return any( + len(value) > self.MAX_GRAPH_FACT_STRING_LENGTH + for value in strings + ) + + @staticmethod + def _serialized_fact_bytes(fact: GraphFact) -> int: + return len(json.dumps( + dict(fact.as_metadata()), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8")) + def syntax_contribution( self, path: str, @@ -292,12 +395,24 @@ def review_contribution( groups = set() diagnostics: list[PluginDiagnostic] = [] for plugin_id in capabilities.repository_plugins: + descriptor = self.catalog.registry.descriptor(plugin_id) + owned_paths = tuple( + path for path in paths + if self._plugin_root_for_path( + descriptor.kind, + plugin_id, + path, + capabilities, + ) is not None + ) + if not owned_paths: + continue implementation = self.catalog.implementation(plugin_id) contributor = getattr(implementation, "review", None) if contributor is None: continue try: - outcome = contributor(paths) + outcome = contributor(owned_paths) except Exception as exception: diagnostics.append( PluginDiagnostic( @@ -366,6 +481,14 @@ def validate_with_diagnostics( results: list[ValidationResult] = [] diagnostics: list[PluginDiagnostic] = [] for plugin_id in capabilities.repository_plugins: + descriptor = self.catalog.registry.descriptor(plugin_id) + if self._plugin_root_for_path( + descriptor.kind, + plugin_id, + claim.path, + capabilities, + ) is None: + continue implementation = self.catalog.implementation(plugin_id) validator = getattr(implementation, "validate", None) if validator is None: @@ -385,6 +508,80 @@ def validate_with_diagnostics( results.append(outcome.value) return tuple(results), tuple(diagnostics) + @staticmethod + def _plugin_root_for_path( + kind: PluginKind, + plugin_id: str, + path: str, + capabilities: ProjectCapabilities, + ) -> str | None: + if kind is PluginKind.LANGUAGE: + return "" if plugin_id in capabilities.file_plugins.get(path, ()) else None + if kind is not PluginKind.FRAMEWORK: + return "" + evidence = capabilities.detection_evidence.get(plugin_id, ()) + roots = tuple( + item.removeprefix("root:") + for item in evidence + if item.startswith("root:") + ) + if not roots: + # Legacy hand-built capabilities had no evidence, while older + # manual projections may only carry their explicit-selection tag. + # Repository-derived evidence without a root is incomplete and + # must not widen a framework contribution to the whole repository. + if not evidence or any( + item.startswith(( + "manual-project-type:", + "manual-project-type-dependency:", + )) + for item in evidence + ): + return "" + return None + matching = tuple( + "" if root == "." else root + for root in roots + if root == "." or path == root or path.startswith(root + "/") + ) + return max(matching, key=lambda root: (root.count("/"), len(root))) if matching else None + + @staticmethod + def _relative_to_root(path: str, root: str) -> str: + if not root: + return path + return path[len(root) + 1:] + + @classmethod + def _rebase_fact(cls, fact: GraphFact, root: str) -> GraphFact: + if not root: + return fact + return GraphFact( + fact.kind, + fact.source, + fact.relation, + fact.target, + f"{root}/{fact.path}", + fact.line, + fact.attributes, + tuple(f"{root}/{path}" for path in fact.related_paths), + ) + + @staticmethod + def _rebase_diagnostic( + diagnostic: PluginDiagnostic, + root: str, + ) -> PluginDiagnostic: + if not root or diagnostic.path is None: + return diagnostic + return PluginDiagnostic( + diagnostic.code, + diagnostic.message, + diagnostic.plugin_id, + f"{root}/{diagnostic.path}", + diagnostic.recoverable, + ) + class RepositoryAnalysisHandle: """Host-owned streaming composition of repository semantic contributors.""" diff --git a/analysis-plugins/contracts/python/codecrow_plugins/selection.py b/analysis-plugins/contracts/python/codecrow_plugins/selection.py index 2690b074..63b374ff 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/selection.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/selection.py @@ -2,15 +2,28 @@ import hashlib import json +from bisect import bisect_left +from collections.abc import Iterable from pathlib import PurePosixPath from typing import Mapping from .api import DetectionAlternative, PluginDescriptor, PluginKind, ProjectCapabilities, RepositoryFacts +from .plugin_glob import plugin_glob_matches from .registry import PluginRegistry MAX_DETECTION_EVIDENCE_PER_PLUGIN = 64 +def _bounded_evidence(evidence: Iterable[str]) -> tuple[str, ...]: + """Keep dispatch-critical roots when descriptive evidence is capped.""" + ordered = tuple(sorted(set(evidence))) + roots = tuple(item for item in ordered if item.startswith("root:")) + non_roots = tuple(item for item in ordered if not item.startswith("root:")) + retained_roots = roots[:MAX_DETECTION_EVIDENCE_PER_PLUGIN] + remaining = MAX_DETECTION_EVIDENCE_PER_PLUGIN - len(retained_roots) + return tuple(sorted((*retained_roots, *non_roots[:remaining]))) + + def _under_source_root(path: str, source_root: str | None) -> bool: return ( source_root is None @@ -42,6 +55,15 @@ def _under_root(path: str, root: str) -> str | None: return path[len(prefix):] if path.startswith(prefix) else None +def _paths_under_root(paths: tuple[str, ...], root: str) -> tuple[str, ...]: + if not root: + return paths + prefix = root + "/" + start = bisect_left(paths, prefix) + end = bisect_left(paths, prefix + "\U0010ffff", lo=start) + return paths[start:end] + + def _group_evidence(group: DetectionAlternative, facts: RepositoryFacts) -> tuple[str, ...] | None: candidate_sets = [ _suffix_roots(facts.paths, relative) @@ -68,7 +90,10 @@ def _group_evidence(group: DetectionAlternative, facts: RepositoryFacts) -> tupl if facts.source_root is not None: candidate_roots.intersection_update({facts.source_root}) - for root in sorted(candidate_roots, key=lambda value: (value.count("/"), value)): + matched_evidence: set[str] = set() + matched_roots = 0 + marker_items = tuple(sorted(facts.marker_contents.items())) + for root in sorted(candidate_roots): file_all_hits = tuple( f"{root}/{relative}" if root else relative for relative in group.files_all @@ -81,19 +106,19 @@ def _group_evidence(group: DetectionAlternative, facts: RepositoryFacts) -> tupl ) if group.files_any and not file_any_hits: continue + rooted_paths = _paths_under_root(facts.paths, root) + root_prefix_length = len(root) + 1 if root else 0 pattern_hits_all = { pattern: tuple( - path for path in facts.paths - if (relative := _under_root(path, root)) is not None - and PurePosixPath(relative).match(pattern) + path for path in rooted_paths + if plugin_glob_matches(pattern, path[root_prefix_length:]) ) for pattern in group.path_patterns_all } pattern_hits_any = { pattern: tuple( - path for path in facts.paths - if (relative := _under_root(path, root)) is not None - and PurePosixPath(relative).match(pattern) + path for path in rooted_paths + if plugin_glob_matches(pattern, path[root_prefix_length:]) ) for pattern in group.path_patterns_any } @@ -114,9 +139,9 @@ def _group_evidence(group: DetectionAlternative, facts: RepositoryFacts) -> tupl pattern_marker_hits = tuple( (marker, path) for marker in group.content_pattern_markers - for path, content in facts.marker_contents.items() + for path, content in marker_items if (relative := _under_root(path, root)) is not None - and PurePosixPath(relative).match(marker.path_pattern) + and plugin_glob_matches(marker.path_pattern, relative) and marker.contains in content ) if any( @@ -137,8 +162,11 @@ def _group_evidence(group: DetectionAlternative, facts: RepositoryFacts) -> tupl f"content-pattern:{marker.path_pattern}:{path}:{marker.contains}" for marker, path in pattern_marker_hits ) - return tuple(sorted(evidence)[:MAX_DETECTION_EVIDENCE_PER_PLUGIN]) - return None + matched_evidence.update(evidence) + matched_roots += 1 + if matched_roots == MAX_DETECTION_EVIDENCE_PER_PLUGIN: + break + return _bounded_evidence(matched_evidence) if matched_evidence else None def _rule_evidence(descriptor: PluginDescriptor, facts: RepositoryFacts) -> tuple[str, ...] | None: @@ -171,7 +199,7 @@ def _rule_evidence(descriptor: PluginDescriptor, facts: RepositoryFacts) -> tupl evidence = {f"extension:{path}" for path in extension_hits} for matched in group_evidence: evidence.update(matched) - return tuple(sorted(evidence)[:MAX_DETECTION_EVIDENCE_PER_PLUGIN]) + return _bounded_evidence(evidence) class ProjectSelector: diff --git a/analysis-plugins/contracts/python/tests/test_builtin_plugins.py b/analysis-plugins/contracts/python/tests/test_builtin_plugins.py index e409ebfc..9fa6634a 100644 --- a/analysis-plugins/contracts/python/tests/test_builtin_plugins.py +++ b/analysis-plugins/contracts/python/tests/test_builtin_plugins.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from pathlib import Path import pytest @@ -21,6 +22,20 @@ PLUGINS_ROOT = Path(__file__).resolve().parents[3] +def test_manifest_schema_does_not_accept_an_absent_pattern_marker_as_evidence(): + schema = json.loads( + (PLUGINS_ROOT / "contracts" / "manifest" / "plugin.schema.json") + .read_text(encoding="utf-8") + ) + alternatives = schema["$defs"]["detectionAlternative"]["anyOf"] + pattern_branch = next( + branch for branch in alternatives + if "contentPatternMarkers" in branch.get("properties", {}) + ) + + assert pattern_branch["required"] == ["contentPatternMarkers"] + + def test_python_host_accepts_an_empty_plugin_root(tmp_path: Path): (tmp_path / "languages").mkdir() (tmp_path / "frameworks").mkdir() @@ -150,6 +165,154 @@ def test_selected_language_plugins_own_their_syntax_declarations(): assert syntax.query_resource == "python/resources/rag-chunks.scm" +def test_source_root_bounds_language_and_framework_per_file_contributions(): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + runtime = PluginRuntime(catalog) + inside = "services/shop/app/views.py" + outside = "tools/views.py" + capabilities = ProjectSelector(catalog.registry).select(RepositoryFacts( + revision="0123456789abcdef", + paths=(inside, outside), + project_type="django", + source_root="services/shop", + )) + source = ( + "from django.views.decorators.http import require_GET\n" + "@require_GET\n" + "def health(request):\n" + " return None\n" + ) + + inside_facts, inside_diagnostics = runtime.graph_facts( + FileArtifact(inside, source), + capabilities, + ) + outside_facts, outside_diagnostics = runtime.graph_facts( + FileArtifact(outside, source), + capabilities, + ) + contribution, review_diagnostics = runtime.review_contribution( + (inside, outside), + capabilities, + ) + + assert inside_diagnostics == () + assert outside_diagnostics == () + assert {fact.kind for fact in inside_facts} >= { + "django-view", + "python-callable", + } + assert outside_facts == () + assert review_diagnostics == () + assert {request.identifier for request in contribution.evidence_requests} == { + inside, + } + + +def test_inferred_framework_root_uses_plugin_relative_identifiers_and_repository_paths(): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + runtime = PluginRuntime(catalog) + inside = "services/shop/shop/models.py" + outside = "tools/models.py" + capabilities = ProjectSelector(catalog.registry).select(RepositoryFacts( + revision="0123456789abcdef", + paths=( + "services/shop/manage.py", + "services/shop/project/settings.py", + "services/shop/project/urls.py", + inside, + outside, + ), + )) + source = "from django.db import models\nclass Order(models.Model):\n pass\n" + + inside_facts, inside_diagnostics = runtime.graph_facts( + FileArtifact(inside, source), + capabilities, + ) + outside_facts, outside_diagnostics = runtime.graph_facts( + FileArtifact(outside, source), + capabilities, + ) + + model = next(fact for fact in inside_facts if fact.kind == "django-model") + assert inside_diagnostics == () + assert model.target == "shop.models.Order" + assert model.path == inside + assert outside_diagnostics == () + assert not any(fact.kind.startswith("django-") for fact in outside_facts) + + +def test_evidence_cap_keeps_framework_root_and_never_widens_dispatch(): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + runtime = PluginRuntime(catalog) + root = "services/shop" + inside = f"{root}/shop/models.py" + outside = "tools/models.py" + paths = tuple(sorted({ + f"{root}/manage.py", + f"{root}/project/urls.py", + inside, + outside, + *(f"{root}/apps/app{index:02d}/settings.py" for index in range(70)), + })) + capabilities = ProjectSelector(catalog.registry).select(RepositoryFacts( + revision="0123456789abcdef", + paths=paths, + )) + source = "from django.db import models\nclass Order(models.Model):\n pass\n" + + outside_facts, diagnostics = runtime.graph_facts( + FileArtifact(outside, source), + capabilities, + ) + + django_evidence = capabilities.detection_evidence["django"] + assert len(django_evidence) == 64 + assert f"root:{root}" in django_evidence + assert diagnostics == () + assert not any(fact.kind.startswith("django-") for fact in outside_facts) + + incomplete = ProjectCapabilities( + repository_plugins=capabilities.repository_plugins, + file_plugins=capabilities.file_plugins, + detection_evidence={ + **capabilities.detection_evidence, + "django": tuple( + item for item in django_evidence + if not item.startswith("root:") + ), + }, + fingerprint="sha256:" + ("0" * 64), + descriptor_fingerprint=capabilities.descriptor_fingerprint, + ) + inside_facts, incomplete_diagnostics = runtime.graph_facts( + FileArtifact(inside, source), + incomplete, + ) + + assert incomplete_diagnostics == () + assert not any(fact.kind.startswith("django-") for fact in inside_facts) + + legacy_manual = ProjectCapabilities( + repository_plugins=capabilities.repository_plugins, + file_plugins=capabilities.file_plugins, + detection_evidence={ + **capabilities.detection_evidence, + "django": ("manual-project-type:django",), + }, + fingerprint="sha256:" + ("0" * 64), + descriptor_fingerprint=capabilities.descriptor_fingerprint, + ) + manual_facts, manual_diagnostics = runtime.graph_facts( + FileArtifact(inside, source), + legacy_manual, + ) + + assert manual_diagnostics == () + assert any(fact.kind == "django-model" for fact in manual_facts) + + def test_unassigned_file_uses_neutral_syntax_fallback_in_polyglot_repository(): catalog = PluginCatalog.discover(PLUGINS_ROOT) runtime = PluginRuntime(catalog) @@ -213,8 +376,10 @@ def test_discovers_and_selects_php_and_magento_deterministically(): assert set(catalog.registry.ordered_ids) == { "bash", "c", "c-sharp", "cpp", "css", "go", "haskell", "html", - "data-contracts", "fastapi", "hyva", "java", "javascript", "json", "magento", "php", "python", "ruby", "spring", - "rust", "scala", "tsx", "typescript", + "data-contracts", "django", "ember", "express", "fastapi", "hyva", + "java", "javascript", "json", "magento", "nextjs", "php", "python", + "quarkus", "rails", "ruby", "spring", "rust", "scala", "tsx", + "typescript", } assert capabilities.repository_plugins == ("json", "php", "magento") assert capabilities.file_plugins == { diff --git a/analysis-plugins/contracts/python/tests/test_django_framework_plugin.py b/analysis-plugins/contracts/python/tests/test_django_framework_plugin.py new file mode 100644 index 00000000..d52290fc --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_django_framework_plugin.py @@ -0,0 +1,446 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +from codecrow_plugins import ( + CandidateClaim, + FileArtifact, + OutcomeStatus, + PluginRegistry, + ProjectSelector, + RepositoryFacts, + ValidationDecision, + load_descriptor, +) + + +PLUGINS_ROOT = Path(__file__).resolve().parents[3] +DJANGO_ROOT = PLUGINS_ROOT / "frameworks" / "django" +sys.path.insert(0, str(DJANGO_ROOT / "python")) + +from codecrow_plugin_django import create_plugin # noqa: E402 + + +def _plugin(): + return create_plugin(load_descriptor(DJANGO_ROOT / "plugin.json")) + + +def _facts(path: str, content: str): + outcome = _plugin().index_file(FileArtifact(path, content)) + assert outcome.status is OutcomeStatus.HANDLED + return outcome.value + + +def test_django_detection_requires_one_coherent_project_root(): + registry = PluginRegistry(( + load_descriptor(PLUGINS_ROOT / "languages" / "python" / "plugin.json"), + load_descriptor(DJANGO_ROOT / "plugin.json"), + )) + selector = ProjectSelector(registry) + + split = selector.select(RepositoryFacts( + revision="0123456789abcdef", + paths=tuple(sorted(( + "backend/project/settings.py", + "backend/project/urls.py", + "frontend/manage.py", + ))), + )) + coherent = selector.select(RepositoryFacts( + revision="0123456789abcdef", + paths=tuple(sorted(( + "services/shop/manage.py", + "services/shop/project/settings.py", + "services/shop/project/urls.py", + ))), + )) + + assert split.repository_plugins == ("python",) + assert coherent.repository_plugins == ("python", "django") + assert "root:services/shop" in coherent.detection_evidence["django"] + + +def test_django_indexes_settings_apps_middleware_urls_views_models_and_signals(): + settings = _facts("project/settings.py", ''' +INSTALLED_APPS = ["shop.apps.ShopConfig"] +MIDDLEWARE = ["django.middleware.security.SecurityMiddleware"] +ROOT_URLCONF = "project.urls" +''') + urls = _facts("project/urls.py", ''' +from django.urls import include, path +from shop.views import OrderView +urlpatterns = [ + path("orders/", OrderView.as_view(), name="orders"), + path("api/", include("api.urls")), +] +''') + apps = _facts("shop/apps.py", ''' +from django.apps import AppConfig +class ShopConfig(AppConfig): + name = "shop" + label = "store" +''') + models = _facts("shop/models.py", ''' +from django.db import models +class Order(models.Model): + customer = models.ForeignKey("Customer", on_delete=models.CASCADE, related_name="orders") + tags = models.ManyToManyField("Tag") +''') + views = _facts("shop/views.py", ''' +from django.views import View +class OrderView(View): + def get(self, request): + pass +def health(request): + pass +''') + signals = _facts("shop/signals.py", ''' +from django.db.models.signals import post_save +from django.dispatch import receiver +@receiver(post_save, sender=Order) +def publish_order(sender, **kwargs): + pass +post_save.connect(publish_order, sender=Order) +''') + + all_facts = (*settings, *urls, *apps, *models, *views, *signals) + triples = {(fact.kind, fact.relation, fact.target) for fact in all_facts} + assert ("django-installed-app", "installs", "shop.apps.ShopConfig") in triples + assert ("django-middleware", "uses", "django.middleware.security.SecurityMiddleware") in triples + assert ("django-url-configuration", "uses", "project.urls") in triples + assert ("django-url-route", "dispatches-to", "OrderView.as_view") in triples + assert ("django-url-include", "includes", "api.urls") in triples + assert ("django-app-config", "configures", "shop") in triples + assert ("django-model", "declares", "shop.models.Order") in triples + assert ("django-model-relation", "many-to-one", "Customer") in triples + assert ("django-model-relation", "many-to-many", "Tag") in triples + assert ("django-view", "declares", "shop.views.OrderView") in triples + assert ("django-view-action", "handles", "GET") in triples + assert ("django-signal-receiver", "notifies", "shop.signals.publish_order") in triples + + +def test_django_resolves_import_aliases_for_owned_framework_symbols(): + apps = _facts("shop/apps.py", ''' +from django.apps.config import AppConfig as DjangoConfig +class ShopConfig(DjangoConfig): + name = "shop" +''') + models = _facts("shop/models.py", ''' +from django.db.models import CharField as Text, ForeignKey as BelongsTo, Model as DjangoModel +from django.db.models.deletion import CASCADE +class Order(DjangoModel): + reference = Text(max_length=20) + customer = BelongsTo("Customer", on_delete=CASCADE) +''') + urls = _facts("project/urls.py", ''' +from django.urls import include as nest, path as route, re_path as regex +from shop.views import OrderView +urlpatterns = [ + route("orders/", OrderView.as_view()), + regex(r"^api/", nest("api.urls")), +] +''') + views = _facts("shop/views.py", ''' +from django.views.decorators.http import require_GET as only_get +from django.views.generic import DetailView as DjangoDetailView +from rest_framework.viewsets import ModelViewSet as RestModelViewSet +class OrderDetail(DjangoDetailView): + def get(self, request): + pass +class OrderApi(RestModelViewSet): + def post(self, request): + pass +@only_get +def health(request): + pass +''') + signals = _facts("shop/signals.py", ''' +from django.db.models.signals import post_save as saved +from django.dispatch import Signal as Event, receiver as listens +@listens(saved, sender=Order) +def publish_order(sender, **kwargs): + pass +order_published = Event() +def publish_custom(sender, **kwargs): + pass +order_published.connect(publish_custom) +''') + + all_facts = (*apps, *models, *urls, *views, *signals) + triples = {(fact.kind, fact.relation, fact.target) for fact in all_facts} + assert ("django-app-config", "configures", "shop") in triples + assert ("django-model", "declares", "shop.models.Order") in triples + assert ("django-model-field", "declares", "shop.models.Order.reference") in triples + assert ("django-model-relation", "many-to-one", "Customer") in triples + assert ("django-url-route", "dispatches-to", "OrderView.as_view") in triples + assert ("django-url-include", "includes", "api.urls") in triples + assert ("django-view", "declares", "shop.views.OrderDetail") in triples + assert ("django-view", "declares", "shop.views.OrderApi") in triples + assert ("django-view", "declares", "shop.views.health") in triples + assert ("django-signal-receiver", "notifies", "shop.signals.publish_order") in triples + assert ("django-signal-receiver", "notifies", "shop.signals.publish_custom") in triples + + +def test_django_abstains_from_same_named_local_and_third_party_constructs(): + outcome = _plugin().index_file(FileArtifact("shop/views.py", ''' +from eventbus import post_save, receiver +from records import Model, TextField +from tables import TableView +class Order(Model): + label = TextField() +class OrderTable(TableView): + def get(self, request): + pass +def health(request): + pass +@receiver(post_save) +def publish_order(sender, **kwargs): + pass +post_save.connect(publish_order) +''')) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_django_abstains_from_unproven_url_helpers_and_dynamic_aliases(): + outcome = _plugin().index_file(FileArtifact("project/urls.py", ''' +from django.urls import path +def include(module): + return module +route = path +urlpatterns = [ + route("orders/", order_view), + path("api/", include("api.urls")), +] +''')) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_django_skips_symbolic_settings_and_route_values(): + settings = _facts("project/settings.py", ''' +APP_NAME = "dynamic.app" +MIDDLEWARE_NAME = "dynamic.Middleware" +URLCONF = "dynamic.urls" +INSTALLED_APPS = [APP_NAME, "shop.apps.ShopConfig"] +MIDDLEWARE = [MIDDLEWARE_NAME, "django.middleware.security.SecurityMiddleware"] +ROOT_URLCONF = URLCONF +''') + urls = _facts("project/urls.py", ''' +from django.urls import path +from shop.views import OrderView +PREFIX = "dynamic/" +urlpatterns = [ + path(PREFIX, OrderView.as_view()), + path("orders/", OrderView.as_view()), +] +''') + + triples = {(fact.kind, fact.target) for fact in (*settings, *urls)} + assert ("django-installed-app", "shop.apps.ShopConfig") in triples + assert ("django-middleware", "django.middleware.security.SecurityMiddleware") in triples + assert ("django-url-route", "OrderView.as_view") in triples + assert not any(fact.target == "APP_NAME" for fact in settings) + assert not any(fact.target == "MIDDLEWARE_NAME" for fact in settings) + assert not any(fact.kind == "django-url-configuration" for fact in settings) + assert not any(fact.source.endswith(":PREFIX") for fact in urls) + + +def test_django_requires_owned_model_fields_and_static_function_view_decorators(): + facts = _facts("shop/views.py", ''' +from django.db import models +from django.views.decorators.http import require_GET +from toolkit import view_decorator +class Order(models.Model): + owned = models.CharField(max_length=20) + unproven = CustomField() +def undecorated(request): + pass +@view_decorator +def third_party_decorated(request): + pass +@require_GET +def health(request): + pass +''') + + targets = {(fact.kind, fact.target) for fact in facts} + assert ("django-model-field", "shop.views.Order.owned") in targets + assert ("django-model-field", "shop.views.Order.unproven") not in targets + assert ("django-view", "shop.views.undecorated") not in targets + assert ("django-view", "shop.views.third_party_decorated") not in targets + assert ("django-view", "shop.views.health") in targets + + +def test_django_abstains_after_framework_imports_or_custom_signals_are_rebound(): + outcome = _plugin().index_file(FileArtifact("shop/views.py", ''' +from django.db import models +from django.dispatch import Signal +from django.views.decorators.http import require_GET +models = record_library +require_GET = custom_decorator +event = Signal() +event = event_bus +class Order(models.Model): + label = models.CharField(max_length=20) +@require_GET +def health(request): + pass +event.connect(health) +''')) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_django_abstains_after_conditional_module_rebinding(): + outcome = _plugin().index_file(FileArtifact("shop/models.py", ''' +from django.db import models +if use_alternate_records: + models = record_library +class Order(models.Model): + label = models.CharField(max_length=20) +''')) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_django_signal_connect_respects_nested_scope_shadowing(): + shadowed = _plugin().index_file(FileArtifact("shop/apps.py", ''' +from django.db.models.signals import post_save +def ready(post_save): + post_save.connect(handler) +''')) + proven = _facts("shop/apps.py", ''' +from django.db.models.signals import post_save +def ready(): + post_save.connect(handler) +''') + + assert shadowed.status is OutcomeStatus.ABSTAINED + assert any( + fact.kind == "django-signal-receiver" + and fact.target == "shop.apps.handler" + for fact in proven + ) + + +def test_django_does_not_treat_an_arbitrary_connect_call_as_a_signal(): + outcome = _plugin().index_file(FileArtifact( + "shop/services.py", + "client.connect(handler, sender=Order)\n", + )) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_django_does_not_treat_arbitrary_connect_as_signal_even_in_signals_module(): + outcome = _plugin().index_file(FileArtifact( + "shop/signals.py", + "from eventbus import order_saved\norder_saved.connect(handler)\n", + )) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_django_validation_rejects_only_a_relevant_contradicted_absence(): + model_fact = next( + fact for fact in _facts("shop/models.py", ''' +from django.db import models +class Order(models.Model): + pass +''') + if fact.kind == "django-model" + ) + rejected = _plugin().validate(CandidateClaim( + category="django-model", + claim_kind="django-model", + path="shop/models.py", + line=2, + message="The Django model shop.models.Order is missing.", + evidence=(model_fact,), + )) + contextual = _plugin().validate(CandidateClaim( + category="django-model", + claim_kind="django-model", + path="shop/models.py", + line=2, + message="The Order model may persist the wrong state.", + evidence=(model_fact,), + )) + + assert rejected.value.decision is ValidationDecision.REJECT + assert rejected.value.code == "django-absence-contradicted" + assert contextual.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert contextual.value.code == "django-topology-not-defect-proof" + + +def test_django_validation_does_not_match_get_inside_an_unrelated_word(): + action_fact = next( + fact for fact in _facts("shop/views.py", ''' +from django.views import View +class OrderView(View): + def get(self, request): + pass +''') + if fact.kind == "django-view-action" + ) + result = _plugin().validate(CandidateClaim( + category="django-view-action", + claim_kind="django-view-action", + path="shop/views.py", + line=3, + message="The target widget is missing.", + evidence=(action_fact,), + )) + + assert result.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert result.value.code == "django-cited-identifier-mismatch" + + +def test_django_validation_does_not_bind_an_unrelated_absence_to_a_model(): + model_fact = next( + fact for fact in _facts("shop/models.py", ''' +from django.db import models +class Order(models.Model): + pass +''') + if fact.kind == "django-model" + ) + unrelated = _plugin().validate(CandidateClaim( + category="django-model", + claim_kind="django-model", + path="shop/models.py", + line=2, + message="Order fails closed when its cache entry is missing.", + evidence=(model_fact,), + )) + unknown = _plugin().validate(CandidateClaim( + category="django-cache", + claim_kind="django-cache", + path="shop/models.py", + line=2, + message="The Order cache is missing.", + evidence=(model_fact,), + )) + + assert unrelated.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert unrelated.value.code == "django-topology-not-defect-proof" + assert unknown.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert unknown.value.code == "django-unknown-fact-kind" + + +def test_django_direct_indexing_and_review_contributions_are_bounded(): + fields = "\n".join( + f" field_{index} = models.CharField(max_length=20)" + for index in range(300) + ) + facts = _facts( + "shop/models.py", + "from django.db import models\nclass Large(models.Model):\n" + fields, + ) + review = _plugin().review(tuple(f"app_{index}/models.py" for index in range(100))) + + assert len(facts) == 160 + assert {fact.kind for fact in facts} == {"django-model", "django-model-field"} + assert len(review.value.evidence_requests) == 40 diff --git a/analysis-plugins/contracts/python/tests/test_javascript_framework_plugins.py b/analysis-plugins/contracts/python/tests/test_javascript_framework_plugins.py new file mode 100644 index 00000000..81cf4c4a --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_javascript_framework_plugins.py @@ -0,0 +1,780 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from codecrow_plugins import ( + CandidateClaim, + FileArtifact, + GraphFact, + OutcomeStatus, + PluginCatalog, + PluginRuntime, + ProjectSelector, + RepositoryFacts, + ValidationDecision, +) + + +PLUGINS_ROOT = Path(__file__).resolve().parents[3] + + +@pytest.fixture(scope="module") +def catalog() -> PluginCatalog: + return PluginCatalog.discover(PLUGINS_ROOT) + + +@pytest.mark.parametrize( + ("framework", "dependency", "source_path", "language"), + ( + ("ember", "ember-source", "packages/web/app/router.js", "javascript"), + ("express", "express", "packages/web/src/server.js", "javascript"), + ("nextjs", "next", "packages/web/src/app/page.js", "javascript"), + ("ember", "ember-source", "packages/web/app/router.ts", "typescript"), + ("express", "express", "packages/web/src/server.ts", "typescript"), + ("nextjs", "next", "packages/web/src/app/page.tsx", "tsx"), + ), +) +def test_package_root_detection_supports_javascript_and_typescript_only_sources( + catalog: PluginCatalog, + framework: str, + dependency: str, + source_path: str, + language: str, +): + package_path = "packages/web/package.json" + capabilities = ProjectSelector(catalog.registry).select(RepositoryFacts( + revision="0123456789abcdef", + paths=tuple(sorted((package_path, source_path))), + marker_contents={ + package_path: f'{{"dependencies":{{"{dependency}":"latest"}}}}', + }, + )) + + assert {"json", language, framework} <= set(capabilities.repository_plugins) + assert capabilities.file_plugins[source_path] == (language,) + assert f"root:packages/web" in capabilities.detection_evidence[framework] + + +def test_package_detection_does_not_join_source_roots(catalog: PluginCatalog): + capabilities = ProjectSelector(catalog.registry).select(RepositoryFacts( + revision="0123456789abcdef", + paths=("one/package.json", "two/src/server.ts"), + marker_contents={"one/package.json": '{"dependencies":{"express":"latest"}}'}, + source_root="two", + )) + + assert "express" not in capabilities.repository_plugins + + +def test_package_detection_retains_every_matching_framework_root( + catalog: PluginCatalog, +): + capabilities = ProjectSelector(catalog.registry).select(RepositoryFacts( + revision="0123456789abcdef", + paths=( + "apps/admin/package.json", + "apps/admin/server.js", + "apps/store/package.json", + "apps/store/server.js", + ), + marker_contents={ + "apps/admin/package.json": '{"dependencies":{"express":"latest"}}', + "apps/store/package.json": '{"dependencies":{"express":"latest"}}', + }, + )) + + assert {"root:apps/admin", "root:apps/store"} <= set( + capabilities.detection_evidence["express"] + ) + runtime = PluginRuntime(catalog) + for path in ("apps/admin/server.js", "apps/store/server.js"): + facts, diagnostics = runtime.graph_facts( + FileArtifact( + path, + "import express from 'express'; const app = express();", + ), + capabilities, + ) + assert not diagnostics + assert any( + fact.kind == "express-application" and fact.path == path + for fact in facts + ) + + +def test_ember_indexes_nested_routes_framework_roles_and_templates(catalog: PluginCatalog): + plugin = catalog.implementation("ember") + artifacts = ( + FileArtifact( + "app/router.js", + """import EmberRouter from '@ember/routing/router'; +export default class Router extends EmberRouter {} +Router.map(function () { + this.route('posts', { path: '/articles' }, function () { + this.route('new'); + }); + this.route('admin', function () { + this.route('users', { path: 'people' }); + }); +});""", + ), + FileArtifact( + "app/routes/posts.ts", + """import Route from '@ember/routing/route'; +import { service } from '@ember/service'; +export default class PostsRoute extends Route { + @service('session') currentUser; +}""", + ), + FileArtifact( + "app/models/post.ts", + """import Model, { belongsTo, hasMany } from '@ember-data/model'; +export default class Post extends Model { + @belongsTo('user') author; + @hasMany('comment') comments; +}""", + ), + FileArtifact("app/templates/posts.hbs", " {{legacy-card}}"), + ) + + facts = { + fact + for artifact in artifacts + for fact in plugin.index_file(artifact).value + } + + nested = next(fact for fact in facts if fact.kind == "ember-route" and fact.target == "posts.new") + assert dict(nested.attributes) == {"parent": "posts", "path": "/articles/new"} + parent = next(fact for fact in facts if fact.kind == "ember-route" and fact.target == "admin") + nested_option = next( + fact for fact in facts if fact.kind == "ember-route" and fact.target == "admin.users" + ) + assert dict(parent.attributes)["path"] == "/admin" + assert dict(nested_option.attributes)["path"] == "/admin/people" + assert any( + fact.kind == "ember-service-injection" + and fact.source == "posts" + and fact.target == "session" + for fact in facts + ) + assert { + (fact.relation, fact.target, dict(fact.attributes)["property"]) + for fact in facts + if fact.kind == "ember-data-relationship" + } == { + ("belongs-to", "user", "author"), + ("has-many", "comment", "comments"), + } + assert { + fact.target for fact in facts if fact.kind == "ember-template-component" + } == {"PostList", "legacy-card"} + assert any( + fact.kind == "ember-template-association" + and fact.relation == "renders-route" + and fact.target == "posts" + for fact in facts + ) + + +def test_ember_text_fallbacks_ignore_comments_and_string_literals(catalog: PluginCatalog): + plugin = catalog.implementation("ember") + model_facts = plugin.index_file(FileArtifact( + "app/models/post.ts", + """export default class Post extends Model { + // @belongsTo('user') author; + note = "@hasMany('comment') comments"; +}""", + )).value + template_facts = plugin.index_file(FileArtifact( + "app/templates/posts.hbs", + "{{!-- --}} {{! {{old-commented}} }} ", + )).value + + assert not any(fact.kind == "ember-data-relationship" for fact in model_facts) + assert { + fact.target for fact in template_facts if fact.kind == "ember-template-component" + } == {"RealCard"} + + +def test_ember_routes_require_the_exported_app_router_owner(catalog: PluginCatalog): + plugin = catalog.implementation("ember") + facts = plugin.index_file(FileArtifact( + "app/router.js", + """import EmberRouter from '@ember/routing/router'; +export default class Router extends EmberRouter {} +Router.map(function () { this.route('real'); }); +client.map(function () { this.route('fake'); });""", + )).value + + assert { + fact.target for fact in facts if fact.kind == "ember-route" + } == {"real"} + + lookalike = plugin.index_file(FileArtifact( + "app/router.js", + "client.map(function () { this.route('fake'); });", + )) + assert lookalike.status is OutcomeStatus.ABSTAINED + + +def test_ember_service_and_model_macros_require_visible_proven_imports( + catalog: PluginCatalog, +): + plugin = catalog.implementation("ember") + service_facts = plugin.index_file(FileArtifact( + "app/routes/posts.ts", + """import Route from '@ember/routing/route'; +import { inject as useService } from '@ember/service'; +export default class PostsRoute extends Route { @useService('session') currentUser; } +function lookalike(useService) { + class FakeRoute { @useService('fake') fakeService; } +}""", + )).value + model_facts = plugin.index_file(FileArtifact( + "app/models/post.ts", + """import Model, { belongsTo as ownerOf } from '@ember-data/model'; +export default class Post extends Model { @ownerOf('user') author; } +function lookalike(ownerOf) { + class FakeModel { @ownerOf('fake') fakeOwner; } +}""", + )).value + unproven = plugin.index_file(FileArtifact( + "app/services/cart.ts", + "export default class Cart { @service('session') currentUser; }", + )).value + + assert { + fact.target for fact in service_facts if fact.kind == "ember-service-injection" + } == {"session"} + assert { + fact.target for fact in model_facts if fact.kind == "ember-data-relationship" + } == {"user"} + assert not any(fact.kind == "ember-service-injection" for fact in unproven) + + +def test_express_indexes_routes_mounts_middleware_and_error_handlers(catalog: PluginCatalog): + plugin = catalog.implementation("express") + outcome = plugin.index_file(FileArtifact( + "src/server.ts", + """import express, { Router as CreateRouter } from 'express'; +const app = express(); +const router = CreateRouter(); +const audit = (_req, _res, next) => next(); +function failures(error, request, response, next) { next(error); } +router.route('/users/:id').get(audit, showUser).delete(deleteUser); +app.use('/api', router); +app.use(audit); +app.use(failures);""", + )) + + assert outcome.status is OutcomeStatus.HANDLED + facts = set(outcome.value) + assert any(fact.kind == "express-application" and fact.target == "app" for fact in facts) + assert any(fact.kind == "express-router" and fact.target == "router" for fact in facts) + assert { + fact.target for fact in facts if fact.kind == "express-route" + } == {"DELETE /users/:id", "GET /users/:id"} + assert any( + fact.kind == "express-mount" + and fact.source == "app" + and fact.target == "router" + and dict(fact.attributes)["mountPath"] == "/api" + for fact in facts + ) + assert any(fact.kind == "express-middleware" and fact.target == "audit" for fact in facts) + assert any(fact.kind == "express-error-handler" and fact.target == "failures" for fact in facts) + + +def test_express_does_not_guess_that_unknown_path_middleware_is_a_router_mount(catalog: PluginCatalog): + plugin = catalog.implementation("express") + facts = plugin.index_file(FileArtifact( + "server.js", + """const express = require('express'); +const app = express(); +app.use('/private', authenticate);""", + )).value + + assert any(fact.kind == "express-middleware" and fact.target == "authenticate" for fact in facts) + assert not any(fact.kind == "express-mount" for fact in facts) + + +def test_express_does_not_treat_commented_imports_as_factory_evidence(catalog: PluginCatalog): + plugin = catalog.implementation("express") + outcome = plugin.index_file(FileArtifact( + "server.ts", + """// import express, { Router } from 'express'; +const app = express(); +app.get('/not-express-evidence', handler);""", + )) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +@pytest.mark.parametrize( + "content", + ( + """import express from 'express'; +function build(express) { + const app = express(); + app.get('/fake', handler); +}""", + """import { Router } from 'express'; +function build(Router) { + const router = Router(); + router.get('/fake', handler); +}""", + ), +) +def test_express_factories_respect_parameter_shadowing( + catalog: PluginCatalog, + content: str, +): + outcome = catalog.implementation("express").index_file(FileArtifact("server.ts", content)) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_express_suppresses_owner_facts_after_rebinding(catalog: PluginCatalog): + outcome = catalog.implementation("express").index_file(FileArtifact( + "server.ts", + """import express from 'express'; +let app = express(); +app = fakeRouter; +app.get('/fake', handler);""", + )) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_nextjs_indexes_both_routers_handlers_layouts_boundaries_and_loaders(catalog: PluginCatalog): + plugin = catalog.implementation("nextjs") + artifacts = ( + FileArtifact( + "src/app/(shop)/products/[id]/page.tsx", + """// A leading license or framework comment is not a statement. +'use strict'; +'use client'; +export default function ProductPage() { return
; }""", + ), + FileArtifact( + "src/app/(shop)/products/layout.tsx", + "export default function Layout({ children }) { return children; }", + ), + FileArtifact( + "src/app/api/products/route.ts", + """export async function GET() { return Response.json([]); } +export const POST = async () => new Response(null, { status: 201 });""", + ), + FileArtifact( + "pages/blog/[slug].tsx", + """export async function getServerSideProps() { return { props: {} }; } +export default function BlogPost() { return null; }""", + ), + FileArtifact( + "pages/api/orders.ts", + """export default function handler(req, res) { + if (req.method === 'POST') res.end(); +}""", + ), + FileArtifact( + "src/middleware.ts", + """export const config = { matcher: ['/account/:path*', '/admin/:path*'] }; +export function middleware(request) { return NextResponse.next(); }""", + ), + ) + + facts = { + fact + for artifact in artifacts + for fact in plugin.index_file(artifact).value + } + + assert any( + fact.kind == "nextjs-page-route" and fact.target == "/products/[id]" + and dict(fact.attributes)["router"] == "app" + for fact in facts + ) + assert any( + fact.kind == "nextjs-layout" and fact.relation == "wraps" and fact.target == "/products" + for fact in facts + ) + assert any(fact.kind == "nextjs-client-boundary" for fact in facts) + assert any( + fact.kind == "nextjs-server-boundary" + and fact.path.endswith("layout.tsx") + for fact in facts + ) + assert { + fact.target for fact in facts + if fact.kind == "nextjs-route-handler" and fact.path.endswith("route.ts") + } == {"GET /api/products", "POST /api/products"} + assert any( + fact.kind == "nextjs-route-handler" and fact.target == "POST /api/orders" + for fact in facts + ) + assert any( + fact.kind == "nextjs-data-loader" and fact.target == "getServerSideProps" + and fact.source == "/blog/[slug]" + for fact in facts + ) + assert { + fact.target for fact in facts if fact.kind == "nextjs-middleware" + } == {"/account/:path*", "/admin/:path*"} + + +def test_nextjs_server_action_requires_a_function_directive_prologue(catalog: PluginCatalog): + plugin = catalog.implementation("nextjs") + facts = plugin.index_file(FileArtifact( + "src/app/actions.ts", + """export async function save() { + // Comments may precede a directive. + 'use strict'; + 'use server'; + await persist(); +} +export async function notAnAction() { + await inspect(); + 'use server'; +}""", + )).value + + assert { + fact.target for fact in facts if fact.kind == "nextjs-server-action" + } == {"save"} + + +def test_nextjs_pages_api_method_checks_are_scoped_to_the_exported_handler(catalog: PluginCatalog): + plugin = catalog.implementation("nextjs") + facts = plugin.index_file(FileArtifact( + "pages/api/items.ts", + """function helper(req) { + if (req.method === 'DELETE') return true; +} +export default function handler(req, res) { res.end(); }""", + )).value + + assert { + fact.target for fact in facts if fact.kind == "nextjs-route-handler" + } == {"ANY /api/items"} + + +def test_nextjs_resolves_a_separately_exported_static_middleware_config(catalog: PluginCatalog): + plugin = catalog.implementation("nextjs") + facts = plugin.index_file(FileArtifact( + "middleware.ts", + """const config = { matcher: ['/private/:path*'] }; +export { config }; +export function middleware() {}""", + )).value + + assert { + fact.target for fact in facts if fact.kind == "nextjs-middleware" + } == {"/private/:path*"} + + +def test_nextjs_http_exports_are_exact_bindings_not_incidental_names(catalog: PluginCatalog): + plugin = catalog.implementation("nextjs") + facts = plugin.index_file(FileArtifact( + "src/app/api/items/route.ts", + """const read = async () => Response.json([]); +export { read as GET }; +export const description = 'POST'; +export function helper() { const DELETE = 'not exported'; return DELETE; }""", + )).value + + assert { + fact.target for fact in facts if fact.kind == "nextjs-route-handler" + } == {"GET /api/items"} + + +@pytest.mark.parametrize( + ("path", "expected_route"), + ( + ("pages/help.tsx", "/help"), + ("src/pages/help.tsx", "/help"), + ("app/help/page.tsx", "/help"), + ("src/app/help/page.tsx", "/help"), + ), +) +def test_nextjs_route_roots_are_anchored_but_supported_roots_remain_exact( + catalog: PluginCatalog, + path: str, + expected_route: str, +): + facts = catalog.implementation("nextjs").index_file(FileArtifact( + path, + "export default function Page() { return null; }", + )).value + + assert any( + fact.kind == "nextjs-page-route" and fact.target == expected_route + for fact in facts + ) + + +def test_nextjs_ignores_nested_lookalike_roots_and_private_app_segments( + catalog: PluginCatalog, +): + plugin = catalog.implementation("nextjs") + for path in ("docs/pages/help.tsx", "src/app/_private/demo/page.tsx"): + outcome = plugin.index_file(FileArtifact( + path, + "export default function Page() { return null; }", + )) + assert outcome.status is OutcomeStatus.ABSTAINED + + +@pytest.mark.parametrize( + ("path", "content"), + ( + ("app/help/page.tsx", "export function Page() { return null; }"), + ("app/help/layout.tsx", "export function Layout({ children }) { return children; }"), + ("pages/help.tsx", "export function Page() { return null; }"), + ("pages/api/items.ts", "export function handler(req, res) { res.end(); }"), + ("app/api/items/route.ts", "export type GET = () => Response;"), + ("app/api/items/route.ts", "type GET = () => Response; export { type GET };"), + ("app/api/items/route.ts", "export const GET = 'not a function';"), + ("middleware.ts", "export const config = { matcher: '/private' };"), + ), +) +def test_nextjs_requires_runtime_exports_for_file_system_facts( + catalog: PluginCatalog, + path: str, + content: str, +): + outcome = catalog.implementation("nextjs").index_file(FileArtifact(path, content)) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +@pytest.mark.parametrize( + ("plugin_id", "path", "content", "diagnostic"), + ( + ("ember", "app/router.js", "Router.map(function( {", "ember-script-syntax-error"), + ("express", "src/server.ts", "const app = express(;", "express-script-syntax-error"), + ("nextjs", "src/app/page.tsx", "export default function Page( {", "nextjs-script-syntax-error"), + ), +) +def test_framework_indexers_do_not_emit_partial_facts_from_error_trees( + catalog: PluginCatalog, + plugin_id: str, + path: str, + content: str, + diagnostic: str, +): + outcome = catalog.implementation(plugin_id).index_file(FileArtifact(path, content)) + + assert outcome.status is OutcomeStatus.FAILED + assert outcome.value is None + assert outcome.diagnostic.code == diagnostic + assert outcome.diagnostic.recoverable is True + + +@pytest.mark.parametrize( + ("plugin_id", "path", "fact", "absence_message"), + ( + ( + "ember", + "app/router.js", + GraphFact("ember-route", "Router", "declares", "posts", "app/router.js", 2), + "The Router does not declare the posts route.", + ), + ( + "express", + "src/server.js", + GraphFact("express-route", "application", "handles", "GET /health", "src/server.js", 4), + "The application does not handle GET /health.", + ), + ( + "nextjs", + "src/app/products/page.tsx", + GraphFact("nextjs-page-route", "page module", "defines", "/products", "src/app/products/page.tsx", 1), + "The page module does not define /products.", + ), + ), +) +def test_framework_validation_rejects_contradicted_absence_but_never_treats_topology_as_defect_proof( + catalog: PluginCatalog, + plugin_id: str, + path: str, + fact: GraphFact, + absence_message: str, +): + plugin = catalog.implementation(plugin_id) + contradicted = plugin.validate(CandidateClaim( + category="bug-risk", + path=path, + line=fact.line, + message=absence_message, + evidence=(fact,), + claim_kind=fact.kind, + )) + topology_only = plugin.validate(CandidateClaim( + category="bug-risk", + path=path, + line=fact.line, + message=f"{fact.source} has framework topology related to {fact.target}.", + evidence=(fact,), + claim_kind=fact.kind, + )) + + assert contradicted.value.decision is ValidationDecision.REJECT + assert topology_only.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + + +@pytest.mark.parametrize( + ("plugin_id", "path", "fact", "message"), + ( + ( + "ember", + "app/templates/admin.hbs", + GraphFact( + "ember-template-component", "app/templates/admin.hbs", "invokes", + "AdminPanel", "app/templates/admin.hbs", 1, + ), + "AdminPanel is not rendered when account data is missing.", + ), + ( + "express", + "src/server.js", + GraphFact( + "express-route", "application", "handles", "GET /health", + "src/server.js", 4, + ), + "GET /health is not handled when authentication fails.", + ), + ( + "nextjs", + "app/api/items/route.ts", + GraphFact( + "nextjs-route-handler", "/api/items", "handles", "GET /api/items", + "app/api/items/route.ts", 1, + ), + "GET /api/items is not handled when authentication fails.", + ), + ), +) +def test_framework_validation_does_not_confuse_unsafe_behavior_with_absence( + catalog: PluginCatalog, + plugin_id: str, + path: str, + fact: GraphFact, + message: str, +): + outcome = catalog.implementation(plugin_id).validate(CandidateClaim( + category="bug-risk", + path=path, + line=fact.line, + message=message, + evidence=(fact,), + claim_kind=fact.kind, + )) + + assert outcome.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert "absence" not in outcome.value.code + + +@pytest.mark.parametrize( + ("plugin_id", "path", "category", "fact"), + ( + ( + "ember", "app/router.js", "ember-invented-kind", + GraphFact("ember-route", "Router", "declares", "posts", "app/router.js", 1), + ), + ( + "express", "src/server.js", "express-invented-kind", + GraphFact( + "express-route", "application", "handles", "GET /health", + "src/server.js", 1, + ), + ), + ( + "nextjs", "app/page.tsx", "nextjs-invented-kind", + GraphFact("nextjs-page-route", "app/page.tsx", "defines", "/", "app/page.tsx", 1), + ), + ), +) +def test_framework_validation_does_not_treat_unknown_prefixed_categories_as_umbrella( + catalog: PluginCatalog, + plugin_id: str, + path: str, + category: str, + fact: GraphFact, +): + outcome = catalog.implementation(plugin_id).validate(CandidateClaim( + category=category, + path=path, + line=1, + message=f"{fact.target} is missing.", + evidence=(fact,), + )) + + assert outcome.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert outcome.value.code == f"{plugin_id}-unknown-fact-kind" + + +def test_express_validation_does_not_match_short_identifier_inside_an_unrelated_word( + catalog: PluginCatalog, +): + plugin = catalog.implementation("express") + fact = GraphFact("express-application", "server.js", "declares", "app", "server.js", 1) + + outcome = plugin.validate(CandidateClaim( + category="bug-risk", + path="server.js", + line=1, + message="The wrapper does not declare a server factory.", + evidence=(fact,), + claim_kind="express-application", + )) + + assert outcome.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + + +def test_ember_conventional_template_lookup_does_not_prove_the_template_file_exists( + catalog: PluginCatalog, +): + plugin = catalog.implementation("ember") + fact = GraphFact( + "ember-template-association", + "posts", + "uses-conventional-template", + "posts", + "app/routes/posts.ts", + 1, + (("ownerKind", "route"),), + ) + + outcome = plugin.validate(CandidateClaim( + category="bug-risk", + path="app/routes/posts.ts", + line=1, + message="The posts route is missing template file posts.hbs.", + evidence=(fact,), + claim_kind="ember-template-association", + )) + + assert outcome.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + + +@pytest.mark.parametrize( + ("plugin_id", "path", "kind"), + ( + ("ember", "app/router.ts", "ember-framework"), + ("express", "src/server.ts", "express-framework"), + ("nextjs", "src/app/page.tsx", "nextjs-framework"), + ), +) +def test_framework_review_requests_exact_evidence( + catalog: PluginCatalog, + plugin_id: str, + path: str, + kind: str, +): + contribution = catalog.implementation(plugin_id).review((path,)).value + + assert [request.kind for request in contribution.evidence_requests] == [kind] + assert contribution.evidence_requests[0].identifier == path + assert any("topology" in rule.casefold() for rule in contribution.rules) diff --git a/analysis-plugins/contracts/python/tests/test_plugin_glob.py b/analysis-plugins/contracts/python/tests/test_plugin_glob.py new file mode 100644 index 00000000..f12dade0 --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_plugin_glob.py @@ -0,0 +1,21 @@ +import json +from pathlib import Path + +from codecrow_plugins.plugin_glob import plugin_glob_matches + + +FIXTURE = ( + Path(__file__).resolve().parents[3] + / "contracts" + / "fixtures" + / "plugin-globs.json" +) + + +def test_plugin_globs_match_the_shared_anchored_projection(): + cases = json.loads(FIXTURE.read_text(encoding="utf-8")) + + assert [ + plugin_glob_matches(case["glob"], case["path"]) + for case in cases + ] == [case["matches"] for case in cases] diff --git a/analysis-plugins/contracts/python/tests/test_quarkus_plugin.py b/analysis-plugins/contracts/python/tests/test_quarkus_plugin.py new file mode 100644 index 00000000..e30fbbe9 --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_quarkus_plugin.py @@ -0,0 +1,390 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +from codecrow_plugins import ( + CandidateClaim, + FileArtifact, + OutcomeStatus, + PluginRegistry, + ProjectSelector, + RepositoryFacts, + ValidationDecision, + load_descriptor, +) + + +PLUGINS_ROOT = Path(__file__).resolve().parents[3] +QUARKUS_ROOT = PLUGINS_ROOT / "frameworks/quarkus" +JAVA_DESCRIPTOR = PLUGINS_ROOT / "languages/java/plugin.json" +QUARKUS_DESCRIPTOR = QUARKUS_ROOT / "plugin.json" +JAVA_PATH = "services/catalog/src/main/java/example/ItemResource.java" + + +def _plugin(): + module_name = "_codecrow_test_quarkus" + module = sys.modules.get(module_name) + if module is None: + package = QUARKUS_ROOT / "python/codecrow_plugin_quarkus" + spec = importlib.util.spec_from_file_location( + module_name, + package / "__init__.py", + submodule_search_locations=[str(package)], + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module.create_plugin(load_descriptor(QUARKUS_DESCRIPTOR)) + + +def _selector() -> ProjectSelector: + return ProjectSelector(PluginRegistry(( + load_descriptor(JAVA_DESCRIPTOR), + load_descriptor(QUARKUS_DESCRIPTOR), + ))) + + +@pytest.mark.parametrize( + ("build_file", "content"), + ( + ( + "pom.xml", + "io.quarkus", + ), + ("build.gradle", "plugins { id 'io.quarkus' }"), + ("build.gradle.kts", 'plugins { id("io.quarkus") }'), + ), +) +def test_detects_nested_quarkus_builds_at_one_coherent_root( + build_file: str, + content: str, +): + build_path = f"services/catalog/{build_file}" + capabilities = _selector().select(RepositoryFacts( + revision="0123456789abcdef", + paths=tuple(sorted((build_path, JAVA_PATH))), + marker_contents={build_path: content}, + )) + + assert capabilities.repository_plugins == ("java", "quarkus") + assert "root:services/catalog" in capabilities.detection_evidence["quarkus"] + + +def test_java_marker_does_not_cross_an_unrelated_build_root(): + build_path = "services/unrelated/pom.xml" + marker_path = "services/catalog/src/main/java/example/Main.java" + capabilities = _selector().select(RepositoryFacts( + revision="0123456789abcdef", + paths=tuple(sorted((build_path, marker_path))), + marker_contents={ + build_path: "", + marker_path: "import io.quarkus.runtime.Quarkus;", + }, + )) + + assert capabilities.repository_plugins == ("java",) + + +def test_indexes_exact_quarkus_java_relationships(): + source = '''package example; +import io.quarkus.hibernate.orm.panache.PanacheEntity; +import io.quarkus.hibernate.orm.panache.PanacheRepository; +import io.quarkus.scheduler.Scheduled; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.eclipse.microprofile.reactive.messaging.Incoming; +import org.eclipse.microprofile.reactive.messaging.Outgoing; + +@ApplicationScoped +@Path("/items") +public class ItemResource extends PanacheEntity { + @Inject ItemService service; + @ConfigProperty(name = "item.limit", defaultValue = "10") int limit; + + @Inject + ItemResource(ItemRepository repository) {} + + @GET + @Path("/{id}") + String get() { return "ok"; } + + @Scheduled(every = "10s", delayed = "1s") + void refresh() {} + + @Incoming("prices") + @Outgoing("quotes") + String relay(String value) { return value; } +} + +interface ItemRepository extends PanacheRepository {} +''' + outcome = _plugin().index_file(FileArtifact(JAVA_PATH, source)) + + assert outcome.status is OutcomeStatus.HANDLED + facts = outcome.value + assert facts == tuple(sorted(facts)) + assert { + (fact.kind, fact.source, fact.relation, fact.target) + for fact in facts + } >= { + ( + "quarkus-cdi-bean", + "example.ItemResource", + "scoped-as", + "ApplicationScoped", + ), + ( + "quarkus-cdi-injection", + "example.ItemResource", + "depends-on", + "ItemService", + ), + ( + "quarkus-cdi-injection", + "example.ItemResource", + "depends-on", + "ItemRepository", + ), + ( + "quarkus-config-property", + "example.ItemResource#limit", + "reads", + "item.limit", + ), + ( + "quarkus-jaxrs-resource", + "example.ItemResource", + "serves", + "/items", + ), + ( + "quarkus-jaxrs-route", + "example.ItemResource#get", + "handles", + "GET /items/{id}", + ), + ( + "quarkus-panache-entity", + "example.ItemResource", + "extends", + "io.quarkus.hibernate.orm.panache.PanacheEntity", + ), + ( + "quarkus-panache-repository", + "example.ItemRepository", + "manages", + "ItemResource", + ), + ( + "quarkus-reactive-channel", + "example.ItemResource#relay", + "consumes", + "prices", + ), + ( + "quarkus-reactive-channel", + "example.ItemResource#relay", + "produces", + "quotes", + ), + ( + "quarkus-scheduled-method", + "example.ItemResource#refresh", + "runs-on", + "every=10s", + ), + } + scheduled = next( + fact for fact in facts if fact.kind == "quarkus-scheduled-method" + ) + assert scheduled.attributes == (("delayed", "1s"), ("every", "10s")) + + +def test_annotation_short_names_require_relevant_imports(): + source = '''package example; +@ApplicationScoped +@Path("/not-quarkus") +class CustomType { + @Scheduled(every = "1s") void run() {} +} +''' + + assert _plugin().index_file(FileArtifact(JAVA_PATH, source)).status is ( + OutcomeStatus.ABSTAINED + ) + + +@pytest.mark.parametrize( + "source", + ( + '''package example; +import jakarta.inject.Inject; +class Plain { + @interface Inject {} + @Inject ItemService service; +} +''', + '''package example; +import jakarta.ws.rs.*; +@Path("/items") +class ItemResource { + @GET String get() { return "ok"; } +} +''', + ), +) +def test_annotations_abstain_for_local_shadowing_and_wildcard_imports(source: str): + outcome = _plugin().index_file(FileArtifact(JAVA_PATH, source)) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_malformed_java_returns_a_recoverable_diagnostic_without_partial_facts(): + source = '''package example; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +@Path("/items") class ItemResource { + @GET String get( { +} +''' + + outcome = _plugin().index_file(FileArtifact(JAVA_PATH, source)) + + assert outcome.status is OutcomeStatus.FAILED + assert outcome.value is None + assert outcome.diagnostic.code == "quarkus-java-parse-incomplete" + assert outcome.diagnostic.recoverable is True + assert outcome.diagnostic.path == JAVA_PATH + + +def test_indexes_only_safe_bounded_application_property_keys(): + path = "services/catalog/src/main/resources/application.properties" + content = "\n".join(( + "# comments are ignored", + "quarkus.http.port=8080", + "%test.quarkus.datasource.db-kind: postgresql", + "greeting.message = hello", + "continued.value=secret\\", + " continuation-that-is-not-a-key", + "unsupported whitespace key = ignored", + "", + )) + + outcome = _plugin().index_file(FileArtifact(path, content)) + + assert outcome.status is OutcomeStatus.HANDLED + assert {(fact.target, fact.line) for fact in outcome.value} == { + ("%test.quarkus.datasource.db-kind", 3), + ("greeting.message", 4), + ("quarkus.http.port", 2), + } + assert all("postgresql" not in repr(fact) for fact in outcome.value) + profiled = next(fact for fact in outcome.value if fact.target.startswith("%")) + assert profiled.attributes == (("profile", "test"),) + + bounded = _plugin().index_file(FileArtifact( + path, + "\n".join(f"key.{index}=value" for index in range(140)), + )) + assert len(bounded.value) == 128 + + +def test_review_requests_exact_facts_and_validation_never_promotes_topology(): + plugin = _plugin() + source = '''package example; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +@Path("/items") class ItemResource { + @GET @Path("/{id}") String get() { return "ok"; } +} +''' + route = next( + fact + for fact in plugin.index_file(FileArtifact(JAVA_PATH, source)).value + if fact.kind == "quarkus-jaxrs-route" + ) + review = plugin.review(( + JAVA_PATH, + "services/catalog/src/main/resources/application.properties", + "README.md", + )) + + assert review.status is OutcomeStatus.HANDLED + assert tuple( + request.identifier for request in review.value.evidence_requests + ) == ( + JAVA_PATH, + "services/catalog/src/main/resources/application.properties", + ) + + contradicted = plugin.validate(CandidateClaim( + category="framework-risk", + path=JAVA_PATH, + line=5, + message="The GET /items/{id} route is missing.", + evidence=(route,), + claim_kind="quarkus-jaxrs-route", + )) + topology_only = plugin.validate(CandidateClaim( + category="framework-risk", + path=JAVA_PATH, + line=5, + message="GET /items/{id} lacks an authorization check.", + evidence=(route,), + claim_kind="quarkus-jaxrs-route", + )) + unrelated_absence = plugin.validate(CandidateClaim( + category="framework-risk", + path=JAVA_PATH, + line=5, + message=( + "Authorization is missing from GET /items/{id}, so the route " + "may expose data." + ), + evidence=(route,), + claim_kind="quarkus-jaxrs-route", + )) + wrong_identifier = plugin.validate(CandidateClaim( + category="framework-risk", + path=JAVA_PATH, + line=5, + message="The GET /other route is missing.", + evidence=(route,), + claim_kind="quarkus-jaxrs-route", + )) + absence_for_other_route = plugin.validate(CandidateClaim( + category="framework-risk", + path=JAVA_PATH, + line=5, + message="No route exists for /other, while GET /items/{id} is slow.", + evidence=(route,), + claim_kind="quarkus-jaxrs-route", + )) + unknown_kind = plugin.validate(CandidateClaim( + category="quarkus-cache", + path=JAVA_PATH, + line=5, + message="The GET /items/{id} route is missing.", + evidence=(route,), + claim_kind="", + )) + + assert contradicted.value.decision is ValidationDecision.REJECT + assert topology_only.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert topology_only.value.code == "quarkus-topology-not-defect-proof" + assert unrelated_absence.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert unrelated_absence.value.code == "quarkus-topology-not-defect-proof" + assert wrong_identifier.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert wrong_identifier.value.code == "quarkus-cited-identifier-mismatch" + assert absence_for_other_route.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert absence_for_other_route.value.code == "quarkus-topology-not-defect-proof" + assert unknown_kind.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert unknown_kind.value.code == "quarkus-unknown-fact-kind" diff --git a/analysis-plugins/contracts/python/tests/test_rails_framework_plugin.py b/analysis-plugins/contracts/python/tests/test_rails_framework_plugin.py new file mode 100644 index 00000000..18449f1a --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_rails_framework_plugin.py @@ -0,0 +1,397 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +from codecrow_plugins import ( + CandidateClaim, + FileArtifact, + OutcomeStatus, + PluginRegistry, + ProjectSelector, + RepositoryFacts, + ValidationDecision, + load_descriptor, +) + + +PLUGINS_ROOT = Path(__file__).resolve().parents[3] +RAILS_ROOT = PLUGINS_ROOT / "frameworks" / "rails" +sys.path.insert(0, str(RAILS_ROOT / "python")) + +from codecrow_plugin_rails import create_plugin # noqa: E402 + + +def _plugin(): + return create_plugin(load_descriptor(RAILS_ROOT / "plugin.json")) + + +def _facts(path: str, content: str): + outcome = _plugin().index_file(FileArtifact(path, content)) + assert outcome.status is OutcomeStatus.HANDLED + return outcome.value + + +def test_rails_detection_requires_one_coherent_project_root(): + registry = PluginRegistry(( + load_descriptor(PLUGINS_ROOT / "languages" / "ruby" / "plugin.json"), + load_descriptor(RAILS_ROOT / "plugin.json"), + )) + selector = ProjectSelector(registry) + marker = 'source "https://rubygems.org"\ngem "rails"\n' + + split = selector.select(RepositoryFacts( + revision="0123456789abcdef", + paths=("backend/config/routes.rb", "frontend/Gemfile"), + marker_contents={"frontend/Gemfile": marker}, + )) + coherent = selector.select(RepositoryFacts( + revision="0123456789abcdef", + paths=("services/shop/Gemfile", "services/shop/config/routes.rb"), + marker_contents={"services/shop/Gemfile": marker}, + )) + + assert split.repository_plugins == ("ruby",) + assert coherent.repository_plugins == ("ruby", "rails") + assert "root:services/shop" in coherent.detection_evidence["rails"] + + +def test_rails_engine_detection_does_not_combine_pattern_evidence_across_roots(): + registry = PluginRegistry(( + load_descriptor(PLUGINS_ROOT / "languages" / "ruby" / "plugin.json"), + load_descriptor(RAILS_ROOT / "plugin.json"), + )) + selector = ProjectSelector(registry) + split = selector.select(RepositoryFacts( + revision="0123456789abcdef", + paths=( + "a/example.gemspec", + "b/config/routes.rb", + "c/lib/example/engine.rb", + ), + marker_contents={ + "c/lib/example/engine.rb": "class Example < Rails::Engine\nend\n", + }, + )) + coherent = selector.select(RepositoryFacts( + revision="0123456789abcdef", + paths=( + "services/blog/blog.gemspec", + "services/blog/config/routes.rb", + "services/blog/lib/blog/engine.rb", + ), + marker_contents={ + "services/blog/lib/blog/engine.rb": "class Blog < Rails::Engine\nend\n", + }, + )) + nested_but_not_root_relative = selector.select(RepositoryFacts( + revision="0123456789abcdef", + paths=( + "services/blog/config/routes.rb", + "services/blog/nested/blog.gemspec", + "services/blog/vendor/lib/blog/engine.rb", + ), + marker_contents={ + "services/blog/vendor/lib/blog/engine.rb": "class Blog < Rails::Engine\nend\n", + }, + )) + + assert split.repository_plugins == ("ruby",) + assert coherent.repository_plugins == ("ruby", "rails") + assert "root:services/blog" in coherent.detection_evidence["rails"] + assert nested_but_not_root_relative.repository_plugins == ("ruby",) + + +def test_rails_indexes_routes_controllers_models_callbacks_associations_and_jobs(): + routes = _facts("config/routes.rb", ''' +Rails.application.routes.draw do + namespace :admin do + resources :users, only: [:index, :show] + get "health", to: "health#show" + root "dashboard#index" + end + root "home#index" +end +''') + controller = _facts("app/controllers/admin/users_controller.rb", ''' +module Admin + class UsersController < ApplicationController + before_action :authenticate!, only: [:show] + def index; end + def show; end + private + def helper; end + end +end +''') + model = _facts("app/models/order.rb", ''' +class Order < ApplicationRecord + belongs_to :customer, optional: true + has_many :items, class_name: "LineItem", dependent: :destroy + before_save :normalize_total + after_commit :publish, on: :create +end +''') + job = _facts("app/jobs/import_job.rb", ''' +class ImportJob < ApplicationJob + queue_as :low + retry_on Timeout::Error, attempts: 3 + def perform(account_id); end +end +''') + + all_facts = (*routes, *controller, *model, *job) + triples = {(fact.kind, fact.relation, fact.target) for fact in all_facts} + assert ("rails-route", "declares", "RESOURCES /admin/users") in triples + assert ("rails-route", "handles", "GET /admin/health") in triples + assert ("rails-route", "handles", "GET /admin") in triples + assert ("rails-route", "handles", "GET /") in triples + assert ("rails-controller", "declares", "Admin::UsersController") in triples + assert ("rails-controller-action", "exposes", "Admin::UsersController#index") in triples + assert ("rails-controller-action", "exposes", "Admin::UsersController#show") in triples + assert ("rails-controller-action", "exposes", "Admin::UsersController#helper") not in triples + assert ("rails-model", "declares", "Order") in triples + assert ("rails-association", "belongs-to", "customer") in triples + assert ("rails-association", "has-many", "items") in triples + assert ("rails-callback", "registers", "normalize_total") in triples + assert ("rails-callback", "registers", "publish") in triples + assert ("rails-job", "declares", "ImportJob") in triples + assert ("rails-job-queue", "queues-on", "low") in triples + assert ("rails-job-policy", "retry-on", "Timeout::Error") in triples + assert ("rails-job-perform", "executes", "ImportJob#perform") in triples + + +def test_rails_abstains_for_unrelated_ruby_classes_even_in_a_selected_project(): + outcome = _plugin().index_file(FileArtifact( + "app/models/value.rb", + "class Value < DataRecord\nend\n", + )) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_rails_abstains_for_routes_under_unresolved_resource_prefixes(): + facts = _facts("config/routes.rb", ''' +Rails.application.routes.draw do + resources :users do + get :profile, on: :member + resources :posts + end +end +''') + + targets = {fact.target for fact in facts if fact.kind == "rails-route"} + assert "RESOURCES /users" in targets + assert not any("profile" in target or "posts" in target for target in targets) + + +def test_rails_requires_route_dsl_to_be_inside_a_routes_draw_block(): + outcome = _plugin().index_file(FileArtifact("config/routes.rb", ''' +get "outside", to: "outside#show" +draw do + get "plain-draw", to: "plain#show" +end +routes.draw do + get "receiver-without-owner", to: "plain#show" +end +client.routes.draw do + get "dynamic-owner", to: "plain#show" +end +Client.routes.draw do + get "unproven-constant-owner", to: "plain#show" +end +''')) + + assert outcome.status is OutcomeStatus.ABSTAINED + + +def test_rails_accepts_canonical_and_static_constant_route_set_owners(): + facts = _facts("config/routes.rb", ''' +Rails.application.routes.draw do + get "canonical", to: "health#show" +end +Shop::Engine.routes.draw do + get "engine", to: "shop#show" +end +Legacy::Application.routes.draw do + get "application", to: "legacy#show" +end +''') + + targets = {fact.target for fact in facts if fact.kind == "rails-route"} + assert targets == {"GET /application", "GET /canonical", "GET /engine"} + + +def test_rails_ignores_receiver_qualified_route_and_model_lookalikes(): + routes = _facts("config/routes.rb", ''' +Rails.application.routes.draw do + client.get "fake", to: "fake#show" + client.resources :fake_records + self.get "health", to: "health#show" +end +''') + model = _facts("app/models/order.rb", ''' +class Order < ApplicationRecord + client.has_many :fake_items + client.before_save :fake_callback + self.has_many :items + self.before_save :normalize_total +end +''') + + triples = { + (fact.kind, fact.relation, fact.target) + for fact in (*routes, *model) + } + assert ("rails-route", "handles", "GET /health") in triples + assert ("rails-association", "has-many", "items") in triples + assert ("rails-callback", "registers", "normalize_total") in triples + assert not any("fake" in target for _, _, target in triples) + + +def test_rails_uses_only_static_resource_path_overrides(): + facts = _facts("config/routes.rb", ''' +Rails.application.routes.draw do + resources :photos, path: "images" + resources :reports, path: dynamic_segment +end +''') + + targets = {fact.target for fact in facts if fact.kind == "rails-route"} + assert "RESOURCES /images" in targets + assert not any("photos" in target or "reports" in target for target in targets) + + +def test_rails_skips_symbolic_route_paths_prefixes_and_verbs(): + facts = _facts("config/routes.rb", ''' +Rails.application.routes.draw do + path_value = "dynamic" + get path_value, to: "dynamic#show" + namespace path_value do + get "nested", to: "dynamic#nested" + end + scope path: path_value do + get "scoped", to: "dynamic#scoped" + end + match "unknown-verb", via: METHODS, to: "dynamic#match" + get "health", to: "health#show" +end +''') + + targets = {fact.target for fact in facts if fact.kind == "rails-route"} + assert targets == {"GET /health"} + + +def test_rails_symbolic_visibility_does_not_expose_hidden_controller_actions(): + facts = _facts("app/controllers/users_controller.rb", ''' +class UsersController < ApplicationController + def index; end + def secret; end + private :secret +end +''') + + actions = {fact.target for fact in facts if fact.kind == "rails-controller-action"} + assert "UsersController#index" in actions + assert "UsersController#secret" not in actions + + +def test_rails_validation_rejects_only_a_relevant_contradicted_absence(): + association = next( + fact for fact in _facts("app/models/order.rb", ''' +class Order < ApplicationRecord + has_many :items +end +''') + if fact.kind == "rails-association" + ) + rejected = _plugin().validate(CandidateClaim( + category="rails-association", + claim_kind="rails-association", + path="app/models/order.rb", + line=2, + message="Order has no association named items.", + evidence=(association,), + )) + contextual = _plugin().validate(CandidateClaim( + category="rails-association", + claim_kind="rails-association", + path="app/models/order.rb", + line=2, + message="The items association may load too much data.", + evidence=(association,), + )) + + assert rejected.value.decision is ValidationDecision.REJECT + assert rejected.value.code == "rails-absence-contradicted" + assert contextual.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert contextual.value.code == "rails-topology-not-defect-proof" + + +def test_rails_validation_does_not_match_get_inside_an_unrelated_word(): + route = next( + fact for fact in _facts("config/routes.rb", ''' +Rails.application.routes.draw do + get "health", to: "health#show" +end +''') + if fact.kind == "rails-route" + ) + result = _plugin().validate(CandidateClaim( + category="rails-route", + claim_kind="rails-route", + path="config/routes.rb", + line=2, + message="The target widget route is missing.", + evidence=(route,), + )) + + assert result.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert result.value.code == "rails-cited-identifier-mismatch" + + +def test_rails_validation_does_not_bind_an_unrelated_absence_to_a_model(): + model_fact = next( + fact for fact in _facts("app/models/order.rb", ''' +class Order < ApplicationRecord +end +''') + if fact.kind == "rails-model" + ) + unrelated = _plugin().validate(CandidateClaim( + category="rails-model", + claim_kind="rails-model", + path="app/models/order.rb", + line=1, + message="Order fails closed when its cache entry is missing.", + evidence=(model_fact,), + )) + unknown = _plugin().validate(CandidateClaim( + category="rails-service", + claim_kind="rails-service", + path="app/models/order.rb", + line=1, + message="The Order service is missing.", + evidence=(model_fact,), + )) + + assert unrelated.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert unrelated.value.code == "rails-topology-not-defect-proof" + assert unknown.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE + assert unknown.value.code == "rails-unknown-fact-kind" + + +def test_rails_direct_indexing_and_review_contributions_are_bounded(): + routes = "\n".join( + f' get "item-{index}", to: "items#show"' + for index in range(300) + ) + facts = _facts( + "config/routes.rb", + "Rails.application.routes.draw do\n" + routes + "\nend\n", + ) + review = _plugin().review(tuple(f"app/models/model_{index}.rb" for index in range(100))) + + assert len(facts) == 160 + assert {fact.kind for fact in facts} == {"rails-route"} + assert len(review.value.evidence_requests) == 40 diff --git a/analysis-plugins/contracts/python/tests/test_repository_facts.py b/analysis-plugins/contracts/python/tests/test_repository_facts.py index 37e197fb..1e76f853 100644 --- a/analysis-plugins/contracts/python/tests/test_repository_facts.py +++ b/analysis-plugins/contracts/python/tests/test_repository_facts.py @@ -49,6 +49,117 @@ def test_overlay_recomputes_path_and_content_detection_from_complete_inventory( assert "python" in selector.select(updated).repository_plugins +def test_overlay_promotes_persisted_pattern_evidence_when_first_match_is_deleted( + tmp_path, +): + catalog = discover_builtin_plugins() + selector = ProjectSelector(catalog.registry) + paths = ("build.gradle", "src/First.java", "src/Second.java") + _write(tmp_path, "build.gradle", "plugins { id 'java' }\n") + _write(tmp_path, "src/First.java", "import io.quarkus.runtime.Startup;\n") + _write(tmp_path, "src/Second.java", "import io.quarkus.scheduler.Scheduled;\n") + + baseline = build_repository_facts( + tmp_path, + "base", + paths, + catalog.registry, + ) + + assert set(baseline.marker_contents) == { + "src/First.java", + "src/Second.java", + } + assert "quarkus" in selector.select(baseline).repository_plugins + + updated = overlay_repository_facts( + baseline, + None, + "changed", + (), + ("src/First.java",), + catalog.registry, + ) + + assert set(updated.marker_contents) == {"src/Second.java"} + assert "quarkus" in selector.select(updated).repository_plugins + + +def test_nested_framework_pattern_marker_is_acquired_relative_to_its_root( + tmp_path, +): + catalog = discover_builtin_plugins() + selector = ProjectSelector(catalog.registry) + root = "services/blog" + paths = ( + f"{root}/blog.gemspec", + f"{root}/config/routes.rb", + f"{root}/lib/blog/engine.rb", + ) + _write(tmp_path, f"{root}/blog.gemspec", "Gem::Specification.new\n") + _write(tmp_path, f"{root}/config/routes.rb", "Blog::Engine.routes.draw do\nend\n") + _write( + tmp_path, + f"{root}/lib/blog/engine.rb", + "module Blog\n class Engine < Rails::Engine\n end\nend\n", + ) + + facts = build_repository_facts( + tmp_path, + "base", + paths, + catalog.registry, + ) + + assert facts.marker_contents == { + f"{root}/lib/blog/engine.rb": ( + "module Blog\n class Engine < Rails::Engine\n end\nend\n" + ), + } + selected = selector.select(facts) + assert "rails" in selected.repository_plugins + assert f"root:{root}" in selected.detection_evidence["rails"] + + +def test_overlay_activates_framework_when_new_anchors_make_persisted_pattern_relevant( + tmp_path, +): + catalog = discover_builtin_plugins() + selector = ProjectSelector(catalog.registry) + root = "services/blog" + engine = f"{root}/lib/blog/engine.rb" + _write( + tmp_path, + engine, + "module Blog\n class Engine < Rails::Engine\n end\nend\n", + ) + baseline = build_repository_facts( + tmp_path, + "base", + (engine,), + catalog.registry, + ) + assert baseline.marker_contents.keys() == {engine} + assert "rails" not in selector.select(baseline).repository_plugins + + gemspec = f"{root}/blog.gemspec" + routes = f"{root}/config/routes.rb" + _write(tmp_path, gemspec, "Gem::Specification.new\n") + _write(tmp_path, routes, "Blog::Engine.routes.draw do\nend\n") + updated = overlay_repository_facts( + baseline, + tmp_path, + "changed", + (gemspec, routes), + (), + catalog.registry, + ) + + selected = selector.select(updated) + assert "rails" in selected.repository_plugins + assert f"root:{root}" in selected.detection_evidence["rails"] + + def test_overlay_adds_exact_framework_marker_and_removes_deleted_paths(tmp_path): catalog = discover_builtin_plugins() selector = ProjectSelector(catalog.registry) @@ -208,6 +319,141 @@ def test_marker_byte_budget_degrades_detection_without_failing_index(tmp_path, c assert "reduced automatic plugin-detection evidence" in caplog.text +def test_pattern_marker_scan_budgets_non_matching_files_before_reading( + tmp_path, + caplog, + monkeypatch, +): + catalog = discover_builtin_plugins() + paths = tuple(f"src/Type{index}.java" for index in range(3)) + for path in paths: + _write(tmp_path, path, "final class Type {}\n") + + original_read_text = Path.read_text + reads = [] + + def counted_read_text(path, *args, **kwargs): + reads.append(path) + return original_read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", counted_read_text) + facts = build_repository_facts( + tmp_path, + "base", + paths, + catalog.registry, + max_marker_bytes=1_024, + max_marker_files=1, + ) + + assert facts.marker_contents == {} + assert len(reads) == 1 + assert "file inspection budget" in caplog.text + + +def test_incremental_pattern_marker_scan_preserves_last_evidence_when_budget_is_exhausted( + tmp_path, + caplog, + monkeypatch, +): + catalog = discover_builtin_plugins() + path = "src/Changed.java" + _write(tmp_path, "build.gradle", "plugins { id 'java' }\n") + _write(tmp_path, path, "final class Changed {}\n") + baseline = RepositoryFacts( + revision="base", + paths=("build.gradle", path), + marker_contents={path: "import io.quarkus.runtime.Startup;\n"}, + ) + + monkeypatch.setattr( + Path, + "read_text", + lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("over-budget updated marker must not be read") + ), + ) + updated = overlay_repository_facts( + baseline, + tmp_path, + "changed", + (path,), + (), + catalog.registry, + max_marker_bytes=1, + ) + + assert updated.marker_contents == { + path: "import io.quarkus.runtime.Startup;\n", + } + assert "quarkus" in ProjectSelector(catalog.registry).select( + updated + ).repository_plugins + assert "byte inspection budget" in caplog.text + + +@pytest.mark.parametrize("unsafe_kind", ("outside-symlink", "invalid-utf8")) +def test_optional_marker_read_failure_degrades_without_failing_index( + tmp_path, + caplog, + unsafe_kind, +): + catalog = discover_builtin_plugins() + selector = ProjectSelector(catalog.registry) + marker = tmp_path / "package.json" + if unsafe_kind == "outside-symlink": + outside = tmp_path.parent / f"{tmp_path.name}-outside-package.json" + outside.write_text('{"dependencies":{"express":"*"}}', encoding="utf-8") + marker.symlink_to(outside) + else: + marker.write_bytes(b"\xff\xfe") + _write(tmp_path, "src/app.js", "export const app = true;\n") + + facts = build_repository_facts( + tmp_path, + "base", + ("package.json", "src/app.js"), + catalog.registry, + ) + + assert facts.marker_contents == {} + assert "express" not in selector.select(facts).repository_plugins + assert "reduced automatic plugin-detection evidence" in caplog.text + + +def test_incremental_unreadable_marker_preserves_last_reliable_evidence( + tmp_path, + caplog, +): + catalog = discover_builtin_plugins() + marker = tmp_path / "package.json" + marker.write_bytes(b"\xff\xfe") + baseline = RepositoryFacts( + revision="base", + paths=("package.json", "src/app.js"), + marker_contents={ + "package.json": '{"dependencies":{"express":"*"}}', + }, + ) + + updated = overlay_repository_facts( + baseline, + tmp_path, + "changed", + ("package.json",), + (), + catalog.registry, + ) + + assert updated.marker_contents == { + "package.json": '{"dependencies":{"express":"*"}}', + } + assert "express" in ProjectSelector(catalog.registry).select( + updated + ).repository_plugins + assert "reduced automatic plugin-detection evidence" in caplog.text + + def test_automatic_marker_reads_stay_within_configured_source_root(tmp_path): catalog = discover_builtin_plugins() paths = ( diff --git a/analysis-plugins/contracts/python/tests/test_runtime_graph_fact_limits.py b/analysis-plugins/contracts/python/tests/test_runtime_graph_fact_limits.py new file mode 100644 index 00000000..5076658f --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_runtime_graph_fact_limits.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace + +from codecrow_plugins import ( + Capability, + DetectionRules, + FileArtifact, + GraphFact, + PluginDescriptor, + PluginKind, + PluginOutcome, + PluginRuntime, + ProjectCapabilities, +) + + +class _FactPlugin: + def __init__(self, facts: tuple[GraphFact, ...]): + self.facts = facts + + def index_file(self, _artifact: FileArtifact): + return PluginOutcome.handled(self.facts) + + +def _runtime( + contributions: dict[str, tuple[GraphFact, ...]], +) -> tuple[PluginRuntime, ProjectCapabilities]: + descriptors = { + plugin_id: PluginDescriptor( + id=plugin_id, + kind=PluginKind.DOMAIN, + requires=(), + capabilities=(Capability.GRAPH,), + detection=DetectionRules(), + ) + for plugin_id in contributions + } + implementations = { + plugin_id: _FactPlugin(facts) + for plugin_id, facts in contributions.items() + } + catalog = SimpleNamespace( + registry=SimpleNamespace( + descriptor=lambda plugin_id: descriptors[plugin_id], + ), + implementation=lambda plugin_id: implementations[plugin_id], + ) + capabilities = ProjectCapabilities( + repository_plugins=tuple(contributions), + fingerprint="sha256:" + "0" * 64, + ) + return PluginRuntime(catalog), capabilities + + +def _fact( + kind: str, + source: str, + *, + relation: str = "declares", + target: str = "target", + path: str = "src/example.py", + attributes: tuple[tuple[str, str], ...] = (), + related_paths: tuple[str, ...] = (), +) -> GraphFact: + return GraphFact( + kind=kind, + source=source, + relation=relation, + target=target, + path=path, + attributes=attributes, + related_paths=related_paths, + ) + + +def _serialized_facts_bytes(facts: tuple[GraphFact, ...]) -> int: + return len(json.dumps( + [dict(fact.as_metadata()) for fact in facts], + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8")) + + +def test_graph_facts_reject_every_overlong_string_location_without_truncating(): + limit = PluginRuntime.MAX_GRAPH_FACT_STRING_LENGTH + overlong = "x" * (limit + 1) + valid = _fact("valid", "kept") + invalid = ( + _fact(overlong, "kind"), + _fact("source", overlong), + _fact("relation", "source", relation=overlong), + _fact("target", "source", target=overlong), + _fact("path", "source", path=overlong), + _fact("attribute-key", "source", attributes=((overlong, "value"),)), + _fact("attribute-value", "source", attributes=(("key", overlong),)), + _fact("related-path", "source", related_paths=(overlong,)), + ) + runtime, capabilities = _runtime({"bounded": (valid, *invalid)}) + + facts, diagnostics = runtime.graph_facts( + FileArtifact("src/example.py", "pass"), + capabilities, + ) + + assert facts == (valid,) + assert len(diagnostics) == 1 + assert diagnostics[0].code == "plugin-index-output-limit" + assert diagnostics[0].plugin_id == "bounded" + assert diagnostics[0].path == "src/example.py" + assert diagnostics[0].recoverable is True + assert "8 fact(s)" in diagnostics[0].message + assert str(limit) in diagnostics[0].message + + +def test_graph_fact_byte_budget_is_global_deterministic_and_per_artifact(): + alpha = _fact("alpha", "alpha", target="α" * 32) + beta = _fact("beta", "beta", target="β" * 32) + runtime, capabilities = _runtime({ + "first": (alpha,), + "second": (beta, beta), + }) + runtime.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT = _serialized_facts_bytes((alpha,)) + + first_facts, first_diagnostics = runtime.graph_facts( + FileArtifact("src/first.py", "pass"), + capabilities, + ) + second_facts, second_diagnostics = runtime.graph_facts( + FileArtifact("src/second.py", "pass"), + capabilities, + ) + + assert first_facts == second_facts == (alpha,) + assert _serialized_facts_bytes(first_facts) <= ( + runtime.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT + ) + assert [diagnostic.plugin_id for diagnostic in first_diagnostics] == ["second"] + assert [diagnostic.path for diagnostic in first_diagnostics] == ["src/first.py"] + assert [diagnostic.path for diagnostic in second_diagnostics] == ["src/second.py"] + assert all(diagnostic.recoverable for diagnostic in first_diagnostics) + + +def test_graph_fact_byte_budget_preserves_balanced_kind_selection(): + facts = ( + _fact("kind-a", "a-1", target="x" * 32), + _fact("kind-a", "a-0", target="x" * 32), + _fact("kind-b", "b-1", target="x" * 32), + _fact("kind-b", "b-0", target="x" * 32), + ) + expected = tuple(sorted((facts[1], facts[3]))) + runtime, capabilities = _runtime({"balanced": tuple(reversed(facts))}) + runtime.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT = _serialized_facts_bytes(expected) + + selected, diagnostics = runtime.graph_facts( + FileArtifact("src/example.py", "pass"), + capabilities, + ) + + assert selected == expected + assert {fact.kind for fact in selected} == {"kind-a", "kind-b"} + assert len(diagnostics) == 1 + assert diagnostics[0].plugin_id == "balanced" + assert "2 fact(s)" in diagnostics[0].message diff --git a/analysis-plugins/frameworks/django/java/pom.xml b/analysis-plugins/frameworks/django/java/pom.xml new file mode 100644 index 00000000..1e29b35f --- /dev/null +++ b/analysis-plugins/frameworks/django/java/pom.xml @@ -0,0 +1,15 @@ + + + 4.0.0 + + org.rostilos.codecrow + codecrow-framework-plugins + 1.0 + ../../pom.xml + + codecrow-plugin-django + jar + django + diff --git a/analysis-plugins/frameworks/django/java/src/main/java/org/rostilos/codecrow/plugins/django/DjangoPlugin.java b/analysis-plugins/frameworks/django/java/src/main/java/org/rostilos/codecrow/plugins/django/DjangoPlugin.java new file mode 100644 index 00000000..0a6561a4 --- /dev/null +++ b/analysis-plugins/frameworks/django/java/src/main/java/org/rostilos/codecrow/plugins/django/DjangoPlugin.java @@ -0,0 +1,23 @@ +package org.rostilos.codecrow.plugins.django; + +import org.rostilos.codecrow.plugins.CodeCrowPlugin; +import org.rostilos.codecrow.plugins.PluginDescriptor; +import org.rostilos.codecrow.plugins.PluginManifestLoader; + +public final class DjangoPlugin implements CodeCrowPlugin { + private final PluginDescriptor descriptor; + + public DjangoPlugin() { + try (var input = DjangoPlugin.class.getResourceAsStream( + "/META-INF/codecrow/plugins/django/plugin.json")) { + descriptor = new PluginManifestLoader().loadDescriptor(input); + } catch (Exception exception) { + throw new IllegalStateException("cannot load Django plugin descriptor", exception); + } + } + + @Override + public PluginDescriptor descriptor() { + return descriptor; + } +} diff --git a/analysis-plugins/frameworks/django/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin b/analysis-plugins/frameworks/django/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin new file mode 100644 index 00000000..87bb725d --- /dev/null +++ b/analysis-plugins/frameworks/django/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin @@ -0,0 +1 @@ +org.rostilos.codecrow.plugins.django.DjangoPlugin diff --git a/analysis-plugins/frameworks/django/java/src/test/java/org/rostilos/codecrow/plugins/django/DjangoPluginTest.java b/analysis-plugins/frameworks/django/java/src/test/java/org/rostilos/codecrow/plugins/django/DjangoPluginTest.java new file mode 100644 index 00000000..e8d5a871 --- /dev/null +++ b/analysis-plugins/frameworks/django/java/src/test/java/org/rostilos/codecrow/plugins/django/DjangoPluginTest.java @@ -0,0 +1,20 @@ +package org.rostilos.codecrow.plugins.django; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.plugins.PluginCapability; +import org.rostilos.codecrow.plugins.PluginKind; + +import static org.assertj.core.api.Assertions.assertThat; + +class DjangoPluginTest { + @Test + void packagedManifestLoadsThroughTheJavaContract() { + var descriptor = new DjangoPlugin().descriptor(); + assertThat(descriptor.id()).isEqualTo("django"); + assertThat(descriptor.kind()).isEqualTo(PluginKind.FRAMEWORK); + assertThat(descriptor.requires()).containsExactly("python"); + assertThat(descriptor.capabilities()).contains( + PluginCapability.GRAPH, PluginCapability.INDEX, PluginCapability.VALIDATION); + assertThat(descriptor.detection().alternatives()).hasSize(4); + } +} diff --git a/analysis-plugins/frameworks/django/plugin.json b/analysis-plugins/frameworks/django/plugin.json new file mode 100644 index 00000000..1b76f3b7 --- /dev/null +++ b/analysis-plugins/frameworks/django/plugin.json @@ -0,0 +1,52 @@ +{ + "id": "django", + "kind": "framework", + "requires": ["python"], + "capabilities": ["context", "graph", "index", "planning", "prompt", "validation"], + "detection": { + "extensions": [], + "filesAll": [], + "filesAny": [], + "contentMarkers": [], + "alternatives": [ + { + "filesAll": ["manage.py"], + "filesAny": [], + "pathPatternsAll": ["**/settings.py", "**/urls.py"], + "pathPatternsAny": [], + "contentMarkers": [] + }, + { + "filesAll": ["pyproject.toml"], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": ["**/apps.py", "**/settings.py"], + "contentMarkers": [ + {"path": "pyproject.toml", "contains": "jango"} + ] + }, + { + "filesAll": ["requirements.txt"], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": ["**/apps.py", "**/settings.py"], + "contentMarkers": [ + {"path": "requirements.txt", "contains": "jango"} + ] + }, + { + "filesAll": ["setup.cfg"], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": ["**/apps.py", "**/settings.py"], + "contentMarkers": [ + {"path": "setup.cfg", "contains": "jango"} + ] + } + ] + }, + "entrypoints": { + "java": "org.rostilos.codecrow.plugins.django.DjangoPlugin", + "python": "codecrow_plugin_django:create_plugin" + } +} diff --git a/analysis-plugins/frameworks/django/python/codecrow_plugin_django/__init__.py b/analysis-plugins/frameworks/django/python/codecrow_plugin_django/__init__.py new file mode 100644 index 00000000..aff7c0bd --- /dev/null +++ b/analysis-plugins/frameworks/django/python/codecrow_plugin_django/__init__.py @@ -0,0 +1,973 @@ +from __future__ import annotations + +import ast +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from codecrow_plugins import ( + CandidateClaim, + EvidenceRequest, + FileArtifact, + GraphFact, + PluginDescriptor, + PluginDiagnostic, + PluginOutcome, + ReviewContribution, + ValidationDecision, + ValidationResult, +) + + +_EXTENSIONS = (".py", ".pyi", ".pyw") +_MAX_FACTS_PER_FILE = 160 +_MAX_EVIDENCE_REQUESTS = 40 +_MODEL_RELATIONS = { + "ForeignKey": "many-to-one", + "ManyToManyField": "many-to-many", + "OneToOneField": "one-to-one", +} +_VIEW_METHODS = frozenset({ + "delete", "get", "head", "options", "patch", "post", "put", "trace", +}) +_FACT_KINDS = frozenset({ + "django-app-config", + "django-installed-app", + "django-middleware", + "django-middleware-component", + "django-model", + "django-model-field", + "django-model-relation", + "django-signal-receiver", + "django-url-configuration", + "django-url-include", + "django-url-route", + "django-view", + "django-view-action", +}) +_RELATION_LABELS = { + "django-app-config": ("app", "app config", "application"), + "django-installed-app": ("app", "installed app"), + "django-middleware": ("middleware",), + "django-middleware-component": ("component", "middleware"), + "django-model": ("model",), + "django-model-field": ("field", "model field"), + "django-model-relation": ("model relation", "relation", "relationship"), + "django-signal-receiver": ("receiver", "signal receiver"), + "django-url-configuration": ("url config", "url configuration"), + "django-url-include": ("include", "url include"), + "django-url-route": ("route", "url", "url route"), + "django-view": ("view",), + "django-view-action": ("action", "view action"), +} +_RELATION_STATES = { + "django-app-config": ("is not configured",), + "django-installed-app": ("is not installed", "is not registered"), + "django-middleware": ("is not configured", "is not installed"), + "django-middleware-component": ("is not configured",), + "django-signal-receiver": ("is not connected", "is not registered"), + "django-url-configuration": ("is not configured",), + "django-url-include": ("is not included",), + "django-url-route": ("is not configured", "is not registered"), + "django-view-action": ("is not handled",), +} +_COMMON_RELATION_STATES = ( + "does not exist", + "doesn't exist", + "is absent", + "is missing", + "is not declared", + "is not defined", +) +_ABSENCE_END = ( + r"(?=$|[.!?,;:]|\s+(?:and|because|despite|even|for|from|in|into|on|" + r"so|therefore|when|while|with|without)\b)" +) + + +def _name(node: ast.AST | None) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + prefix = _name(node.value) + return f"{prefix}.{node.attr}" if prefix else node.attr + if isinstance(node, ast.Call): + return _name(node.func) + if isinstance(node, ast.Subscript): + return _name(node.value) + return "" + + +def _literal(node: ast.AST | None) -> str: + if isinstance(node, ast.Constant) and isinstance(node.value, (str, int, float, bool)): + return str(node.value) + if isinstance(node, (ast.Name, ast.Attribute)): + return _name(node) + return "" + + +def _string_literal(node: ast.AST | None) -> str: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return "" + + +def _keyword(call: ast.Call, name: str) -> ast.AST | None: + return next((keyword.value for keyword in call.keywords if keyword.arg == name), None) + + +def _assignment(node: ast.stmt) -> tuple[str, ast.AST | None]: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + return _name(node.targets[0]), node.value + if isinstance(node, ast.AnnAssign): + return _name(node.target), node.value + return "", None + + +def _sequence_values(node: ast.AST | None) -> tuple[ast.AST, ...]: + """Return only statically visible values; dynamic expansions are ignored.""" + if isinstance(node, (ast.List, ast.Tuple, ast.Set)): + return tuple(node.elts) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return (*_sequence_values(node.left), *_sequence_values(node.right)) + return () + + +def _bound_names(node: ast.AST) -> tuple[str, ...]: + if isinstance(node, ast.Name): + return (node.id,) + if isinstance(node, (ast.List, ast.Tuple)): + return tuple(name for item in node.elts for name in _bound_names(item)) + if isinstance(node, ast.Starred): + return _bound_names(node.value) + return () + + +class _ScopeBindingCollector(ast.NodeVisitor): + """Collect bindings in one scope without descending into nested scopes.""" + + def __init__(self) -> None: + self.names: set[str] = set() + + def visit_Name(self, node: ast.Name) -> None: # noqa: N802 + if isinstance(node.ctx, (ast.Store, ast.Del)): + self.names.add(node.id) + + def visit_Import(self, node: ast.Import) -> None: # noqa: N802 + self.names.update( + alias.asname or alias.name.split(".", 1)[0] + for alias in node.names + ) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: # noqa: N802 + self.names.update( + alias.asname or alias.name + for alias in node.names + if alias.name != "*" + ) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: # noqa: N802 + self.names.add(node.name) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: # noqa: N802 + self.names.add(node.name) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: # noqa: N802 + self.names.add(node.name) + + def visit_Lambda(self, node: ast.Lambda) -> None: # noqa: N802 + return + + def visit_ListComp(self, node: ast.ListComp) -> None: # noqa: N802 + return + + def visit_SetComp(self, node: ast.SetComp) -> None: # noqa: N802 + return + + def visit_DictComp(self, node: ast.DictComp) -> None: # noqa: N802 + return + + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> None: # noqa: N802 + return + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None: # noqa: N802 + if node.name: + self.names.add(node.name) + for statement in node.body: + self.visit(statement) + + def visit_MatchAs(self, node: ast.MatchAs) -> None: # noqa: N802 + if node.name: + self.names.add(node.name) + if node.pattern: + self.visit(node.pattern) + + def visit_MatchStar(self, node: ast.MatchStar) -> None: # noqa: N802 + if node.name: + self.names.add(node.name) + + def visit_MatchMapping(self, node: ast.MatchMapping) -> None: # noqa: N802 + if node.rest: + self.names.add(node.rest) + for pattern in node.patterns: + self.visit(pattern) + + +def _scope_bound_names(statements: tuple[ast.stmt, ...] | list[ast.stmt]) -> frozenset[str]: + collector = _ScopeBindingCollector() + for statement in statements: + collector.visit(statement) + return frozenset(collector.names) + + +@dataclass(frozen=True) +class _ImportResolver: + """Resolve only module-level names whose imports are statically visible.""" + + bindings: dict[str, tuple[tuple[int, str | None], ...]] + + @classmethod + def from_tree(cls, tree: ast.Module) -> _ImportResolver: + mutable: dict[str, list[tuple[int, str | None]]] = {} + + def bind(name: str, line: int, canonical: str | None) -> None: + mutable.setdefault(name, []).append((line, canonical)) + + for statement in tree.body: + if isinstance(statement, ast.Import): + for alias in statement.names: + bound = alias.asname or alias.name.split(".", 1)[0] + canonical = alias.name if alias.asname else bound + bind(bound, statement.lineno, canonical) + continue + if isinstance(statement, ast.ImportFrom): + for alias in statement.names: + if alias.name == "*": + continue + bound = alias.asname or alias.name + canonical = ( + f"{statement.module}.{alias.name}" + if statement.level == 0 and statement.module + else None + ) + bind(bound, statement.lineno, canonical) + continue + + rebound = _scope_bound_names([statement]) + + # The prior binding remains valid while decorators, bases, and the + # assignment RHS are evaluated. Invalidate it for later statements. + active_from = (getattr(statement, "end_lineno", None) or statement.lineno) + 1 + for name in sorted(rebound): + bind(name, active_from, None) + + return cls({name: tuple(events) for name, events in mutable.items()}) + + def resolve(self, node: ast.AST | None) -> str: + if isinstance(node, ast.Name): + line = getattr(node, "lineno", 0) + canonical = "" + for active_from, candidate in self.bindings.get(node.id, ()): + if active_from > line: + break + canonical = candidate or "" + return canonical + if isinstance(node, ast.Attribute): + parent = self.resolve(node.value) + return f"{parent}.{node.attr}" if parent else "" + return "" + + +def _pattern_calls( + node: ast.AST | None, + imports: _ImportResolver, +) -> tuple[ast.Call, ...]: + if isinstance(node, (ast.List, ast.Tuple, ast.Set)): + return tuple( + call + for value in node.elts + for call in _pattern_calls(value, imports) + ) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return ( + *_pattern_calls(node.left, imports), + *_pattern_calls(node.right, imports), + ) + if not isinstance(node, ast.Call): + return () + operation = imports.resolve(node.func) + if operation in {"django.urls.path", "django.urls.re_path"}: + return (node,) + if operation == "django.conf.urls.i18n.i18n_patterns": + return tuple( + call + for value in node.args + for call in _pattern_calls(value, imports) + ) + return () + + +def _attributes(**values: str) -> tuple[tuple[str, str], ...]: + return tuple(sorted( + (key, value) + for key, value in values.items() + if value + )) + + +def _module_name(path: str) -> str: + module = path + for extension in _EXTENSIONS: + if module.casefold().endswith(extension): + module = module[:-len(extension)] + break + module = module.replace("/", ".") + return module.removesuffix(".__init__") + + +def _qualified(module: str, name: str) -> str: + return f"{module}.{name}" if module else name + + +def _bounded_facts(facts: set[GraphFact]) -> tuple[GraphFact, ...]: + """Keep direct calls bounded without starving a topology kind.""" + by_kind: dict[str, list[GraphFact]] = {} + for fact in sorted(facts): + by_kind.setdefault(fact.kind, []).append(fact) + selected: list[GraphFact] = [] + offset = 0 + kinds = tuple(sorted(by_kind)) + while len(selected) < _MAX_FACTS_PER_FILE: + added = False + for kind in kinds: + values = by_kind[kind] + if offset < len(values): + selected.append(values[offset]) + added = True + if len(selected) == _MAX_FACTS_PER_FILE: + break + if not added: + break + offset += 1 + return tuple(sorted(selected)) + + +def _include_target(call: ast.Call, imports: _ImportResolver) -> str: + if not call.args: + return "" + candidate = call.args[0] + direct = _string_literal(candidate) + if direct: + return direct + if isinstance(candidate, (ast.List, ast.Tuple)) and candidate.elts: + return _string_literal(candidate.elts[0]) + if isinstance(candidate, (ast.Name, ast.Attribute)): + return imports.resolve(candidate) + return "" + + +def _fact_identifiers(fact: GraphFact) -> frozenset[str]: + values = [fact.source, fact.target, *(value for _, value in fact.attributes)] + identifiers: set[str] = set() + for value in values: + normalized = value.casefold().strip() + if not normalized: + continue + identifiers.add(normalized) + for separator in ("#", "/", ".", ":"): + parts = normalized.replace("<", " ").replace(">", " ").split(separator) + identifiers.update(part.strip(" _-()[],'\"") for part in parts) + return frozenset(value for value in identifiers if len(value) >= 3) + + +def _mentions_identifier(message: str, identifier: str) -> bool: + if identifier.replace("_", "").isalnum(): + return re.search( + rf"(? str: + escaped = re.escape(identifier) + if identifier.replace("_", "").isalnum(): + return rf"(? bool: + labels = _RELATION_LABELS.get(fact.kind, ()) + label_pattern = "|".join(re.escape(label) for label in labels) + optional_label = rf"(?:\s+(?:{label_pattern}))?" if labels else "" + states = (*_COMMON_RELATION_STATES, *_RELATION_STATES.get(fact.kind, ())) + state_pattern = "|".join(re.escape(state) for state in states) + named = r"(?:named\s+)?" + for identifier in _fact_identifiers(fact): + identifier_pattern = _identifier_pattern(identifier) + if re.search( + rf"{identifier_pattern}{optional_label}\s+(?:{state_pattern}){_ABSENCE_END}", + message, + ): + return True + if labels and re.search( + rf"(? bool: + return canonical in { + "django.db.models.Model", + "django.db.models.base.Model", + } + + +def _is_app_config(canonical: str) -> bool: + return canonical in { + "django.apps.AppConfig", + "django.apps.config.AppConfig", + } + + +def _model_field_type(canonical: str) -> str: + if not canonical.startswith("django.db.models."): + return "" + field_type = canonical.rsplit(".", 1)[-1] + if field_type.endswith("Field") or field_type in _MODEL_RELATIONS: + return field_type + return "" + + +def _view_base(canonical: str) -> str: + if canonical.startswith("django.views.") or canonical.startswith("rest_framework."): + base = canonical.rsplit(".", 1)[-1] + if base.endswith(("View", "ViewSet")): + return base + return "" + + +def _is_function_view_decorator(canonical: str) -> bool: + return ( + canonical.startswith("django.views.decorators.") + or canonical in { + "django.contrib.admin.views.decorators.staff_member_required", + "django.contrib.auth.decorators.login_required", + "django.contrib.auth.decorators.permission_required", + "django.contrib.auth.decorators.user_passes_test", + "rest_framework.decorators.api_view", + } + ) + + +@dataclass(frozen=True) +class _CustomSignalResolver: + bindings: dict[str, tuple[tuple[int, bool], ...]] + + @classmethod + def from_tree( + cls, + tree: ast.Module, + imports: _ImportResolver, + ) -> _CustomSignalResolver: + mutable: dict[str, list[tuple[int, bool]]] = {} + for statement in tree.body: + names: tuple[str, ...] = () + proven = False + if isinstance(statement, ast.Assign): + names = tuple( + name for target in statement.targets for name in _bound_names(target) + ) + proven = ( + len(names) == 1 + and isinstance(statement.value, ast.Call) + and imports.resolve(statement.value.func) == "django.dispatch.Signal" + ) + elif isinstance(statement, ast.AnnAssign): + names = _bound_names(statement.target) + proven = ( + len(names) == 1 + and isinstance(statement.value, ast.Call) + and imports.resolve(statement.value.func) == "django.dispatch.Signal" + ) + elif isinstance(statement, ast.AugAssign): + names = _bound_names(statement.target) + elif isinstance(statement, ast.Import): + names = tuple( + alias.asname or alias.name.split(".", 1)[0] + for alias in statement.names + ) + elif isinstance(statement, ast.ImportFrom): + names = tuple( + alias.asname or alias.name + for alias in statement.names + if alias.name != "*" + ) + elif isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names = (statement.name,) + elif isinstance(statement, ast.Delete): + names = tuple( + name for target in statement.targets for name in _bound_names(target) + ) + else: + names = tuple(sorted(_scope_bound_names([statement]))) + active_from = (getattr(statement, "end_lineno", None) or statement.lineno) + 1 + for name in names: + mutable.setdefault(name, []).append((active_from, proven)) + return cls({name: tuple(events) for name, events in mutable.items()}) + + def contains(self, name: str, line: int) -> bool: + proven = False + for active_from, candidate in self.bindings.get(name, ()): + if active_from > line: + break + proven = candidate + return proven + + +def _is_proven_signal( + node: ast.AST, + imports: _ImportResolver, + custom_signals: _CustomSignalResolver, +) -> bool: + canonical = imports.resolve(node) + if canonical.startswith("django.") and ".signals." in canonical: + return True + return ( + isinstance(node, ast.Name) + and custom_signals.contains(node.id, getattr(node, "lineno", 0)) + ) + + +def _root_name(node: ast.AST) -> str: + while isinstance(node, ast.Attribute): + node = node.value + return node.id if isinstance(node, ast.Name) else "" + + +def _function_bindings( + node: ast.FunctionDef | ast.AsyncFunctionDef, +) -> frozenset[str]: + arguments = { + argument.arg + for argument in ( + *node.args.posonlyargs, + *node.args.args, + *node.args.kwonlyargs, + ) + } + if node.args.vararg: + arguments.add(node.args.vararg.arg) + if node.args.kwarg: + arguments.add(node.args.kwarg.arg) + return frozenset((*arguments, *_scope_bound_names(node.body))) + + +class _ScopedCallCollector(ast.NodeVisitor): + """Collect calls while recording lexical names that shadow module imports.""" + + _MODULE_COMPOUNDS = ( + ast.AsyncFor, + ast.AsyncWith, + ast.For, + ast.If, + ast.Match, + ast.Try, + ast.TryStar, + ast.While, + ast.With, + ) + + def __init__(self) -> None: + self.calls: list[tuple[ast.Call, frozenset[str]]] = [] + self._blocked: list[frozenset[str]] = [frozenset()] + + def visit_Module(self, node: ast.Module) -> None: # noqa: N802 + for statement in node.body: + # Conditional module-level bindings cannot be resolved exactly. + # Skip calls inside those statements instead of guessing a branch. + if isinstance(statement, self._MODULE_COMPOUNDS): + continue + self.visit(statement) + + def _visit_scope(self, body: list[ast.stmt], bindings: frozenset[str]) -> None: + self._blocked.append(self._blocked[-1] | bindings) + for statement in body: + self.visit(statement) + self._blocked.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: # noqa: N802 + self._visit_scope(node.body, _function_bindings(node)) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: # noqa: N802 + self._visit_scope(node.body, _function_bindings(node)) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: # noqa: N802 + self._visit_scope(node.body, _scope_bound_names(node.body)) + + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 + self.calls.append((node, self._blocked[-1])) + self.generic_visit(node) + + +def _scoped_calls(tree: ast.Module) -> tuple[tuple[ast.Call, frozenset[str]], ...]: + collector = _ScopedCallCollector() + collector.visit(tree) + return tuple(collector.calls) + + +@dataclass(frozen=True) +class DjangoPlugin: + descriptor: PluginDescriptor + + # Django relationships are emitted per file. There is deliberately no + # repository session: URL imports and setting overrides cannot be restored + # exactly from a partial snapshot without owning a complete Python resolver. + def index_file(self, artifact: FileArtifact): + if artifact.deleted or not artifact.path.casefold().endswith(_EXTENSIONS): + return PluginOutcome.abstained() + try: + tree = ast.parse(artifact.content, filename=artifact.path, type_comments=True) + except SyntaxError as exception: + return PluginOutcome.failed(PluginDiagnostic( + "django-python-parse-failed", + f"SyntaxError at line {exception.lineno}: {exception.msg}", + self.descriptor.id, + path=artifact.path, + recoverable=True, + )) + + module = _module_name(artifact.path) + filename = PurePosixPath(artifact.path).name.casefold() + imports = _ImportResolver.from_tree(tree) + custom_signals = _CustomSignalResolver.from_tree(tree, imports) + facts: set[GraphFact] = set() + self._settings_and_urls(tree, imports, artifact.path, module, facts) + self._classes(tree, imports, artifact.path, module, filename, facts) + self._function_views_and_signals( + tree, imports, custom_signals, artifact.path, module, facts, + ) + self._signal_connections( + tree, imports, custom_signals, artifact.path, module, facts, + ) + + if not facts: + return PluginOutcome.abstained() + return PluginOutcome.handled(_bounded_facts(facts)) + + @staticmethod + def _settings_and_urls( + tree: ast.Module, + imports: _ImportResolver, + path: str, + module: str, + facts: set[GraphFact], + ) -> None: + pattern_calls: list[ast.Call] = [] + for statement in tree.body: + variable, value = _assignment(statement) + if variable == "INSTALLED_APPS": + for item in _sequence_values(value): + app = _string_literal(item) + if app: + facts.add(GraphFact( + "django-installed-app", module, "installs", app, + path, getattr(item, "lineno", statement.lineno), + )) + elif variable == "MIDDLEWARE": + for item in _sequence_values(value): + middleware = _string_literal(item) + if middleware: + facts.add(GraphFact( + "django-middleware", module, "uses", middleware, + path, getattr(item, "lineno", statement.lineno), + )) + elif variable == "ROOT_URLCONF": + url_configuration = _string_literal(value) + if url_configuration: + facts.add(GraphFact( + "django-url-configuration", module, "uses", + url_configuration, path, statement.lineno, + )) + elif variable == "urlpatterns": + pattern_calls.extend(_pattern_calls(value, imports)) + elif isinstance(statement, ast.AugAssign) and _name(statement.target) == "urlpatterns": + pattern_calls.extend(_pattern_calls(statement.value, imports)) + + for call in pattern_calls: + if len(call.args) < 2: + continue + operation = imports.resolve(call.func).rsplit(".", 1)[-1] + if not ( + isinstance(call.args[0], ast.Constant) + and isinstance(call.args[0].value, str) + ): + continue + route = _string_literal(call.args[0]) + route = route or "/" + destination = call.args[1] + route_name = _string_literal(_keyword(call, "name")) + source = f"{module}:{route}" + if ( + isinstance(destination, ast.Call) + and imports.resolve(destination.func) == "django.urls.include" + ): + included = _include_target(destination, imports) + if not included: + continue + namespace = _string_literal(_keyword(destination, "namespace")) + facts.add(GraphFact( + "django-url-include", source, "includes", included, + path, call.lineno, + _attributes(namespace=namespace, pattern=operation, route_name=route_name), + )) + continue + if isinstance(destination, ast.Call) and not ( + isinstance(destination.func, ast.Attribute) + and destination.func.attr == "as_view" + ): + continue + view = _name(destination) + if not view: + continue + facts.add(GraphFact( + "django-url-route", source, "dispatches-to", view, + path, call.lineno, + _attributes(pattern=operation, route_name=route_name), + )) + + @staticmethod + def _classes( + tree: ast.Module, + imports: _ImportResolver, + path: str, + module: str, + filename: str, + facts: set[GraphFact], + ) -> None: + for node in tree.body: + if not isinstance(node, ast.ClassDef): + continue + qualified = _qualified(module, node.name) + base_names = {imports.resolve(base) for base in node.bases} + + if any(_is_app_config(base) for base in base_names): + values = { + name: _string_literal(value) + for statement in node.body + for name, value in (_assignment(statement),) + if name in {"label", "name", "verbose_name"} + } + facts.add(GraphFact( + "django-app-config", qualified, "configures", + values.get("name") or qualified, path, node.lineno, + _attributes(label=values.get("label", ""), verbose_name=values.get("verbose_name", "")), + )) + + if any(_is_model_base(base) for base in base_names): + facts.add(GraphFact( + "django-model", module, "declares", qualified, path, node.lineno, + )) + for statement in node.body: + field_name, value = _assignment(statement) + if not field_name or not isinstance(value, ast.Call): + continue + field_type = _model_field_type(imports.resolve(value.func)) + if not field_type: + continue + field = f"{qualified}.{field_name}" + facts.add(GraphFact( + "django-model-field", qualified, "declares", field, + path, statement.lineno, _attributes(field_type=field_type), + )) + relation = _MODEL_RELATIONS.get(field_type) + if relation is None or not value.args: + continue + target = _literal(value.args[0]) + if not target: + continue + facts.add(GraphFact( + "django-model-relation", field, relation, target, + path, statement.lineno, + _attributes( + on_delete=_literal(_keyword(value, "on_delete")), + related_name=_literal(_keyword(value, "related_name")), + ), + )) + + view_base = next(filter(None, map(_view_base, sorted(base_names))), "") + if view_base: + facts.add(GraphFact( + "django-view", module, "declares", qualified, path, node.lineno, + _attributes(base=view_base), + )) + for statement in node.body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)) and statement.name in _VIEW_METHODS: + facts.add(GraphFact( + "django-view-action", qualified, "handles", + statement.name.upper(), path, statement.lineno, + )) + + if filename == "middleware.py": + hooks = tuple(sorted( + statement.name + for statement in node.body + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)) + and (statement.name == "__call__" or statement.name.startswith("process_")) + )) + if hooks: + facts.add(GraphFact( + "django-middleware-component", module, "declares", qualified, + path, node.lineno, (("hooks", ",".join(hooks)),), + )) + + @staticmethod + def _function_views_and_signals( + tree: ast.Module, + imports: _ImportResolver, + custom_signals: _CustomSignalResolver, + path: str, + module: str, + facts: set[GraphFact], + ) -> None: + for node in tree.body: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + qualified = _qualified(module, node.name) + decorators = tuple( + imports.resolve(decorator.func if isinstance(decorator, ast.Call) else decorator) + for decorator in node.decorator_list + ) + if any(_is_function_view_decorator(decorator) for decorator in decorators): + facts.add(GraphFact( + "django-view", module, "declares", qualified, path, node.lineno, + (("style", "function"),), + )) + for decorator in node.decorator_list: + if not isinstance(decorator, ast.Call): + continue + if ( + imports.resolve(decorator.func) != "django.dispatch.receiver" + or not decorator.args + or not _is_proven_signal(decorator.args[0], imports, custom_signals) + ): + continue + signal = _name(decorator.args[0]) + if not signal: + continue + facts.add(GraphFact( + "django-signal-receiver", signal, "notifies", qualified, + path, decorator.lineno, + _attributes(sender=_literal(_keyword(decorator, "sender"))), + )) + + @staticmethod + def _signal_connections( + tree: ast.Module, + imports: _ImportResolver, + custom_signals: _CustomSignalResolver, + path: str, + module: str, + facts: set[GraphFact], + ) -> None: + for node, blocked_names in _scoped_calls(tree): + if not isinstance(node.func, ast.Attribute): + continue + if node.func.attr != "connect" or not node.args: + continue + if _root_name(node.func.value) in blocked_names: + continue + if not _is_proven_signal(node.func.value, imports, custom_signals): + continue + signal = _name(node.func.value) + receiver = _name(node.args[0]) + if not signal or not receiver: + continue + if "." not in receiver: + receiver = _qualified(module, receiver) + facts.add(GraphFact( + "django-signal-receiver", signal, "notifies", receiver, + path, node.lineno, + _attributes(sender=_literal(_keyword(node, "sender"))), + )) + + def review(self, paths: tuple[str, ...]): + selected = tuple(sorted( + path for path in paths + if path.casefold().endswith(_EXTENSIONS) + ))[:_MAX_EVIDENCE_REQUESTS] + if not selected: + return PluginOutcome.abstained() + rules = tuple(sorted(( + "Resolve Django URL dispatch through urlpatterns, include prefixes, and exact view declarations before judging endpoint reachability.", + "Treat settings, app configuration, middleware, model relations, and signal receivers as topology context; their presence alone is not defect proof.", + ))) + return PluginOutcome.handled(ReviewContribution( + rules=rules, + evidence_requests=tuple(EvidenceRequest( + "django-topology", + path, + "exact Django settings, app, middleware, URL, view, model-relation, and signal facts", + ) for path in selected), + )) + + def validate(self, claim: CandidateClaim): + requested_kind = claim.claim_kind or claim.category + if not requested_kind.startswith("django-"): + return PluginOutcome.abstained() + if not claim.path.casefold().endswith(_EXTENSIONS): + return PluginOutcome.abstained() + + if requested_kind == "django-topology": + expected_kinds = _FACT_KINDS + elif requested_kind in _FACT_KINDS: + expected_kinds = frozenset({requested_kind}) + else: + return PluginOutcome.handled(ValidationResult( + ValidationDecision.INSUFFICIENT_EVIDENCE, + "django-unknown-fact-kind", + "The Django claim kind is not owned by an exact validator.", + )) + matching = tuple( + fact for fact in claim.evidence + if fact.kind in expected_kinds + and claim.path in {fact.path, *fact.related_paths} + ) + message = claim.message.casefold() + relevant = tuple( + fact for fact in matching + if any( + _mentions_identifier(message, identifier) + for identifier in _fact_identifiers(fact) + ) + ) + if any(_is_absence_claim(fact, message) for fact in relevant): + return PluginOutcome.handled(ValidationResult( + ValidationDecision.REJECT, + "django-absence-contradicted", + "The candidate claims Django topology is absent, but an exact matching framework fact exists.", + )) + if relevant: + return PluginOutcome.handled(ValidationResult( + ValidationDecision.INSUFFICIENT_EVIDENCE, + "django-topology-not-defect-proof", + "The cited Django relationship exists, but structural presence alone does not prove defective behavior.", + )) + if matching: + return PluginOutcome.handled(ValidationResult( + ValidationDecision.INSUFFICIENT_EVIDENCE, + "django-cited-identifier-mismatch", + "Django topology facts exist for the path, but their identifiers do not match the candidate message.", + )) + return PluginOutcome.handled(ValidationResult( + ValidationDecision.INSUFFICIENT_EVIDENCE, + "django-evidence-unavailable", + "No exact matching Django topology evidence was supplied for this framework claim.", + )) + + +def create_plugin(descriptor: PluginDescriptor) -> DjangoPlugin: + return DjangoPlugin(descriptor) diff --git a/analysis-plugins/frameworks/ember/java/pom.xml b/analysis-plugins/frameworks/ember/java/pom.xml new file mode 100644 index 00000000..1b6756c1 --- /dev/null +++ b/analysis-plugins/frameworks/ember/java/pom.xml @@ -0,0 +1,15 @@ + + + 4.0.0 + + org.rostilos.codecrow + codecrow-framework-plugins + 1.0 + ../../pom.xml + + codecrow-plugin-ember + jar + ember + diff --git a/analysis-plugins/frameworks/ember/java/src/main/java/org/rostilos/codecrow/plugins/ember/EmberPlugin.java b/analysis-plugins/frameworks/ember/java/src/main/java/org/rostilos/codecrow/plugins/ember/EmberPlugin.java new file mode 100644 index 00000000..1cea7d16 --- /dev/null +++ b/analysis-plugins/frameworks/ember/java/src/main/java/org/rostilos/codecrow/plugins/ember/EmberPlugin.java @@ -0,0 +1,23 @@ +package org.rostilos.codecrow.plugins.ember; + +import org.rostilos.codecrow.plugins.CodeCrowPlugin; +import org.rostilos.codecrow.plugins.PluginDescriptor; +import org.rostilos.codecrow.plugins.PluginManifestLoader; + +public final class EmberPlugin implements CodeCrowPlugin { + private final PluginDescriptor descriptor; + + public EmberPlugin() { + try (var input = EmberPlugin.class.getResourceAsStream( + "/META-INF/codecrow/plugins/ember/plugin.json")) { + descriptor = new PluginManifestLoader().loadDescriptor(input); + } catch (Exception exception) { + throw new IllegalStateException("cannot load Ember plugin descriptor", exception); + } + } + + @Override + public PluginDescriptor descriptor() { + return descriptor; + } +} diff --git a/analysis-plugins/frameworks/ember/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin b/analysis-plugins/frameworks/ember/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin new file mode 100644 index 00000000..310fdc7b --- /dev/null +++ b/analysis-plugins/frameworks/ember/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin @@ -0,0 +1 @@ +org.rostilos.codecrow.plugins.ember.EmberPlugin diff --git a/analysis-plugins/frameworks/ember/java/src/test/java/org/rostilos/codecrow/plugins/ember/EmberPluginTest.java b/analysis-plugins/frameworks/ember/java/src/test/java/org/rostilos/codecrow/plugins/ember/EmberPluginTest.java new file mode 100644 index 00000000..c16aa89d --- /dev/null +++ b/analysis-plugins/frameworks/ember/java/src/test/java/org/rostilos/codecrow/plugins/ember/EmberPluginTest.java @@ -0,0 +1,20 @@ +package org.rostilos.codecrow.plugins.ember; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.plugins.PluginCapability; +import org.rostilos.codecrow.plugins.PluginKind; + +import static org.assertj.core.api.Assertions.assertThat; + +class EmberPluginTest { + @Test + void packagedManifestLoadsThroughTheJavaContract() { + var descriptor = new EmberPlugin().descriptor(); + assertThat(descriptor.id()).isEqualTo("ember"); + assertThat(descriptor.kind()).isEqualTo(PluginKind.FRAMEWORK); + assertThat(descriptor.requires()).containsExactly("json"); + assertThat(descriptor.capabilities()).contains( + PluginCapability.GRAPH, PluginCapability.INDEX, PluginCapability.VALIDATION); + assertThat(descriptor.detection().alternatives()).hasSize(2); + } +} diff --git a/analysis-plugins/frameworks/ember/plugin.json b/analysis-plugins/frameworks/ember/plugin.json new file mode 100644 index 00000000..c2128897 --- /dev/null +++ b/analysis-plugins/frameworks/ember/plugin.json @@ -0,0 +1,32 @@ +{ + "id": "ember", + "kind": "framework", + "requires": ["json"], + "capabilities": ["context", "graph", "index", "planning", "prompt", "validation"], + "detection": { + "extensions": [], + "filesAll": [], + "filesAny": [], + "contentMarkers": [], + "alternatives": [ + { + "filesAll": [], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [{"path": "package.json", "contains": "\"ember-cli\""}] + }, + { + "filesAll": [], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [{"path": "package.json", "contains": "\"ember-source\""}] + } + ] + }, + "entrypoints": { + "java": "org.rostilos.codecrow.plugins.ember.EmberPlugin", + "python": "codecrow_plugin_ember:create_plugin" + } +} diff --git a/analysis-plugins/frameworks/ember/python/codecrow_plugin_ember/__init__.py b/analysis-plugins/frameworks/ember/python/codecrow_plugin_ember/__init__.py new file mode 100644 index 00000000..ae938f5e --- /dev/null +++ b/analysis-plugins/frameworks/ember/python/codecrow_plugin_ember/__init__.py @@ -0,0 +1,683 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from codecrow_plugins import ( + CandidateClaim, + EvidenceRequest, + FileArtifact, + GraphFact, + PluginDescriptor, + PluginDiagnostic, + PluginOutcome, + ReviewContribution, + TreeSitterDocument, + ValidationDecision, + ValidationResult, +) + + +_SCRIPT_EXTENSIONS = (".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx") +_SUPPORTED_EXTENSIONS = (*_SCRIPT_EXTENSIONS, ".hbs") +_GRAMMARS = { + ".cjs": ("tree_sitter_javascript", "language"), + ".cts": ("tree_sitter_typescript", "language_typescript"), + ".js": ("tree_sitter_javascript", "language"), + ".jsx": ("tree_sitter_javascript", "language"), + ".mjs": ("tree_sitter_javascript", "language"), + ".mts": ("tree_sitter_typescript", "language_typescript"), + ".ts": ("tree_sitter_typescript", "language_typescript"), + ".tsx": ("tree_sitter_typescript", "language_tsx"), +} +_PATH_ROLE = re.compile( + r"(?:^|/)app/(?Proutes|controllers|components|services|models)/" + r"(?P.+)\.(?:cjs|cts|js|jsx|mjs|mts|ts|tsx)$", + re.IGNORECASE, +) +_ROUTE_TEMPLATE = re.compile(r"(?:^|/)app/templates/(?P(?!components/).+)\.hbs$", re.IGNORECASE) +_COMPONENT_TEMPLATE = re.compile( + r"(?:^|/)app/(?:templates/)?components/(?P.+)\.hbs$", re.IGNORECASE +) +_APP_ROUTER_PATH = re.compile( + r"^app/router\.(?:cjs|cts|js|jsx|mjs|mts|ts|tsx)$", re.IGNORECASE, +) +_SERVICE_DECORATOR = re.compile( + r"@service(?:\s*\(\s*(['\"])(?P[^'\"]+)\1\s*\))?" + r"\s+(?:declare\s+)?(?P[A-Za-z_$][\w$]*)" +) +_SERVICE_PROPERTY = re.compile( + r"(?P[A-Za-z_$][\w$]*)\s*=\s*(?:service|inject\.service)\s*\(" + r"\s*(?:(['\"])(?P[^'\"]+)\2\s*)?\)" +) +_DATA_DECORATOR = re.compile( + r"@(?PbelongsTo|hasMany)\s*\(\s*(['\"])(?P[^'\"]+)\2[^)]*\)" + r"\s+(?:declare\s+)?(?P[A-Za-z_$][\w$]*)", + re.DOTALL, +) +_DATA_PROPERTY = re.compile( + r"(?P[A-Za-z_$][\w$]*)\s*(?::|=)\s*" + r"(?PbelongsTo|hasMany)\s*\(\s*(['\"])(?P[^'\"]+)\3", + re.DOTALL, +) +_ANGLE_COMPONENT = re.compile(r"<(?P[A-Z][A-Za-z0-9]*(?:::[A-Z][A-Za-z0-9]*)*)\b") +_CLASSIC_COMPONENT = re.compile(r"{{\s*(?P[a-z][a-z0-9]*(?:[-/][a-z0-9-]+)+)\b") +_COMPONENT_HELPER = re.compile(r"{{\s*component\s+(['\"])(?P[^'\"]+)\1") +_TEMPLATE_COMMENT = re.compile(r"{{!--.*?--}}|{{!.*?}}|", re.DOTALL) +_TOKEN_CHARACTER = re.compile(r"[A-Za-z0-9_$]") +_FACT_KINDS = frozenset({ + "ember-component", "ember-controller", "ember-data-model", + "ember-data-relationship", "ember-route", "ember-route-controller", + "ember-route-module", "ember-service", "ember-service-injection", + "ember-template", "ember-template-association", "ember-template-component", +}) +_RELATION_LABELS = { + "ember-component": ("component",), + "ember-controller": ("controller",), + "ember-data-model": ("model",), + "ember-data-relationship": ("relation", "relationship"), + "ember-route": ("route",), + "ember-route-controller": ("controller",), + "ember-route-module": ("route", "route module"), + "ember-service": ("service",), + "ember-service-injection": ("injection", "service", "service injection"), + "ember-template": ("template",), + "ember-template-association": ("template", "template association"), + "ember-template-component": ("component",), +} +_RELATION_ACTIONS = { + "belongs-to": ("does not belong to", "doesn't belong to"), + "declares": ("does not declare", "doesn't declare"), + "defines": ("does not define", "doesn't define"), + "depends-on": ("does not inject", "doesn't inject", "does not use", "doesn't use"), + "has-many": ("does not have", "doesn't have"), + "invokes": ("does not invoke", "doesn't invoke", "does not render", "doesn't render"), + "renders-component": ("does not render", "doesn't render"), + "renders-route": ("does not render", "doesn't render"), + "uses-controller": ("does not use", "doesn't use"), +} +_RELATION_STATES = { + "ember-data-relationship": ("is not associated", "is not declared"), + "ember-route": ("is not declared", "is not nested", "is not registered"), + "ember-route-controller": ("is not associated",), + "ember-service-injection": ("is not injected",), + "ember-template-association": ("is not associated", "is not rendered"), + "ember-template-component": ("is not invoked", "is not rendered"), +} +_COMMON_RELATION_STATES = ( + "does not exist", "doesn't exist", "is absent", "is missing", + "is not declared", "is not defined", +) +_ABSENCE_END = r"(?=$|[.!?,;:])" + + +def _line(content: str, offset: int) -> int: + return content.count("\n", 0, offset) + 1 + + +def _parse_script(artifact: FileArtifact) -> TreeSitterDocument: + extension = PurePosixPath(artifact.path.casefold()).suffix + grammar_module, grammar_factory = _GRAMMARS[extension] + return TreeSitterDocument.parse(artifact.content, grammar_module, grammar_factory) + + +def _call_target(document: TreeSitterDocument, node) -> str: + return document.text(node.child_by_field_name("function")).replace(" ", "") + + +def _arguments(node) -> tuple[object, ...]: + arguments = node.child_by_field_name("arguments") + return tuple(arguments.named_children) if arguments is not None else () + + +def _literal(document: TreeSitterDocument, node) -> str: + if node is None or node.type not in {"string", "template_string"}: + return "" + text = document.text(node) + if len(text) < 2 or text[0] not in "'\"`" or text[-1] != text[0]: + return "" + if node.type == "template_string" and "${" in text: + return "" + return text[1:-1] + + +def _route_path_option(document: TreeSitterDocument, call) -> str: + for argument in _arguments(call)[1:]: + if argument.type != "object": + continue + for pair in argument.named_children: + if pair.type != "pair": + continue + key = document.text(pair.child_by_field_name("key")).strip("'\"") + if key == "path": + return _literal(document, pair.child_by_field_name("value")) + return "" + + +def _code_matches(pattern: re.Pattern[str], document: TreeSitterDocument, content: str): + excluded = tuple( + (node.start_byte, node.end_byte) + for node in document.walk() + if node.type in {"comment", "regex", "string", "template_string"} + ) + for match in pattern.finditer(content): + byte_offset = len(content[:match.start()].encode("utf-8")) + if not any(start <= byte_offset < end for start, end in excluded): + yield match + + +def _without_template_comments(content: str) -> str: + return _TEMPLATE_COMMENT.sub( + lambda match: "".join("\n" if character == "\n" else " " for character in match.group()), + content, + ) + + +def _path_role(path: str) -> tuple[str, str] | None: + match = _PATH_ROLE.search(path) + if match is None: + return None + role = match.group("role").casefold() + name = match.group("name").replace("\\", "/") + if role in {"routes", "controllers"}: + name = name.replace("/", ".") + return role, name + + +def _import_bindings( + document: TreeSitterDocument, + source: str, +) -> tuple[set[str], dict[str, str]]: + defaults: set[str] = set() + named: dict[str, str] = {} + for statement in document.root.named_children: + if statement.type != "import_statement": + continue + if _literal(document, statement.child_by_field_name("source")) != source: + continue + clause = next( + (child for child in statement.named_children if child.type == "import_clause"), + None, + ) + if clause is None: + continue + for child in clause.named_children: + if child.type == "identifier": + defaults.add(document.text(child)) + for specifier in document.descendants(clause, "import_specifier"): + imported = document.text(specifier.child_by_field_name("name")) + local = document.text(specifier.child_by_field_name("alias")) or imported + if imported and local: + named[local] = imported + return defaults, named + + +def _binding_names(document: TreeSitterDocument, node) -> set[str]: + if node is None: + return set() + if node.type in {"identifier", "property_identifier", "shorthand_property_identifier_pattern"}: + return {document.text(node)} + names: set[str] = set() + for child in node.named_children: + names.update(_binding_names(document, child)) + return names + + +def _import_is_visible(document: TreeSitterDocument, use, name: str) -> bool: + ancestor = use.parent + while ancestor is not None and ancestor != document.root: + if ancestor.type in { + "arrow_function", "function_declaration", "function_expression", + "generator_function", "generator_function_declaration", "method_definition", + }: + parameters = ancestor.child_by_field_name("parameters") + parameter = ancestor.child_by_field_name("parameter") + if name in _binding_names(document, parameters or parameter): + return False + if ancestor.type == "statement_block": + for statement in ancestor.named_children: + declarations = statement.named_children if statement.type == "export_statement" else (statement,) + for declaration in declarations: + if declaration.type in {"lexical_declaration", "variable_declaration"}: + for variable in declaration.named_children: + if ( + variable.type == "variable_declarator" + and name in _binding_names( + document, variable.child_by_field_name("name"), + ) + ): + return False + elif declaration.type in {"class_declaration", "function_declaration"}: + if document.text(declaration.child_by_field_name("name")) == name: + return False + if ancestor.type in {"class_declaration", "class"}: + if document.text(ancestor.child_by_field_name("name")) == name: + return False + ancestor = ancestor.parent + return True + + +def _module_router_owners(document: TreeSitterDocument) -> set[str]: + router_bases, _ = _import_bindings(document, "@ember/routing/router") + if not router_bases: + return set() + + default_exports: set[str] = set() + for statement in document.root.named_children: + if statement.type != "export_statement": + continue + text = document.text(statement) + if re.match(r"export\s+default\b", text): + value = statement.child_by_field_name("value") + if value is not None and value.type in {"identifier", "type_identifier"}: + default_exports.add(document.text(value)) + for child in statement.named_children: + if child.type in {"class", "class_declaration"}: + name = document.text(child.child_by_field_name("name")) + if name: + default_exports.add(name) + for clause in (child for child in statement.named_children if child.type == "export_clause"): + for specifier in clause.named_children: + if specifier.type != "export_specifier": + continue + if document.text(specifier.child_by_field_name("alias")) != "default": + continue + name = document.text(specifier.child_by_field_name("name")) + if name: + default_exports.add(name) + + candidates: set[str] = set() + for statement in document.root.named_children: + declarations = statement.named_children if statement.type == "export_statement" else (statement,) + for declaration in declarations: + if declaration.type in {"class", "class_declaration"}: + name = document.text(declaration.child_by_field_name("name")) + heritage = next( + (child for child in declaration.named_children if child.type == "class_heritage"), + None, + ) + base = ( + document.text(heritage.named_children[0]) + if heritage is not None and len(heritage.named_children) == 1 + else "" + ) + if name and base in router_bases: + candidates.add(name) + elif declaration.type in {"lexical_declaration", "variable_declaration"}: + for variable in declaration.named_children: + if variable.type != "variable_declarator": + continue + name = document.text(variable.child_by_field_name("name")) + value = variable.child_by_field_name("value") + target = _call_target(document, value) if value is not None and value.type == "call_expression" else "" + if name and any(target == f"{base}.extend" for base in router_bases): + candidates.add(name) + return candidates & default_exports + + +def _top_level_expression_call(document: TreeSitterDocument, call) -> bool: + return ( + call.parent is not None + and call.parent.type == "expression_statement" + and call.parent.parent == document.root + ) + + +def _router_routes(document: TreeSitterDocument, artifact: FileArtifact) -> set[GraphFact]: + facts: set[GraphFact] = set() + if _APP_ROUTER_PATH.fullmatch(artifact.path) is None: + return facts + router_owners = _module_router_owners(document) + if not router_owners: + return facts + for node in document.walk(): + if node.type != "call_expression" or _call_target(document, node) != "this.route": + continue + arguments = _arguments(node) + name = _literal(document, arguments[0]) if arguments else "" + if not name: + continue + + ancestors: list[object] = [] + map_owner = "" + parent = node.parent + while parent is not None: + if parent.type == "call_expression": + target = _call_target(document, parent) + if target == "this.route": + ancestors.append(parent) + elif target.endswith(".map"): + candidate = target[:-4] + if candidate in router_owners and _top_level_expression_call(document, parent): + map_owner = candidate + break + parent = parent.parent + if not map_owner: + continue + + chain: list[tuple[str, str]] = [] + for ancestor in reversed(ancestors): + ancestor_arguments = _arguments(ancestor) + ancestor_name = _literal(document, ancestor_arguments[0]) if ancestor_arguments else "" + if ancestor_name: + option = _route_path_option(document, ancestor) + chain.append((ancestor_name, option or ancestor_name)) + option = _route_path_option(document, node) + chain.append((name, option or name)) + + route_name = ".".join(item[0] for item in chain) + route_path = "/" + "/".join( + segment.strip("/") for _, segment in chain if segment.strip("/") + ) + attributes = [("path", route_path or "/")] + if len(chain) > 1: + attributes.append(("parent", ".".join(item[0] for item in chain[:-1]))) + facts.add(GraphFact( + "ember-route", + map_owner, + "declares", + route_name, + artifact.path, + document.line(node), + tuple(sorted(attributes)), + )) + return facts + + +def _script_facts(document: TreeSitterDocument, artifact: FileArtifact) -> set[GraphFact]: + facts = _router_routes(document, artifact) + path_role = _path_role(artifact.path) + owner = artifact.path + if path_role is not None: + role, name = path_role + owner = name + kind_by_role = { + "components": "ember-component", + "controllers": "ember-controller", + "models": "ember-data-model", + "routes": "ember-route-module", + "services": "ember-service", + } + facts.add(GraphFact(kind_by_role[role], artifact.path, "defines", name, artifact.path, 1)) + if role == "controllers": + facts.add(GraphFact("ember-route-controller", name, "uses-controller", name, artifact.path, 1)) + facts.add(GraphFact( + "ember-template-association", name, "uses-conventional-template", name, + artifact.path, 1, (("ownerKind", "controller"),), + )) + elif role == "routes": + facts.add(GraphFact( + "ember-template-association", name, "uses-conventional-template", name, + artifact.path, 1, (("ownerKind", "route"),), + )) + elif role == "components": + facts.add(GraphFact( + "ember-template-association", name, "uses-conventional-template", name, + artifact.path, 1, (("ownerKind", "component"),), + )) + + _, service_imports = _import_bindings(document, "@ember/service") + service_bindings = { + local for local, imported in service_imports.items() + if imported in {"inject", "service"} + } + _, data_imports = _import_bindings(document, "@ember-data/model") + data_bindings = { + local: imported for local, imported in data_imports.items() + if imported in {"belongsTo", "hasMany"} + } + field_types = {"field_definition", "public_field_definition"} + for field in document.walk(): + if field.type not in field_types: + continue + property_node = field.child_by_field_name("name") or field.child_by_field_name("property") + property_name = document.text(property_node) + if not re.fullmatch(r"[A-Za-z_$][\w$]*", property_name): + continue + + expressions: list[object] = [] + expressions.extend( + decorator.named_children[0] + for decorator in field.named_children + if decorator.type == "decorator" and decorator.named_children + ) + value = field.child_by_field_name("value") + if value is not None: + expressions.append(value) + for expression in expressions: + call = expression if expression.type == "call_expression" else None + binding = ( + _call_target(document, call) + if call is not None + else document.text(expression).lstrip("@") + ) + if "." in binding or not _import_is_visible(document, expression, binding): + continue + arguments = _arguments(call) if call is not None else () + explicit = _literal(document, arguments[0]) if arguments else "" + if binding in service_bindings: + facts.add(GraphFact( + "ember-service-injection", owner, "depends-on", explicit or property_name, + artifact.path, document.line(expression), (("property", property_name),), + )) + imported_relation = data_bindings.get(binding) + if ( + imported_relation is not None + and path_role is not None + and path_role[0] == "models" + and explicit + ): + facts.add(GraphFact( + "ember-data-relationship", path_role[1], + "belongs-to" if imported_relation == "belongsTo" else "has-many", + explicit, artifact.path, document.line(expression), + (("property", property_name),), + )) + return facts + + +def _template_facts(artifact: FileArtifact) -> set[GraphFact]: + facts: set[GraphFact] = set() + content = _without_template_comments(artifact.content) + component_match = _COMPONENT_TEMPLATE.search(artifact.path) + route_match = _ROUTE_TEMPLATE.search(artifact.path) + if component_match is not None: + name = component_match.group("name") + facts.add(GraphFact("ember-template", artifact.path, "defines", name, artifact.path, 1, + (("templateKind", "component"),))) + facts.add(GraphFact("ember-template-association", name, "renders-component", name, artifact.path, 1, + (("ownerKind", "component"),))) + elif route_match is not None: + name = route_match.group("name").replace("/", ".") + facts.add(GraphFact("ember-template", artifact.path, "defines", name, artifact.path, 1, + (("templateKind", "route"),))) + facts.add(GraphFact("ember-template-association", name, "renders-route", name, artifact.path, 1, + (("ownerKind", "route"),))) + + for pattern, syntax in ((_ANGLE_COMPONENT, "angle"), (_CLASSIC_COMPONENT, "classic"), + (_COMPONENT_HELPER, "component-helper")): + for match in pattern.finditer(content): + facts.add(GraphFact( + "ember-template-component", artifact.path, "invokes", match.group("name"), + artifact.path, _line(artifact.content, match.start()), (("syntax", syntax),), + )) + return facts + + +def _mentions(message: str, value: str) -> bool: + normalized = value.casefold() + variants = {normalized} + for separator in ("::", "/", ".", "#"): + variants.add(normalized.rsplit(separator, 1)[-1]) + for identifier in sorted(variants, key=len, reverse=True): + if len(identifier) < 3: + continue + offset = message.find(identifier) + while offset >= 0: + before = message[offset - 1] if offset else "" + end = offset + len(identifier) + after = message[end] if end < len(message) else "" + if ( + (not before or _TOKEN_CHARACTER.fullmatch(before) is None) + and (not after or _TOKEN_CHARACTER.fullmatch(after) is None) + ): + return True + offset = message.find(identifier, offset + 1) + return False + + +def _fact_identifiers(value: str) -> frozenset[str]: + normalized = value.casefold().strip() + if not normalized: + return frozenset() + identifiers = {normalized} + for separator in ("::", "/", ".", "#"): + identifiers.add(normalized.rsplit(separator, 1)[-1]) + return frozenset(identifier for identifier in identifiers if len(identifier) >= 3) + + +def _identifier_pattern(identifier: str) -> str: + prefix = r"(? bool: + if fact.relation == "uses-conventional-template": + return False + labels = _RELATION_LABELS.get(fact.kind, ()) + label_pattern = "|".join(re.escape(label) for label in labels) + optional_label = rf"(?:\s+(?:{label_pattern}))?" if labels else "" + states = (*_COMMON_RELATION_STATES, *_RELATION_STATES.get(fact.kind, ())) + state_pattern = "|".join(re.escape(state) for state in states) + identifiers = _fact_identifiers(fact.source) | _fact_identifiers(fact.target) + for identifier in identifiers: + identifier_pattern = _identifier_pattern(identifier) + if re.search( + rf"{identifier_pattern}{optional_label}\s+(?:{state_pattern}){_ABSENCE_END}", + message, + ): + return True + if labels and re.search( + rf"(? EmberPlugin: + return EmberPlugin(descriptor) diff --git a/analysis-plugins/frameworks/express/java/pom.xml b/analysis-plugins/frameworks/express/java/pom.xml new file mode 100644 index 00000000..c0a785c3 --- /dev/null +++ b/analysis-plugins/frameworks/express/java/pom.xml @@ -0,0 +1,15 @@ + + + 4.0.0 + + org.rostilos.codecrow + codecrow-framework-plugins + 1.0 + ../../pom.xml + + codecrow-plugin-express + jar + express + diff --git a/analysis-plugins/frameworks/express/java/src/main/java/org/rostilos/codecrow/plugins/express/ExpressPlugin.java b/analysis-plugins/frameworks/express/java/src/main/java/org/rostilos/codecrow/plugins/express/ExpressPlugin.java new file mode 100644 index 00000000..58a37f16 --- /dev/null +++ b/analysis-plugins/frameworks/express/java/src/main/java/org/rostilos/codecrow/plugins/express/ExpressPlugin.java @@ -0,0 +1,23 @@ +package org.rostilos.codecrow.plugins.express; + +import org.rostilos.codecrow.plugins.CodeCrowPlugin; +import org.rostilos.codecrow.plugins.PluginDescriptor; +import org.rostilos.codecrow.plugins.PluginManifestLoader; + +public final class ExpressPlugin implements CodeCrowPlugin { + private final PluginDescriptor descriptor; + + public ExpressPlugin() { + try (var input = ExpressPlugin.class.getResourceAsStream( + "/META-INF/codecrow/plugins/express/plugin.json")) { + descriptor = new PluginManifestLoader().loadDescriptor(input); + } catch (Exception exception) { + throw new IllegalStateException("cannot load Express plugin descriptor", exception); + } + } + + @Override + public PluginDescriptor descriptor() { + return descriptor; + } +} diff --git a/analysis-plugins/frameworks/express/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin b/analysis-plugins/frameworks/express/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin new file mode 100644 index 00000000..0eb8e2f9 --- /dev/null +++ b/analysis-plugins/frameworks/express/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin @@ -0,0 +1 @@ +org.rostilos.codecrow.plugins.express.ExpressPlugin diff --git a/analysis-plugins/frameworks/express/java/src/test/java/org/rostilos/codecrow/plugins/express/ExpressPluginTest.java b/analysis-plugins/frameworks/express/java/src/test/java/org/rostilos/codecrow/plugins/express/ExpressPluginTest.java new file mode 100644 index 00000000..ef98dbb8 --- /dev/null +++ b/analysis-plugins/frameworks/express/java/src/test/java/org/rostilos/codecrow/plugins/express/ExpressPluginTest.java @@ -0,0 +1,20 @@ +package org.rostilos.codecrow.plugins.express; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.plugins.PluginCapability; +import org.rostilos.codecrow.plugins.PluginKind; + +import static org.assertj.core.api.Assertions.assertThat; + +class ExpressPluginTest { + @Test + void packagedManifestLoadsThroughTheJavaContract() { + var descriptor = new ExpressPlugin().descriptor(); + assertThat(descriptor.id()).isEqualTo("express"); + assertThat(descriptor.kind()).isEqualTo(PluginKind.FRAMEWORK); + assertThat(descriptor.requires()).containsExactly("json"); + assertThat(descriptor.capabilities()).contains( + PluginCapability.GRAPH, PluginCapability.INDEX, PluginCapability.VALIDATION); + assertThat(descriptor.detection().alternatives()).hasSize(1); + } +} diff --git a/analysis-plugins/frameworks/express/plugin.json b/analysis-plugins/frameworks/express/plugin.json new file mode 100644 index 00000000..758019dc --- /dev/null +++ b/analysis-plugins/frameworks/express/plugin.json @@ -0,0 +1,25 @@ +{ + "id": "express", + "kind": "framework", + "requires": ["json"], + "capabilities": ["context", "graph", "index", "planning", "prompt", "validation"], + "detection": { + "extensions": [], + "filesAll": [], + "filesAny": [], + "contentMarkers": [], + "alternatives": [ + { + "filesAll": [], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [{"path": "package.json", "contains": "\"express\""}] + } + ] + }, + "entrypoints": { + "java": "org.rostilos.codecrow.plugins.express.ExpressPlugin", + "python": "codecrow_plugin_express:create_plugin" + } +} diff --git a/analysis-plugins/frameworks/express/python/codecrow_plugin_express/__init__.py b/analysis-plugins/frameworks/express/python/codecrow_plugin_express/__init__.py new file mode 100644 index 00000000..fc44fa97 --- /dev/null +++ b/analysis-plugins/frameworks/express/python/codecrow_plugin_express/__init__.py @@ -0,0 +1,684 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from codecrow_plugins import ( + CandidateClaim, + EvidenceRequest, + FileArtifact, + GraphFact, + PluginDescriptor, + PluginDiagnostic, + PluginOutcome, + ReviewContribution, + TreeSitterDocument, + ValidationDecision, + ValidationResult, +) + + +_EXTENSIONS = (".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx") +_GRAMMARS = { + ".cjs": ("tree_sitter_javascript", "language"), + ".cts": ("tree_sitter_typescript", "language_typescript"), + ".js": ("tree_sitter_javascript", "language"), + ".jsx": ("tree_sitter_javascript", "language"), + ".mjs": ("tree_sitter_javascript", "language"), + ".mts": ("tree_sitter_typescript", "language_typescript"), + ".ts": ("tree_sitter_typescript", "language_typescript"), + ".tsx": ("tree_sitter_typescript", "language_tsx"), +} +_HTTP_METHODS = { + "all": "ANY", + "delete": "DELETE", + "get": "GET", + "head": "HEAD", + "options": "OPTIONS", + "patch": "PATCH", + "post": "POST", + "put": "PUT", +} +_IMPORT_DEFAULT = re.compile( + r"\bimport\s+(?P[A-Za-z_$][\w$]*)\s*(?:,\s*{[^}]*})?\s*from\s*(['\"])express\2" +) +_IMPORT_NAMED = re.compile( + r"\bimport\s+(?:[A-Za-z_$][\w$]*\s*,\s*)?" + r"{(?P[^}]*)}\s*from\s*(['\"])express\2", + re.DOTALL, +) +_IMPORT_NAMESPACE = re.compile( + r"\bimport\s*\*\s*as\s*(?P[A-Za-z_$][\w$]*)\s*from\s*(['\"])express\2" +) +_IDENTIFIER = re.compile(r"^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$") +_TOKEN_CHARACTER = re.compile(r"[A-Za-z0-9_$]") +_FACT_KINDS = frozenset({ + "express-application", "express-error-handler", "express-middleware", + "express-mount", "express-route", "express-route-handler", "express-router", +}) +_RELATION_LABELS = { + "express-application": ("application", "express application"), + "express-error-handler": ("error handler",), + "express-middleware": ("middleware",), + "express-mount": ("mount", "router mount"), + "express-route": ("route",), + "express-route-handler": ("handler", "route handler"), + "express-router": ("router",), +} +_RELATION_ACTIONS = { + "declares": ("does not declare", "doesn't declare"), + "handled-by": ("is not handled by",), + "handles": ("does not handle", "doesn't handle"), + "mounts": ("does not mount", "doesn't mount"), + "uses": ("does not use", "doesn't use", "does not register", "doesn't register"), + "uses-error-handler": ( + "does not use", "doesn't use", "does not register", "doesn't register", + ), +} +_RELATION_STATES = { + "express-error-handler": ("is not registered", "is not used"), + "express-middleware": ("is not registered", "is not used"), + "express-mount": ("is not mounted",), + "express-route": ("is not handled", "is not registered"), + "express-route-handler": ("is not handled", "is not registered"), +} +_COMMON_RELATION_STATES = ( + "does not exist", "doesn't exist", "is absent", "is missing", + "is not declared", "is not defined", +) +_ABSENCE_END = r"(?=$|[.!?,;:])" + + +def _parse(artifact: FileArtifact) -> TreeSitterDocument: + extension = PurePosixPath(artifact.path.casefold()).suffix + grammar_module, grammar_factory = _GRAMMARS[extension] + return TreeSitterDocument.parse(artifact.content, grammar_module, grammar_factory) + + +def _literal(document: TreeSitterDocument, node) -> str: + if node is None or node.type not in {"string", "template_string"}: + return "" + text = document.text(node) + if len(text) < 2 or text[0] not in "'\"`" or text[-1] != text[0]: + return "" + if node.type == "template_string" and "${" in text: + return "" + return text[1:-1] + + +def _arguments(node) -> tuple[object, ...]: + arguments = node.child_by_field_name("arguments") + return tuple(arguments.named_children) if arguments is not None else () + + +def _call_target(document: TreeSitterDocument, node) -> str: + return document.text(node.child_by_field_name("function")).replace(" ", "") + + +def _member_call(document: TreeSitterDocument, node) -> tuple[str, str] | None: + function = node.child_by_field_name("function") + if function is None or function.type not in {"member_expression", "subscript_expression"}: + return None + if function.type == "subscript_expression": + return None + owner = document.text(function.child_by_field_name("object")).replace(" ", "") + operation = document.text(function.child_by_field_name("property")).casefold() + return (owner, operation) if owner and operation else None + + +def _binding_aliases(bindings: str, imported_name: str) -> set[str]: + aliases: set[str] = set() + for item in bindings.split(","): + parts = re.split(r"\s+as\s+|\s*:\s*", item.strip()) + if parts and parts[0].strip() == imported_name: + alias = parts[-1].strip() + if re.fullmatch(r"[A-Za-z_$][\w$]*", alias): + aliases.add(alias) + return aliases + + +def _function_arity(node) -> int | None: + if node is None or node.type not in { + "arrow_function", "function", "function_declaration", "function_expression", + "generator_function", "generator_function_declaration", + }: + return None + parameters = node.child_by_field_name("parameters") + if parameters is not None: + return len(parameters.named_children) + parameter = node.child_by_field_name("parameter") + return 1 if parameter is not None else 0 + + +def _handler_name(document: TreeSitterDocument, node) -> str: + text = document.text(node).strip() + if _IDENTIFIER.fullmatch(text): + return text + if node.type in {"call_expression", "new_expression"} and len(text) <= 120: + return text + return f"inline@{document.line(node)}" + + +def _route_chain(document: TreeSitterDocument, owner_node) -> tuple[str, str] | None: + if owner_node is None or owner_node.type != "call_expression": + return None + member = _member_call(document, owner_node) + if member is None: + return None + if member[1] == "route": + arguments = _arguments(owner_node) + path = _literal(document, arguments[0]) if arguments else "" + return (member[0], path) if path else None + if member[1] not in _HTTP_METHODS: + return None + function = owner_node.child_by_field_name("function") + return _route_chain(document, function.child_by_field_name("object")) + + +def _declared_functions(document: TreeSitterDocument) -> dict[str, int]: + arities: dict[str, int] = {} + for node in document.walk(): + if node.type in {"function_declaration", "generator_function_declaration"}: + name = document.text(node.child_by_field_name("name")) + arity = _function_arity(node) + if name and arity is not None: + arities[name] = arity + elif node.type == "variable_declarator": + name = document.text(node.child_by_field_name("name")) + value = node.child_by_field_name("value") + arity = _function_arity(value) + if name and arity is not None: + arities[name] = arity + return arities + + +def _arity(document: TreeSitterDocument, node, declared: dict[str, int]) -> int | None: + direct = _function_arity(node) + if direct is not None: + return direct + return declared.get(document.text(node).strip()) + + +@dataclass(frozen=True) +class _LexicalBinding: + name: str + scope_start: int + scope_end: int + declaration_start: int | None + + +def _binding_names(document: TreeSitterDocument, node) -> set[str]: + if node is None: + return set() + if node.type in {"identifier", "shorthand_property_identifier_pattern"}: + return {document.text(node)} + names: set[str] = set() + for child in node.named_children: + names.update(_binding_names(document, child)) + return names + + +def _binding_scope(document: TreeSitterDocument, node): + current = node.parent + while current is not None: + if current.type in {"program", "statement_block"}: + return current + current = current.parent + return document.root + + +def _scope_declarations( + document: TreeSitterDocument, + scope, + name: str, +) -> set[int]: + declarations: set[int] = set() + for statement in scope.named_children: + candidates = statement.named_children if statement.type == "export_statement" else (statement,) + for candidate in candidates: + if candidate.type in {"lexical_declaration", "variable_declaration"}: + for variable in candidate.named_children: + if ( + variable.type == "variable_declarator" + and name in _binding_names(document, variable.child_by_field_name("name")) + ): + declarations.add(variable.start_byte) + elif candidate.type in { + "class_declaration", "function_declaration", "generator_function_declaration", + }: + if document.text(candidate.child_by_field_name("name")) == name: + declarations.add(candidate.start_byte) + return declarations + + +def _binding_visible( + document: TreeSitterDocument, + use, + binding: _LexicalBinding, +) -> bool: + if not (binding.scope_start <= use.start_byte < binding.scope_end): + return False + if binding.declaration_start is not None and use.start_byte <= binding.declaration_start: + return False + ancestor = use.parent + reached_scope = False + while ancestor is not None: + if ancestor.type in { + "arrow_function", "function_declaration", "function_expression", + "generator_function", "generator_function_declaration", "method_definition", + }: + parameters = ancestor.child_by_field_name("parameters") + parameter = ancestor.child_by_field_name("parameter") + if binding.name in _binding_names(document, parameters or parameter): + return False + if ancestor.type in {"program", "statement_block"}: + declarations = _scope_declarations(document, ancestor, binding.name) + declarations.discard(binding.declaration_start) + if declarations: + return False + if ancestor.start_byte == binding.scope_start and ancestor.end_byte == binding.scope_end: + reached_scope = True + break + ancestor = ancestor.parent + return reached_scope + + +def _visible_provider( + document: TreeSitterDocument, + use, + name: str, + bindings: dict[str, list[_LexicalBinding]], +) -> bool: + return any( + _binding_visible(document, use, binding) + for binding in bindings.get(name, ()) + ) + + +def _owner_rebound( + document: TreeSitterDocument, + binding: _LexicalBinding, +) -> bool: + for node in document.walk(): + if node.start_byte <= (binding.declaration_start or -1): + continue + if node.type not in {"assignment_expression", "augmented_assignment_expression"}: + continue + left = node.child_by_field_name("left") + if binding.name not in _binding_names(document, left): + continue + if _binding_visible(document, node, binding): + return True + return False + + +def _framework_declarations( + document: TreeSitterDocument, + artifact: FileArtifact, +) -> tuple[set[str], set[str], dict[str, _LexicalBinding], set[GraphFact]]: + import_statements = tuple( + document.text(node) for node in document.walk() if node.type == "import_statement" + ) + factory_names = { + match.group("name") + for statement in import_statements + for pattern in (_IMPORT_DEFAULT, _IMPORT_NAMESPACE) + for match in pattern.finditer(statement) + } + router_factory_names: set[str] = set() + for statement in import_statements: + for match in _IMPORT_NAMED.finditer(statement): + router_factory_names.update(_binding_aliases(match.group("bindings"), "Router")) + + factories: dict[str, list[_LexicalBinding]] = { + name: [_LexicalBinding(name, document.root.start_byte, document.root.end_byte, None)] + for name in factory_names + } + router_factories: dict[str, list[_LexicalBinding]] = { + name: [_LexicalBinding(name, document.root.start_byte, document.root.end_byte, None)] + for name in router_factory_names + } + + for node in document.walk(): + if node.type != "variable_declarator": + continue + name_node = node.child_by_field_name("name") + name = document.text(name_node) + value = node.child_by_field_name("value") + if not name or value is None or value.type != "call_expression": + continue + if _call_target(document, value) == "require": + arguments = _arguments(value) + if arguments and _literal(document, arguments[0]) == "express": + implicit_require = _LexicalBinding( + "require", document.root.start_byte, document.root.end_byte, None, + ) + if not _binding_visible(document, value, implicit_require): + continue + scope = _binding_scope(document, node) + if name_node.type == "identifier": + factories.setdefault(name, []).append(_LexicalBinding( + name, scope.start_byte, scope.end_byte, node.start_byte, + )) + elif name_node.type == "object_pattern": + for alias in _binding_aliases(name.strip("{}"), "Router"): + router_factories.setdefault(alias, []).append(_LexicalBinding( + alias, scope.start_byte, scope.end_byte, node.start_byte, + )) + + candidates: list[tuple[str, str, object, _LexicalBinding]] = [] + facts: set[GraphFact] = set() + for node in document.walk(): + if node.type != "variable_declarator": + continue + name = document.text(node.child_by_field_name("name")) + value = node.child_by_field_name("value") + if not name or value is None or value.type != "call_expression": + continue + target = _call_target(document, value) + kind = "" + provider = target + provider_bindings = factories + if _visible_provider(document, value, target, factories): + kind = "application" + elif _visible_provider(document, value, target, router_factories): + kind = "router" + provider_bindings = router_factories + elif target.endswith(".Router"): + provider = target.removesuffix(".Router") + if _visible_provider(document, value, provider, factories): + kind = "router" + if not kind: + continue + name_node = node.child_by_field_name("name") + if name_node is None or name_node.type != "identifier": + continue + scope = _binding_scope(document, node) + binding = _LexicalBinding(name, scope.start_byte, scope.end_byte, node.start_byte) + candidates.append((name, kind, node, binding)) + + counts: dict[str, int] = {} + for name, _, _, _ in candidates: + counts[name] = counts.get(name, 0) + 1 + applications: set[str] = set() + routers: set[str] = set() + owner_bindings: dict[str, _LexicalBinding] = {} + for name, kind, node, binding in candidates: + if counts[name] != 1 or _owner_rebound(document, binding): + continue + owner_bindings[name] = binding + if kind == "application": + applications.add(name) + fact_kind = "express-application" + else: + routers.add(name) + fact_kind = "express-router" + facts.add(GraphFact( + fact_kind, artifact.path, "declares", name, + artifact.path, document.line(node), + )) + return applications, routers, owner_bindings, facts + + +def _route_facts( + document: TreeSitterDocument, + artifact: FileArtifact, + applications: set[str], + routers: set[str], + owner_bindings: dict[str, _LexicalBinding], +) -> set[GraphFact]: + facts: set[GraphFact] = set() + declared_functions = _declared_functions(document) + owners = applications | routers + for node in document.walk(): + if node.type != "call_expression": + continue + member = _member_call(document, node) + if member is None: + continue + owner, operation = member + arguments = _arguments(node) + path = "" + handlers: tuple[object, ...] = () + actual_owner = owner + chained = _route_chain(document, node.child_by_field_name("function").child_by_field_name("object")) + if operation in _HTTP_METHODS and chained is not None: + actual_owner, path = chained + handlers = arguments + elif operation in _HTTP_METHODS and owner in owners: + path = _literal(document, arguments[0]) if arguments else "" + handlers = arguments[1:] if path else () + owner_binding = owner_bindings.get(actual_owner) + owner_visible = ( + owner_binding is not None and _binding_visible(document, node, owner_binding) + ) + if operation in _HTTP_METHODS and actual_owner in owners and owner_visible and path and handlers: + endpoint = f"{_HTTP_METHODS[operation]} {path}" + facts.add(GraphFact( + "express-route", actual_owner, "handles", endpoint, + artifact.path, document.line(node), + (("ownerKind", "application" if actual_owner in applications else "router"),), + )) + for position, handler in enumerate(handlers, start=1): + handler_name = _handler_name(document, handler) + facts.add(GraphFact( + "express-route-handler", endpoint, "handled-by", handler_name, + artifact.path, document.line(handler), (("position", str(position)),), + )) + continue + + owner_binding = owner_bindings.get(owner) + if ( + operation != "use" + or owner not in owners + or owner_binding is None + or not _binding_visible(document, node, owner_binding) + or not arguments + ): + continue + mount_path = _literal(document, arguments[0]) + middleware = arguments[1:] if mount_path else arguments + mount_path = mount_path or "/" + for position, handler in enumerate(middleware, start=1): + handler_name = _handler_name(document, handler) + handler_arity = _arity(document, handler, declared_functions) + router_binding = owner_bindings.get(handler_name) + if ( + handler_name in routers + and router_binding is not None + and _binding_visible(document, handler, router_binding) + ): + facts.add(GraphFact( + "express-mount", owner, "mounts", handler_name, + artifact.path, document.line(node), + tuple(sorted((("mountPath", mount_path), ("position", str(position))))), + )) + if handler_arity == 4: + facts.add(GraphFact( + "express-error-handler", owner, "uses-error-handler", handler_name, + artifact.path, document.line(handler), (("mountPath", mount_path),), + )) + else: + facts.add(GraphFact( + "express-middleware", owner, "uses", handler_name, + artifact.path, document.line(handler), + tuple(sorted((("mountPath", mount_path), ("position", str(position))))), + )) + return facts + + +def _message_mentions(message: str, value: str) -> bool: + normalized = value.casefold() + variants = {normalized} + for separator in ("::", "/", ".", "#"): + variants.add(normalized.rsplit(separator, 1)[-1]) + for identifier in sorted(variants, key=len, reverse=True): + if len(identifier) < 3: + continue + offset = message.find(identifier) + while offset >= 0: + before = message[offset - 1] if offset else "" + end = offset + len(identifier) + after = message[end] if end < len(message) else "" + if ( + (not before or _TOKEN_CHARACTER.fullmatch(before) is None) + and (not after or _TOKEN_CHARACTER.fullmatch(after) is None) + ): + return True + offset = message.find(identifier, offset + 1) + return False + + +def _fact_identifiers(value: str) -> frozenset[str]: + normalized = value.casefold().strip() + if not normalized: + return frozenset() + identifiers = {normalized} + if " " in normalized: + identifiers.add(normalized.split(" ", 1)[-1]) + for separator in ("::", "/", ".", "#"): + identifiers.add(normalized.rsplit(separator, 1)[-1]) + return frozenset(identifier for identifier in identifiers if len(identifier) >= 3) + + +def _identifier_pattern(identifier: str) -> str: + prefix = r"(? bool: + labels = _RELATION_LABELS.get(fact.kind, ()) + label_pattern = "|".join(re.escape(label) for label in labels) + optional_label = rf"(?:\s+(?:{label_pattern}))?" if labels else "" + states = (*_COMMON_RELATION_STATES, *_RELATION_STATES.get(fact.kind, ())) + state_pattern = "|".join(re.escape(state) for state in states) + identifiers = _fact_identifiers(fact.source) | _fact_identifiers(fact.target) + for identifier in identifiers: + identifier_pattern = _identifier_pattern(identifier) + if re.search( + rf"{identifier_pattern}{optional_label}\s+(?:{state_pattern}){_ABSENCE_END}", + message, + ): + return True + if labels and re.search( + rf"(? ExpressPlugin: + return ExpressPlugin(descriptor) diff --git a/analysis-plugins/frameworks/nextjs/java/pom.xml b/analysis-plugins/frameworks/nextjs/java/pom.xml new file mode 100644 index 00000000..fcdbe94a --- /dev/null +++ b/analysis-plugins/frameworks/nextjs/java/pom.xml @@ -0,0 +1,15 @@ + + + 4.0.0 + + org.rostilos.codecrow + codecrow-framework-plugins + 1.0 + ../../pom.xml + + codecrow-plugin-nextjs + jar + nextjs + diff --git a/analysis-plugins/frameworks/nextjs/java/src/main/java/org/rostilos/codecrow/plugins/nextjs/NextJsPlugin.java b/analysis-plugins/frameworks/nextjs/java/src/main/java/org/rostilos/codecrow/plugins/nextjs/NextJsPlugin.java new file mode 100644 index 00000000..79a907e6 --- /dev/null +++ b/analysis-plugins/frameworks/nextjs/java/src/main/java/org/rostilos/codecrow/plugins/nextjs/NextJsPlugin.java @@ -0,0 +1,23 @@ +package org.rostilos.codecrow.plugins.nextjs; + +import org.rostilos.codecrow.plugins.CodeCrowPlugin; +import org.rostilos.codecrow.plugins.PluginDescriptor; +import org.rostilos.codecrow.plugins.PluginManifestLoader; + +public final class NextJsPlugin implements CodeCrowPlugin { + private final PluginDescriptor descriptor; + + public NextJsPlugin() { + try (var input = NextJsPlugin.class.getResourceAsStream( + "/META-INF/codecrow/plugins/nextjs/plugin.json")) { + descriptor = new PluginManifestLoader().loadDescriptor(input); + } catch (Exception exception) { + throw new IllegalStateException("cannot load Next.js plugin descriptor", exception); + } + } + + @Override + public PluginDescriptor descriptor() { + return descriptor; + } +} diff --git a/analysis-plugins/frameworks/nextjs/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin b/analysis-plugins/frameworks/nextjs/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin new file mode 100644 index 00000000..906bf10f --- /dev/null +++ b/analysis-plugins/frameworks/nextjs/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin @@ -0,0 +1 @@ +org.rostilos.codecrow.plugins.nextjs.NextJsPlugin diff --git a/analysis-plugins/frameworks/nextjs/java/src/test/java/org/rostilos/codecrow/plugins/nextjs/NextJsPluginTest.java b/analysis-plugins/frameworks/nextjs/java/src/test/java/org/rostilos/codecrow/plugins/nextjs/NextJsPluginTest.java new file mode 100644 index 00000000..fa0efcfd --- /dev/null +++ b/analysis-plugins/frameworks/nextjs/java/src/test/java/org/rostilos/codecrow/plugins/nextjs/NextJsPluginTest.java @@ -0,0 +1,20 @@ +package org.rostilos.codecrow.plugins.nextjs; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.plugins.PluginCapability; +import org.rostilos.codecrow.plugins.PluginKind; + +import static org.assertj.core.api.Assertions.assertThat; + +class NextJsPluginTest { + @Test + void packagedManifestLoadsThroughTheJavaContract() { + var descriptor = new NextJsPlugin().descriptor(); + assertThat(descriptor.id()).isEqualTo("nextjs"); + assertThat(descriptor.kind()).isEqualTo(PluginKind.FRAMEWORK); + assertThat(descriptor.requires()).containsExactly("json"); + assertThat(descriptor.capabilities()).contains( + PluginCapability.GRAPH, PluginCapability.INDEX, PluginCapability.VALIDATION); + assertThat(descriptor.detection().alternatives()).hasSize(1); + } +} diff --git a/analysis-plugins/frameworks/nextjs/plugin.json b/analysis-plugins/frameworks/nextjs/plugin.json new file mode 100644 index 00000000..4f935a8b --- /dev/null +++ b/analysis-plugins/frameworks/nextjs/plugin.json @@ -0,0 +1,25 @@ +{ + "id": "nextjs", + "kind": "framework", + "requires": ["json"], + "capabilities": ["context", "graph", "index", "planning", "prompt", "validation"], + "detection": { + "extensions": [], + "filesAll": [], + "filesAny": [], + "contentMarkers": [], + "alternatives": [ + { + "filesAll": [], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [{"path": "package.json", "contains": "\"next\""}] + } + ] + }, + "entrypoints": { + "java": "org.rostilos.codecrow.plugins.nextjs.NextJsPlugin", + "python": "codecrow_plugin_nextjs:create_plugin" + } +} diff --git a/analysis-plugins/frameworks/nextjs/python/codecrow_plugin_nextjs/__init__.py b/analysis-plugins/frameworks/nextjs/python/codecrow_plugin_nextjs/__init__.py new file mode 100644 index 00000000..d11855d3 --- /dev/null +++ b/analysis-plugins/frameworks/nextjs/python/codecrow_plugin_nextjs/__init__.py @@ -0,0 +1,787 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +from codecrow_plugins import ( + CandidateClaim, + EvidenceRequest, + FileArtifact, + GraphFact, + PluginDescriptor, + PluginDiagnostic, + PluginOutcome, + ReviewContribution, + TreeSitterDocument, + ValidationDecision, + ValidationResult, +) + + +_EXTENSIONS = (".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx") +_GRAMMARS = { + ".cjs": ("tree_sitter_javascript", "language"), + ".cts": ("tree_sitter_typescript", "language_typescript"), + ".js": ("tree_sitter_javascript", "language"), + ".jsx": ("tree_sitter_javascript", "language"), + ".mjs": ("tree_sitter_javascript", "language"), + ".mts": ("tree_sitter_typescript", "language_typescript"), + ".ts": ("tree_sitter_typescript", "language_typescript"), + ".tsx": ("tree_sitter_typescript", "language_tsx"), +} +_EXTENSION_PATTERN = r"(?:cjs|cts|js|jsx|mjs|mts|ts|tsx)" +_PAGES_PATH = re.compile( + rf"^(?:src/)?pages/(?P.+)\.(?P{_EXTENSION_PATTERN})$", + re.IGNORECASE, +) +_APP_PATH = re.compile( + rf"^(?:src/)?app/(?P.+)\.(?P{_EXTENSION_PATTERN})$", + re.IGNORECASE, +) +_MIDDLEWARE_PATH = re.compile( + rf"^(?:src/)?middleware\.(?:{_EXTENSION_PATTERN})$", re.IGNORECASE +) +_HTTP_METHODS = {"DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"} +_DATA_LOADERS = { + "generateMetadata", + "generateStaticParams", + "getServerSideProps", + "getStaticPaths", + "getStaticProps", +} +_TOKEN_CHARACTER = re.compile(r"[A-Za-z0-9_$]") +_INTERCEPT_PREFIX = re.compile(r"^(?:\((?:\.|\.\.|\.\.\.)\))+") +_FACT_KINDS = frozenset({ + "nextjs-api-route", "nextjs-client-boundary", "nextjs-data-loader", + "nextjs-layout", "nextjs-middleware", "nextjs-page-route", + "nextjs-route-endpoint", "nextjs-route-handler", "nextjs-server-action", + "nextjs-server-boundary", +}) +_RELATION_LABELS = { + "nextjs-api-route": ("api route", "route"), + "nextjs-client-boundary": ("boundary", "client boundary"), + "nextjs-data-loader": ("data loader", "loader"), + "nextjs-layout": ("layout",), + "nextjs-middleware": ("middleware",), + "nextjs-page-route": ("page", "page route", "route"), + "nextjs-route-endpoint": ("endpoint", "route", "route endpoint"), + "nextjs-route-handler": ("handler", "route handler"), + "nextjs-server-action": ("action", "server action"), + "nextjs-server-boundary": ("boundary", "server boundary"), +} +_RELATION_ACTIONS = { + "declares": ("does not declare", "doesn't declare"), + "defines": ("does not define", "doesn't define"), + "handles": ("does not handle", "doesn't handle"), + "intercepts": ("does not intercept", "doesn't intercept"), + "uses": ("does not use", "doesn't use", "does not load", "doesn't load"), + "wraps": ("does not wrap", "doesn't wrap"), +} +_RELATION_STATES = { + "nextjs-data-loader": ("is not loaded", "is not used"), + "nextjs-layout": ("is not wrapped",), + "nextjs-middleware": ("is not intercepted",), + "nextjs-route-handler": ("is not handled",), +} +_COMMON_RELATION_STATES = ( + "does not exist", "doesn't exist", "is absent", "is missing", + "is not declared", "is not defined", +) +_ABSENCE_END = r"(?=$|[.!?,;:])" + + +@dataclass(frozen=True) +class _RouteFile: + kind: str + route: str + router: str + role: str + + +def _parse(artifact: FileArtifact) -> TreeSitterDocument: + extension = PurePosixPath(artifact.path.casefold()).suffix + grammar_module, grammar_factory = _GRAMMARS[extension] + return TreeSitterDocument.parse(artifact.content, grammar_module, grammar_factory) + + +def _literal(document: TreeSitterDocument, node) -> str: + if node is None or node.type not in {"string", "template_string"}: + return "" + text = document.text(node) + if len(text) < 2 or text[0] not in "'\"`" or text[-1] != text[0]: + return "" + if node.type == "template_string" and "${" in text: + return "" + return text[1:-1] + + +def _route(segments: list[str]) -> str: + visible: list[str] = [] + for segment in segments: + if not segment or (segment.startswith("(") and segment.endswith(")")) or segment.startswith("@"): + continue + segment = _INTERCEPT_PREFIX.sub("", segment) + if segment: + visible.append(segment) + return "/" + "/".join(visible) + + +def _route_file(path: str) -> _RouteFile | None: + pages_match = _PAGES_PATH.search(path) + if pages_match is not None: + parts = pages_match.group("tail").split("/") + leaf = parts[-1] + if leaf in {"_app", "_document", "_error", "_middleware"}: + return None + if leaf == "index": + parts.pop() + route = _route(parts) + kind = "nextjs-api-route" if parts and parts[0] == "api" else "nextjs-page-route" + return _RouteFile(kind, route, "pages", "api" if kind == "nextjs-api-route" else "page") + + app_match = _APP_PATH.search(path) + if app_match is None: + return None + parts = app_match.group("tail").split("/") + leaf = parts.pop() + if any(part.startswith("_") for part in parts): + return None + route = _route(parts) + if leaf == "page": + return _RouteFile("nextjs-page-route", route, "app", "page") + if leaf == "route": + kind = "nextjs-api-route" if route == "/api" or route.startswith("/api/") else "nextjs-route-endpoint" + return _RouteFile(kind, route, "app", "route") + if leaf == "layout": + return _RouteFile("nextjs-layout", route, "app", "layout") + return None + + +_FUNCTION_TYPES = frozenset({ + "arrow_function", + "function_declaration", + "function_expression", + "generator_function", + "generator_function_declaration", +}) + + +def _runtime_definitions(document: TreeSitterDocument) -> dict[str, object]: + definitions: dict[str, object] = {} + for top_level in document.root.named_children: + declarations = top_level.named_children if top_level.type == "export_statement" else (top_level,) + for declaration in declarations: + if declaration.type in { + "class_declaration", "function_declaration", "generator_function_declaration", + }: + name = document.text(declaration.child_by_field_name("name")) + if name: + definitions[name] = declaration + elif declaration.type in {"lexical_declaration", "variable_declaration"}: + for variable in declaration.named_children: + if variable.type != "variable_declarator": + continue + name = document.text(variable.child_by_field_name("name")) + value = variable.child_by_field_name("value") + if name and value is not None: + definitions[name] = value + return definitions + + +def _resolved_function( + document: TreeSitterDocument, + node, + definitions: dict[str, object], + seen: frozenset[str] = frozenset(), +): + if node is None: + return None + if node.type in _FUNCTION_TYPES: + return node + if node.type not in {"identifier", "property_identifier"}: + return None + name = document.text(node) + if not name or name in seen: + return None + return _resolved_function(document, definitions.get(name), definitions, seen | {name}) + + +def _exported_bindings(document: TreeSitterDocument) -> tuple[dict[str, int], bool]: + definitions = _runtime_definitions(document) + bindings: dict[str, int] = {} + has_default = False + for node in document.root.named_children: + if node.type != "export_statement": + continue + text = document.text(node) + if re.match(r"export\s+type\b", text): + continue + is_default = bool(re.match(r"export\s+default\b", text)) + if is_default: + candidate = node.child_by_field_name("declaration") or node.child_by_field_name("value") + if candidate is None: + candidate = next(iter(node.named_children), None) + if candidate is not None and ( + candidate.type not in { + "abstract_class_declaration", "interface_declaration", "type_alias_declaration", + } + and ( + candidate.type not in {"identifier", "property_identifier"} + or document.text(candidate) in definitions + ) + ): + has_default = True + for child in node.named_children: + if child.type in {"function_declaration", "generator_function_declaration"}: + name = document.text(child.child_by_field_name("name")) + if name: + bindings[name] = document.line(child) + elif child.type in {"lexical_declaration", "variable_declaration"}: + for declaration in child.named_children: + if declaration.type != "variable_declarator": + continue + name = document.text(declaration.child_by_field_name("name")) + if name: + bindings[name] = document.line(declaration) + elif child.type == "export_clause": + for specifier in child.named_children: + if specifier.type != "export_specifier": + continue + if document.text(specifier).lstrip().startswith("type "): + continue + local = document.text(specifier.child_by_field_name("name")) + exported = ( + document.text(specifier.child_by_field_name("alias")) + or local + ) + if local in definitions and exported: + bindings[exported] = document.line(specifier) + if exported == "default": + has_default = True + return bindings, has_default + + +def _exported_function_bindings(document: TreeSitterDocument) -> dict[str, int]: + definitions = _runtime_definitions(document) + bindings: dict[str, int] = {} + for node in document.root.named_children: + if node.type != "export_statement": + continue + text = document.text(node) + if re.match(r"export\s+type\b", text): + continue + for child in node.named_children: + if child.type in {"function_declaration", "generator_function_declaration"}: + name = document.text(child.child_by_field_name("name")) + if name: + bindings[name] = document.line(child) + elif child.type in {"lexical_declaration", "variable_declaration"}: + for variable in child.named_children: + if variable.type != "variable_declarator": + continue + name = document.text(variable.child_by_field_name("name")) + if name and _resolved_function( + document, variable.child_by_field_name("value"), definitions, + ) is not None: + bindings[name] = document.line(variable) + elif child.type == "export_clause": + for specifier in child.named_children: + if specifier.type != "export_specifier": + continue + if document.text(specifier).lstrip().startswith("type "): + continue + local_node = specifier.child_by_field_name("name") + local = document.text(local_node) + exported = document.text(specifier.child_by_field_name("alias")) or local + if _resolved_function( + document, definitions.get(local), definitions, + ) is not None: + bindings[exported] = document.line(specifier) + return bindings + + +def _module_directives(document: TreeSitterDocument) -> tuple[tuple[str, int], ...]: + directives: list[tuple[str, int]] = [] + for node in document.root.named_children: + if node.type == "comment": + continue + if node.type != "expression_statement": + break + named = tuple(node.named_children) + value = _literal(document, named[0]) if named else "" + if not value: + break + if value in {"use client", "use server"}: + directives.append((value, document.line(node))) + return tuple(directives) + + +def _function_name(document: TreeSitterDocument, node) -> str: + name = document.text(node.child_by_field_name("name")) + if name: + return name + current = node.parent + while current is not None: + if current.type in {"function_declaration", "generator_function_declaration"}: + return document.text(current.child_by_field_name("name")) or f"inline@{document.line(current)}" + if current.type == "variable_declarator": + return document.text(current.child_by_field_name("name")) or f"inline@{document.line(current)}" + current = current.parent + return "" + + +def _server_actions(document: TreeSitterDocument, artifact: FileArtifact) -> set[GraphFact]: + facts: set[GraphFact] = set() + function_types = { + "arrow_function", + "function_declaration", + "function_expression", + "generator_function", + "generator_function_declaration", + "method_definition", + } + for node in document.walk(): + if node.type not in function_types: + continue + body = node.child_by_field_name("body") + if body is None or body.type != "statement_block": + continue + directive = None + for statement in body.named_children: + if statement.type == "comment": + continue + named = tuple(statement.named_children) + value = _literal(document, named[0]) if statement.type == "expression_statement" and named else "" + if not value: + break + if value == "use server": + directive = statement + if directive is None: + continue + name = _function_name(document, node) + if name: + facts.add(GraphFact( + "nextjs-server-action", artifact.path, "declares", name, + artifact.path, document.line(directive), (("scope", "function"),), + )) + return facts + + +def _page_api_handlers(document: TreeSitterDocument) -> tuple[object, ...]: + function_types = { + "arrow_function", + "function_declaration", + "function_expression", + "generator_function", + "generator_function_declaration", + } + definitions: dict[str, object] = {} + for node in document.root.named_children: + declarations = node.named_children if node.type == "export_statement" else (node,) + for declaration in declarations: + if declaration.type in {"function_declaration", "generator_function_declaration"}: + name = document.text(declaration.child_by_field_name("name")) + if name: + definitions[name] = declaration + elif declaration.type in {"lexical_declaration", "variable_declaration"}: + for variable in declaration.named_children: + if variable.type != "variable_declarator": + continue + name = document.text(variable.child_by_field_name("name")) + value = variable.child_by_field_name("value") + if name and value is not None and value.type in function_types: + definitions[name] = value + + handlers: list[object] = [] + for node in document.root.named_children: + if node.type == "export_statement" and re.match(r"export\s+default\b", document.text(node)): + value = node.child_by_field_name("value") + direct = next((child for child in node.named_children if child.type in function_types), None) + candidate = direct or value + if candidate is not None and candidate.type in function_types: + handlers.append(candidate) + elif candidate is not None: + resolved = definitions.get(document.text(candidate)) + if resolved is not None: + handlers.append(resolved) + if node.type != "expression_statement": + continue + assignment = next( + (child for child in node.named_children if child.type == "assignment_expression"), None + ) + if assignment is None: + continue + left = document.text(assignment.child_by_field_name("left")).replace(" ", "") + if left not in {"exports.default", "module.exports"}: + continue + right = assignment.child_by_field_name("right") + if right is not None and right.type in function_types: + handlers.append(right) + elif right is not None: + resolved = definitions.get(document.text(right)) + if resolved is not None: + handlers.append(resolved) + return tuple(handlers) + + +def _page_api_methods( + document: TreeSitterDocument, + handlers: tuple[object, ...], +) -> dict[str, int]: + methods: dict[str, int] = {} + request_methods = {"req.method", "request.method"} + nested_function_types = { + "arrow_function", + "function_declaration", + "function_expression", + "generator_function", + "generator_function_declaration", + } + for handler in handlers: + for node in document.walk(handler): + ancestor = node.parent + nested = False + while ancestor is not None and ancestor != handler: + if ancestor.type in nested_function_types: + nested = True + break + ancestor = ancestor.parent + if nested: + continue + if node.type == "binary_expression": + operator = document.text(node.child_by_field_name("operator")) + if operator not in {"==", "==="}: + continue + left = node.child_by_field_name("left") + right = node.child_by_field_name("right") + left_text = document.text(left).replace(" ", "") + right_text = document.text(right).replace(" ", "") + method = "" + if left_text in request_methods: + method = _literal(document, right).upper() + elif right_text in request_methods: + method = _literal(document, left).upper() + if method in _HTTP_METHODS: + methods.setdefault(method, document.line(node)) + elif node.type == "switch_case": + parent = node.parent + while parent is not None and parent.type != "switch_statement" and parent != handler: + parent = parent.parent + if parent is None or parent == handler: + continue + switch_value = document.text(parent.child_by_field_name("value")).strip("() ").replace(" ", "") + method = _literal(document, node.child_by_field_name("value")).upper() + if switch_value in request_methods and method in _HTTP_METHODS: + methods.setdefault(method, document.line(node)) + return methods + + +def _boundary_facts( + document: TreeSitterDocument, + artifact: FileArtifact, + route_file: _RouteFile | None, +) -> set[GraphFact]: + facts: set[GraphFact] = set() + directives = _module_directives(document) + module_values = {value for value, _ in directives} + for value, line in directives: + boundary = "client" if value == "use client" else "server" + facts.add(GraphFact( + f"nextjs-{boundary}-boundary", artifact.path, "declares", boundary, + artifact.path, line, (("scope", "module"),), + )) + if ( + route_file is not None + and route_file.router == "app" + and route_file.role in {"layout", "page"} + and "use client" not in module_values + and "use server" not in module_values + ): + facts.add(GraphFact( + "nextjs-server-boundary", artifact.path, "declares", "server", + artifact.path, 1, (("scope", "app-router-default"),), + )) + + facts.update(_server_actions(document, artifact)) + return facts + + +def _middleware_matchers(document: TreeSitterDocument) -> tuple[str, ...]: + exported, _ = _exported_bindings(document) + if "config" not in exported: + return ("*",) + config_value = None + for top_level in document.root.named_children: + declarations = top_level.named_children if top_level.type == "export_statement" else (top_level,) + for declaration in declarations: + if declaration.type not in {"lexical_declaration", "variable_declaration"}: + continue + for variable in declaration.named_children: + if ( + variable.type == "variable_declarator" + and document.text(variable.child_by_field_name("name")) == "config" + ): + config_value = variable.child_by_field_name("value") + break + if config_value is None or config_value.type != "object": + return () + value = config_value + for pair in value.named_children: + if pair.type != "pair": + continue + key = document.text(pair.child_by_field_name("key")).strip("'\"") + if key != "matcher": + continue + matcher_value = pair.child_by_field_name("value") + direct = _literal(document, matcher_value) + if direct: + return (direct,) + if matcher_value is not None and matcher_value.type == "array": + values = tuple(sorted({ + literal + for child in matcher_value.named_children + if (literal := _literal(document, child)) + })) + return values + return () + return ("*",) + + +def _framework_facts(document: TreeSitterDocument, artifact: FileArtifact) -> set[GraphFact]: + facts: set[GraphFact] = set() + route_file = _route_file(artifact.path) + bindings, has_runtime_default = _exported_bindings(document) + function_bindings = _exported_function_bindings(document) + pages_api_handlers: tuple[object, ...] = () + route_methods: dict[str, int] = {} + if route_file is not None and route_file.kind == "nextjs-api-route" and route_file.router == "pages": + pages_api_handlers = _page_api_handlers(document) + if not pages_api_handlers: + route_file = None + elif route_file is not None and route_file.role == "route": + route_methods = { + method: function_bindings[method] + for method in sorted(_HTTP_METHODS & set(function_bindings)) + } + if not route_methods: + route_file = None + elif route_file is not None and route_file.role in {"layout", "page"}: + if not has_runtime_default: + route_file = None + + if route_file is not None: + relation = "wraps" if route_file.role == "layout" else "defines" + facts.add(GraphFact( + route_file.kind, artifact.path, relation, route_file.route, + artifact.path, 1, + tuple(sorted((("role", route_file.role), ("router", route_file.router)))), + )) + if route_file.kind == "nextjs-api-route" and route_file.router == "pages": + methods = _page_api_methods(document, pages_api_handlers) + if not methods: + methods = {"ANY": document.line(pages_api_handlers[0])} + for method, line in sorted(methods.items()): + facts.add(GraphFact( + "nextjs-route-handler", route_file.route, "handles", f"{method} {route_file.route}", + artifact.path, line, (("router", "pages"),), + )) + elif route_file.role == "route": + for method, line in route_methods.items(): + facts.add(GraphFact( + "nextjs-route-handler", route_file.route, "handles", f"{method} {route_file.route}", + artifact.path, line, (("router", "app"),), + )) + + for loader in sorted(_DATA_LOADERS & set(function_bindings)): + facts.add(GraphFact( + "nextjs-data-loader", route_file.route, "uses", loader, + artifact.path, function_bindings[loader], (("router", route_file.router),), + )) + + if _MIDDLEWARE_PATH.search(artifact.path): + default_handlers = _page_api_handlers(document) + if "middleware" in function_bindings or default_handlers: + for matcher in _middleware_matchers(document): + facts.add(GraphFact( + "nextjs-middleware", artifact.path, "intercepts", matcher, + artifact.path, 1, + )) + + facts.update(_boundary_facts(document, artifact, route_file)) + return facts + + +def _message_mentions(message: str, value: str) -> bool: + normalized = value.casefold() + variants = {normalized} + if " " in normalized: + variants.add(normalized.split(" ", 1)[-1]) + for separator in ("::", "/", ".", "#"): + variants.add(normalized.rsplit(separator, 1)[-1]) + for identifier in sorted(variants, key=len, reverse=True): + if len(identifier) < 3: + continue + offset = message.find(identifier) + while offset >= 0: + before = message[offset - 1] if offset else "" + end = offset + len(identifier) + after = message[end] if end < len(message) else "" + if ( + (not before or _TOKEN_CHARACTER.fullmatch(before) is None) + and (not after or _TOKEN_CHARACTER.fullmatch(after) is None) + ): + return True + offset = message.find(identifier, offset + 1) + return False + + +def _fact_identifiers(value: str) -> frozenset[str]: + normalized = value.casefold().strip() + if not normalized: + return frozenset() + identifiers = {normalized} + if " " in normalized: + identifiers.add(normalized.split(" ", 1)[-1]) + for separator in ("::", "/", ".", "#"): + identifiers.add(normalized.rsplit(separator, 1)[-1]) + return frozenset(identifier for identifier in identifiers if len(identifier) >= 3) + + +def _identifier_pattern(identifier: str) -> str: + prefix = r"(? bool: + labels = _RELATION_LABELS.get(fact.kind, ()) + label_pattern = "|".join(re.escape(label) for label in labels) + optional_label = rf"(?:\s+(?:{label_pattern}))?" if labels else "" + states = (*_COMMON_RELATION_STATES, *_RELATION_STATES.get(fact.kind, ())) + state_pattern = "|".join(re.escape(state) for state in states) + identifiers = _fact_identifiers(fact.source) | _fact_identifiers(fact.target) + for identifier in identifiers: + identifier_pattern = _identifier_pattern(identifier) + if re.search( + rf"{identifier_pattern}{optional_label}\s+(?:{state_pattern}){_ABSENCE_END}", + message, + ): + return True + if labels and re.search( + rf"(? NextJsPlugin: + return NextJsPlugin(descriptor) diff --git a/analysis-plugins/frameworks/pom.xml b/analysis-plugins/frameworks/pom.xml index 6d46e5f0..c193bcae 100644 --- a/analysis-plugins/frameworks/pom.xml +++ b/analysis-plugins/frameworks/pom.xml @@ -39,9 +39,15 @@ + django/java + ember/java + express/java fastapi/java hyva/java magento/java + nextjs/java + quarkus/java + rails/java spring/java diff --git a/analysis-plugins/frameworks/quarkus/java/pom.xml b/analysis-plugins/frameworks/quarkus/java/pom.xml new file mode 100644 index 00000000..1850d67c --- /dev/null +++ b/analysis-plugins/frameworks/quarkus/java/pom.xml @@ -0,0 +1,15 @@ + + + 4.0.0 + + org.rostilos.codecrow + codecrow-framework-plugins + 1.0 + ../../pom.xml + + codecrow-plugin-quarkus + jar + quarkus + diff --git a/analysis-plugins/frameworks/quarkus/java/src/main/java/org/rostilos/codecrow/plugins/quarkus/QuarkusPlugin.java b/analysis-plugins/frameworks/quarkus/java/src/main/java/org/rostilos/codecrow/plugins/quarkus/QuarkusPlugin.java new file mode 100644 index 00000000..ab9d8e10 --- /dev/null +++ b/analysis-plugins/frameworks/quarkus/java/src/main/java/org/rostilos/codecrow/plugins/quarkus/QuarkusPlugin.java @@ -0,0 +1,25 @@ +package org.rostilos.codecrow.plugins.quarkus; + +import org.rostilos.codecrow.plugins.CodeCrowPlugin; +import org.rostilos.codecrow.plugins.PluginDescriptor; +import org.rostilos.codecrow.plugins.PluginManifestLoader; + +public final class QuarkusPlugin implements CodeCrowPlugin { + private final PluginDescriptor descriptor; + + public QuarkusPlugin() { + try (var input = QuarkusPlugin.class.getResourceAsStream( + "/META-INF/codecrow/plugins/quarkus/plugin.json")) { + descriptor = new PluginManifestLoader().loadDescriptor(input); + } catch (Exception exception) { + throw new IllegalStateException( + "cannot load Quarkus plugin descriptor", + exception); + } + } + + @Override + public PluginDescriptor descriptor() { + return descriptor; + } +} diff --git a/analysis-plugins/frameworks/quarkus/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin b/analysis-plugins/frameworks/quarkus/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin new file mode 100644 index 00000000..8e6cb010 --- /dev/null +++ b/analysis-plugins/frameworks/quarkus/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin @@ -0,0 +1 @@ +org.rostilos.codecrow.plugins.quarkus.QuarkusPlugin diff --git a/analysis-plugins/frameworks/quarkus/java/src/test/java/org/rostilos/codecrow/plugins/quarkus/QuarkusPluginTest.java b/analysis-plugins/frameworks/quarkus/java/src/test/java/org/rostilos/codecrow/plugins/quarkus/QuarkusPluginTest.java new file mode 100644 index 00000000..55be56fa --- /dev/null +++ b/analysis-plugins/frameworks/quarkus/java/src/test/java/org/rostilos/codecrow/plugins/quarkus/QuarkusPluginTest.java @@ -0,0 +1,23 @@ +package org.rostilos.codecrow.plugins.quarkus; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.plugins.PluginCapability; +import org.rostilos.codecrow.plugins.PluginKind; + +import static org.assertj.core.api.Assertions.assertThat; + +class QuarkusPluginTest { + @Test + void packagedManifestLoadsThroughTheJavaContract() { + var descriptor = new QuarkusPlugin().descriptor(); + + assertThat(descriptor.id()).isEqualTo("quarkus"); + assertThat(descriptor.kind()).isEqualTo(PluginKind.FRAMEWORK); + assertThat(descriptor.requires()).containsExactly("java"); + assertThat(descriptor.capabilities()).contains( + PluginCapability.GRAPH, + PluginCapability.INDEX, + PluginCapability.VALIDATION); + assertThat(descriptor.detection().alternatives()).hasSize(4); + } +} diff --git a/analysis-plugins/frameworks/quarkus/plugin.json b/analysis-plugins/frameworks/quarkus/plugin.json new file mode 100644 index 00000000..a06c7adc --- /dev/null +++ b/analysis-plugins/frameworks/quarkus/plugin.json @@ -0,0 +1,50 @@ +{ + "id": "quarkus", + "kind": "framework", + "requires": ["java"], + "capabilities": ["context", "graph", "index", "planning", "prompt", "validation"], + "detection": { + "extensions": [], + "filesAll": [], + "filesAny": [], + "contentMarkers": [], + "alternatives": [ + { + "filesAll": [], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [{"path": "build.gradle", "contains": "io.quarkus"}], + "contentPatternMarkers": [] + }, + { + "filesAll": [], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [{"path": "build.gradle.kts", "contains": "io.quarkus"}], + "contentPatternMarkers": [] + }, + { + "filesAll": [], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [{"path": "pom.xml", "contains": "io.quarkus"}], + "contentPatternMarkers": [] + }, + { + "filesAll": [], + "filesAny": ["build.gradle", "build.gradle.kts", "pom.xml"], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [], + "contentPatternMarkers": [{"pathPattern": "**/*.java", "contains": "io.quarkus."}] + } + ] + }, + "entrypoints": { + "java": "org.rostilos.codecrow.plugins.quarkus.QuarkusPlugin", + "python": "codecrow_plugin_quarkus:create_plugin" + } +} diff --git a/analysis-plugins/frameworks/quarkus/python/codecrow_plugin_quarkus/__init__.py b/analysis-plugins/frameworks/quarkus/python/codecrow_plugin_quarkus/__init__.py new file mode 100644 index 00000000..448c7793 --- /dev/null +++ b/analysis-plugins/frameworks/quarkus/python/codecrow_plugin_quarkus/__init__.py @@ -0,0 +1,957 @@ +from __future__ import annotations + +import json +import re +from dataclasses import dataclass + +from codecrow_plugins import ( + CandidateClaim, + EvidenceRequest, + FileArtifact, + GraphFact, + PluginDescriptor, + PluginDiagnostic, + PluginOutcome, + ReviewContribution, + TreeSitterDocument, + ValidationDecision, + ValidationResult, +) + + +_TYPE_NODES = { + "class_declaration", + "enum_declaration", + "interface_declaration", + "record_declaration", +} +_LOCAL_TYPE_NODES = _TYPE_NODES | {"annotation_type_declaration"} +_ANNOTATION_NODES = {"annotation", "marker_annotation"} +_MAX_FACTS_PER_FILE = 160 +_MAX_CONFIG_KEYS = 128 +_MAX_REVIEW_PATHS = 64 +_PROPERTY = re.compile( + r"^[ \t]*(?P%?[A-Za-z0-9_][A-Za-z0-9_.%-]*)" + r"[ \t]*(?:=|:)[ \t]*" +) + +_CDI_SCOPES = { + "jakarta.enterprise.context.ApplicationScoped": "ApplicationScoped", + "jakarta.enterprise.context.ConversationScoped": "ConversationScoped", + "jakarta.enterprise.context.Dependent": "Dependent", + "jakarta.enterprise.context.RequestScoped": "RequestScoped", + "jakarta.enterprise.context.SessionScoped": "SessionScoped", + "jakarta.inject.Singleton": "Singleton", + "javax.enterprise.context.ApplicationScoped": "ApplicationScoped", + "javax.enterprise.context.ConversationScoped": "ConversationScoped", + "javax.enterprise.context.Dependent": "Dependent", + "javax.enterprise.context.RequestScoped": "RequestScoped", + "javax.enterprise.context.SessionScoped": "SessionScoped", + "javax.inject.Singleton": "Singleton", +} +_INJECT = frozenset({"jakarta.inject.Inject", "javax.inject.Inject"}) +_JAXRS_PATH = frozenset({"jakarta.ws.rs.Path", "javax.ws.rs.Path"}) +_JAXRS_METHODS = { + "jakarta.ws.rs.DELETE": "DELETE", + "jakarta.ws.rs.GET": "GET", + "jakarta.ws.rs.HEAD": "HEAD", + "jakarta.ws.rs.OPTIONS": "OPTIONS", + "jakarta.ws.rs.PATCH": "PATCH", + "jakarta.ws.rs.POST": "POST", + "jakarta.ws.rs.PUT": "PUT", + "javax.ws.rs.DELETE": "DELETE", + "javax.ws.rs.GET": "GET", + "javax.ws.rs.HEAD": "HEAD", + "javax.ws.rs.OPTIONS": "OPTIONS", + "javax.ws.rs.PATCH": "PATCH", + "javax.ws.rs.POST": "POST", + "javax.ws.rs.PUT": "PUT", +} +_CONFIG_PROPERTY = frozenset({ + "org.eclipse.microprofile.config.inject.ConfigProperty", +}) +_SCHEDULED = frozenset({"io.quarkus.scheduler.Scheduled"}) +_INCOMING = frozenset({ + "org.eclipse.microprofile.reactive.messaging.Incoming", +}) +_OUTGOING = frozenset({ + "org.eclipse.microprofile.reactive.messaging.Outgoing", +}) +_PANACHE_ENTITY_BASES = frozenset({ + "io.quarkus.hibernate.orm.panache.PanacheEntity", + "io.quarkus.hibernate.orm.panache.PanacheEntityBase", + "io.quarkus.hibernate.reactive.panache.PanacheEntity", + "io.quarkus.hibernate.reactive.panache.PanacheEntityBase", + "io.quarkus.mongodb.panache.PanacheMongoEntity", + "io.quarkus.mongodb.panache.PanacheMongoEntityBase", + "io.quarkus.mongodb.panache.reactive.ReactivePanacheMongoEntity", + "io.quarkus.mongodb.panache.reactive.ReactivePanacheMongoEntityBase", +}) +_PANACHE_REPOSITORY_BASES = frozenset({ + "io.quarkus.hibernate.orm.panache.PanacheRepository", + "io.quarkus.hibernate.orm.panache.PanacheRepositoryBase", + "io.quarkus.hibernate.reactive.panache.PanacheRepository", + "io.quarkus.hibernate.reactive.panache.PanacheRepositoryBase", + "io.quarkus.mongodb.panache.PanacheMongoRepository", + "io.quarkus.mongodb.panache.PanacheMongoRepositoryBase", + "io.quarkus.mongodb.panache.reactive.ReactivePanacheMongoRepository", + "io.quarkus.mongodb.panache.reactive.ReactivePanacheMongoRepositoryBase", +}) +_KNOWN_ANNOTATIONS = frozenset({ + *_CDI_SCOPES, + *_INJECT, + *_JAXRS_PATH, + *_JAXRS_METHODS, + *_CONFIG_PROPERTY, + *_SCHEDULED, + *_INCOMING, + *_OUTGOING, +}) +_KNOWN_PANACHE_TYPES = _PANACHE_ENTITY_BASES | _PANACHE_REPOSITORY_BASES +_FACT_KINDS = frozenset({ + "quarkus-cdi-bean", + "quarkus-cdi-injection", + "quarkus-config-key", + "quarkus-config-property", + "quarkus-jaxrs-resource", + "quarkus-jaxrs-route", + "quarkus-panache-entity", + "quarkus-panache-repository", + "quarkus-reactive-channel", + "quarkus-scheduled-method", +}) +_RELATION_LABELS = { + "quarkus-cdi-bean": ("bean", "cdi bean", "cdi scope", "scope"), + "quarkus-cdi-injection": ("dependency", "injection"), + "quarkus-config-key": ("config key", "key"), + "quarkus-config-property": ("config property", "property"), + "quarkus-jaxrs-resource": ("resource",), + "quarkus-jaxrs-route": ("route",), + "quarkus-panache-entity": ("entity",), + "quarkus-panache-repository": ("repository",), + "quarkus-reactive-channel": ("channel",), + "quarkus-scheduled-method": ("method", "schedule"), +} +_RELATION_STATES = { + "quarkus-cdi-bean": ("is not a bean", "is not scoped"), + "quarkus-cdi-injection": ("is not injected",), + "quarkus-config-key": ("is not configured",), + "quarkus-config-property": ("is not configured", "is not injected"), + "quarkus-jaxrs-resource": ("is not registered",), + "quarkus-jaxrs-route": ("is not registered",), + "quarkus-panache-entity": ("is not an entity",), + "quarkus-panache-repository": ("is not registered",), + "quarkus-reactive-channel": ("is not consumed", "is not produced"), + "quarkus-scheduled-method": ("is not scheduled",), +} +_COMMON_RELATION_STATES = ( + "does not exist", + "doesn't exist", + "is absent", + "is missing", + "is not defined", +) +_ABSENCE_END = ( + r"(?=$|[.!?,;:]|\s+(?:and|because|despite|even|for|from|in|into|on|" + r"so|therefore|when|while|with|without)\b)" +) + + +def _named(node, field: str): + return node.child_by_field_name(field) + + +def _is_application_properties(path: str) -> bool: + return path.rsplit("/", 1)[-1].casefold() == "application.properties" + + +def _package_and_imports(document: TreeSitterDocument): + package = "" + imports: dict[str, str] = {} + for node in document.root.named_children: + if node.type == "package_declaration": + package = ( + document.text(node) + .removeprefix("package ") + .rstrip(";") + .strip() + ) + elif node.type == "import_declaration": + value = ( + document.text(node) + .removeprefix("import ") + .rstrip(";") + .strip() + ) + if value.startswith("static "): + continue + if value.endswith(".*"): + continue + simple_name = value.rsplit(".", 1)[-1] + previous = imports.get(simple_name) + imports[simple_name] = value if previous in {None, value} else "" + return package, imports + + +def _local_type_names(document: TreeSitterDocument) -> frozenset[str]: + return frozenset( + name + for node in document.walk() + if node.type in _LOCAL_TYPE_NODES + and (name := document.text(_named(node, "name"))) + ) + + +def _resolve_known_name( + value: str, + imports: dict[str, str], + local_type_names: frozenset[str], + known: frozenset[str], +) -> str: + normalized = value.strip() + if normalized in known: + return normalized + if normalized in local_type_names: + return "" + imported = imports.get(normalized) + if imported in known: + return imported + return "" + + +def _annotations( + document: TreeSitterDocument, + node, + imports: dict[str, str], + local_type_names: frozenset[str], +) -> tuple[tuple[str, object], ...]: + modifiers = next( + (child for child in node.named_children if child.type == "modifiers"), + None, + ) + if modifiers is None: + return () + result = [] + for annotation in modifiers.named_children: + if annotation.type not in _ANNOTATION_NODES: + continue + name = document.text(_named(annotation, "name")) + qualified = _resolve_known_name( + name, + imports, + local_type_names, + _KNOWN_ANNOTATIONS, + ) + if qualified: + result.append((qualified, annotation)) + return tuple(result) + + +def _string_literal(document: TreeSitterDocument, node) -> str | None: + if node is None or node.type != "string_literal": + return None + try: + value = json.loads(document.text(node)) + except (TypeError, ValueError, json.JSONDecodeError): + return None + return value if isinstance(value, str) else None + + +def _literal_argument( + document: TreeSitterDocument, + annotation, + key: str = "value", +) -> str | None: + arguments = _named(annotation, "arguments") + if arguments is None: + return None + for child in arguments.named_children: + if child.type != "element_value_pair": + if key == "value": + literal = _string_literal(document, child) + if literal is not None: + return literal + continue + if document.text(_named(child, "key")) != key: + continue + return _string_literal(document, _named(child, "value")) + return None + + +def _raw_arguments( + document: TreeSitterDocument, + annotation, + accepted: frozenset[str], +) -> tuple[tuple[str, str], ...]: + arguments = _named(annotation, "arguments") + if arguments is None: + return () + values: dict[str, str] = {} + for child in arguments.named_children: + if child.type != "element_value_pair": + continue + key = document.text(_named(child, "key")) + if key not in accepted: + continue + value_node = _named(child, "value") + literal = _string_literal(document, value_node) + value = literal if literal is not None else " ".join( + document.text(value_node).split() + ) + if value and len(value) <= 256: + values[key] = value + return tuple(sorted(values.items())) + + +def _type_owner(document: TreeSitterDocument, declaration, package: str) -> str: + names = [] + current = declaration + while current is not None: + if current.type in _TYPE_NODES: + name = document.text(_named(current, "name")) + if name: + names.append(name) + current = current.parent + qualified = ".".join(reversed(names)) + return f"{package}.{qualified}" if package and qualified else qualified + + +def _join_route(prefix: str, suffix: str) -> str: + parts = [part.strip("/") for part in (prefix, suffix) if part.strip("/")] + return "/" + "/".join(parts) if parts else "/" + + +def _field_names(document: TreeSitterDocument, field) -> tuple[str, ...]: + return tuple( + name + for child in field.named_children + if child.type == "variable_declarator" + and (name := document.text(_named(child, "name"))) + ) + + +def _parameters(callable_node) -> tuple[object, ...]: + parameters = _named(callable_node, "parameters") + if parameters is None: + return () + return tuple( + parameter + for parameter in parameters.named_children + if parameter.type in {"formal_parameter", "spread_parameter"} + ) + + +def _declared_type(document: TreeSitterDocument, parameter) -> str: + return document.text(_named(parameter, "type")).strip() + + +def _parent_type_nodes(declaration) -> tuple[object, ...]: + result = [] + for child in declaration.named_children: + if child.type == "superclass": + if child.named_child_count: + result.append(child.named_child(0)) + elif child.type in {"extends_interfaces", "super_interfaces"}: + type_list = next( + (candidate for candidate in child.named_children if candidate.type == "type_list"), + None, + ) + if type_list is not None: + result.extend(type_list.named_children) + return tuple(result) + + +def _parent_base(document: TreeSitterDocument, parent) -> str: + if parent.type != "generic_type": + return document.text(parent).strip() + return document.text(parent.named_child(0)).strip() if parent.named_child_count else "" + + +def _first_type_argument(document: TreeSitterDocument, parent) -> str: + if parent.type != "generic_type": + return "" + arguments = next( + (child for child in parent.named_children if child.type == "type_arguments"), + None, + ) + if arguments is None or not arguments.named_child_count: + return "" + return document.text(arguments.named_child(0)).strip() + + +def _bounded_facts(facts: set[GraphFact]) -> tuple[GraphFact, ...]: + by_kind: dict[str, list[GraphFact]] = {} + for fact in sorted(facts): + by_kind.setdefault(fact.kind, []).append(fact) + selected = [] + offset = 0 + kinds = tuple(sorted(by_kind)) + while len(selected) < _MAX_FACTS_PER_FILE: + added = False + for kind in kinds: + values = by_kind[kind] + if offset < len(values): + selected.append(values[offset]) + added = True + if len(selected) == _MAX_FACTS_PER_FILE: + break + if not added: + break + offset += 1 + return tuple(sorted(selected)) + + +def _config_key_facts(artifact: FileArtifact) -> tuple[GraphFact, ...]: + facts: set[GraphFact] = set() + lines = artifact.content.splitlines() + index = 0 + while index < len(lines) and len(facts) < _MAX_CONFIG_KEYS: + line = lines[index] + start_line = index + 1 + index += 1 + stripped = line.lstrip() + if not stripped or stripped.startswith(("#", "!")): + continue + trailing_slashes = len(line) - len(line.rstrip("\\")) + if trailing_slashes % 2: + while index < len(lines): + continuation = lines[index] + index += 1 + trailing_slashes = len(continuation) - len( + continuation.rstrip("\\") + ) + if trailing_slashes % 2 == 0: + break + continue + match = _PROPERTY.match(line) + if match is None: + continue + key = match.group("key") + attributes = () + if key.startswith("%") and "." in key: + profile = key[1:].split(".", 1)[0] + if profile: + attributes = (("profile", profile),) + facts.add(GraphFact( + "quarkus-config-key", + artifact.path, + "defines", + key, + artifact.path, + start_line, + attributes, + )) + return tuple(sorted(facts)) + + +def _identifier_variants(fact: GraphFact) -> frozenset[str]: + values: set[str] = set() + source = fact.source.casefold().strip() + target = fact.target.casefold().strip() + if source: + values.add(source) + local_source = source.rsplit(".", 1)[-1] + values.add(local_source) + if "#" in source: + owner, member = source.rsplit("#", 1) + values.add(member) + values.add(f"{owner.rsplit('.', 1)[-1]}#{member}") + if target: + values.add(target) + if fact.kind == "quarkus-jaxrs-route" and " " in target: + values.add(target.split(" ", 1)[1]) + elif fact.kind not in { + "quarkus-config-key", + "quarkus-config-property", + "quarkus-reactive-channel", + }: + values.add(target.rsplit(".", 1)[-1]) + ignored = { + "defines", + "delete", + "get", + "head", + "options", + "patch", + "post", + "put", + "read", + "resource", + "route", + "run", + "scheduled", + "value", + "write", + } + return frozenset( + value for value in values + if len(value) >= 2 and value not in ignored + ) + + +def _fact_is_relevant(fact: GraphFact, message: str) -> bool: + for identifier in _identifier_variants(fact): + if identifier.replace("_", "").isalnum(): + if re.search( + rf"(? bool: + labels = _RELATION_LABELS.get(fact.kind, ()) + optional_label = ( + rf"(?:\s+(?:{'|'.join(re.escape(label) for label in labels)}))?" + if labels + else "" + ) + states = (*_COMMON_RELATION_STATES, *_RELATION_STATES.get(fact.kind, ())) + state_pattern = "|".join(re.escape(state) for state in states) + for identifier in _identifier_variants(fact): + boundary_start = ( + r"(? QuarkusPlugin: + return QuarkusPlugin(descriptor) diff --git a/analysis-plugins/frameworks/rails/java/pom.xml b/analysis-plugins/frameworks/rails/java/pom.xml new file mode 100644 index 00000000..a67bffe2 --- /dev/null +++ b/analysis-plugins/frameworks/rails/java/pom.xml @@ -0,0 +1,15 @@ + + + 4.0.0 + + org.rostilos.codecrow + codecrow-framework-plugins + 1.0 + ../../pom.xml + + codecrow-plugin-rails + jar + rails + diff --git a/analysis-plugins/frameworks/rails/java/src/main/java/org/rostilos/codecrow/plugins/rails/RailsPlugin.java b/analysis-plugins/frameworks/rails/java/src/main/java/org/rostilos/codecrow/plugins/rails/RailsPlugin.java new file mode 100644 index 00000000..29cde228 --- /dev/null +++ b/analysis-plugins/frameworks/rails/java/src/main/java/org/rostilos/codecrow/plugins/rails/RailsPlugin.java @@ -0,0 +1,23 @@ +package org.rostilos.codecrow.plugins.rails; + +import org.rostilos.codecrow.plugins.CodeCrowPlugin; +import org.rostilos.codecrow.plugins.PluginDescriptor; +import org.rostilos.codecrow.plugins.PluginManifestLoader; + +public final class RailsPlugin implements CodeCrowPlugin { + private final PluginDescriptor descriptor; + + public RailsPlugin() { + try (var input = RailsPlugin.class.getResourceAsStream( + "/META-INF/codecrow/plugins/rails/plugin.json")) { + descriptor = new PluginManifestLoader().loadDescriptor(input); + } catch (Exception exception) { + throw new IllegalStateException("cannot load Rails plugin descriptor", exception); + } + } + + @Override + public PluginDescriptor descriptor() { + return descriptor; + } +} diff --git a/analysis-plugins/frameworks/rails/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin b/analysis-plugins/frameworks/rails/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin new file mode 100644 index 00000000..1a01fa91 --- /dev/null +++ b/analysis-plugins/frameworks/rails/java/src/main/resources/META-INF/services/org.rostilos.codecrow.plugins.CodeCrowPlugin @@ -0,0 +1 @@ +org.rostilos.codecrow.plugins.rails.RailsPlugin diff --git a/analysis-plugins/frameworks/rails/java/src/test/java/org/rostilos/codecrow/plugins/rails/RailsPluginTest.java b/analysis-plugins/frameworks/rails/java/src/test/java/org/rostilos/codecrow/plugins/rails/RailsPluginTest.java new file mode 100644 index 00000000..17f52dd4 --- /dev/null +++ b/analysis-plugins/frameworks/rails/java/src/test/java/org/rostilos/codecrow/plugins/rails/RailsPluginTest.java @@ -0,0 +1,77 @@ +package org.rostilos.codecrow.plugins.rails; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.plugins.PluginCapability; +import org.rostilos.codecrow.plugins.DetectionRules; +import org.rostilos.codecrow.plugins.PluginDescriptor; +import org.rostilos.codecrow.plugins.PluginKind; +import org.rostilos.codecrow.plugins.PluginRegistry; +import org.rostilos.codecrow.plugins.ProjectSelector; +import org.rostilos.codecrow.plugins.RepositoryFacts; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class RailsPluginTest { + @Test + void packagedManifestLoadsThroughTheJavaContract() { + var descriptor = new RailsPlugin().descriptor(); + assertThat(descriptor.id()).isEqualTo("rails"); + assertThat(descriptor.kind()).isEqualTo(PluginKind.FRAMEWORK); + assertThat(descriptor.requires()).containsExactly("ruby"); + assertThat(descriptor.capabilities()).contains( + PluginCapability.GRAPH, PluginCapability.INDEX, PluginCapability.VALIDATION); + assertThat(descriptor.detection().alternatives()).hasSize(5); + } + + @Test + void engineDetectionKeepsAllEvidenceInsideOneNestedRoot() { + var ruby = new PluginDescriptor( + "ruby", + PluginKind.LANGUAGE, + List.of(), + List.of(), + new DetectionRules( + List.of(".rb"), List.of(), List.of(), List.of(), List.of()), + Map.of()); + var selector = new ProjectSelector(new PluginRegistry(List.of( + ruby, + new RailsPlugin().descriptor()))); + + var split = selector.select(new RepositoryFacts( + "abc1234", + List.of( + "a/example.gemspec", + "b/config/routes.rb", + "c/lib/example/engine.rb"), + Map.of( + "c/lib/example/engine.rb", + "class Example < Rails::Engine\nend\n"))); + var coherent = selector.select(new RepositoryFacts( + "abc1234", + List.of( + "services/blog/blog.gemspec", + "services/blog/config/routes.rb", + "services/blog/lib/blog/engine.rb"), + Map.of( + "services/blog/lib/blog/engine.rb", + "class Blog < Rails::Engine\nend\n"))); + var nestedButNotRootRelative = selector.select(new RepositoryFacts( + "abc1234", + List.of( + "services/blog/config/routes.rb", + "services/blog/nested/blog.gemspec", + "services/blog/vendor/lib/blog/engine.rb"), + Map.of( + "services/blog/vendor/lib/blog/engine.rb", + "class Blog < Rails::Engine\nend\n"))); + + assertThat(split.repositoryPlugins()).containsExactly("ruby"); + assertThat(coherent.repositoryPlugins()).containsExactly("ruby", "rails"); + assertThat(coherent.detectionEvidence().get("rails")) + .contains("root:services/blog"); + assertThat(nestedButNotRootRelative.repositoryPlugins()).containsExactly("ruby"); + } +} diff --git a/analysis-plugins/frameworks/rails/plugin.json b/analysis-plugins/frameworks/rails/plugin.json new file mode 100644 index 00000000..01fb438a --- /dev/null +++ b/analysis-plugins/frameworks/rails/plugin.json @@ -0,0 +1,62 @@ +{ + "id": "rails", + "kind": "framework", + "requires": ["ruby"], + "capabilities": ["context", "graph", "index", "planning", "prompt", "validation"], + "detection": { + "extensions": [], + "filesAll": [], + "filesAny": [], + "contentMarkers": [], + "alternatives": [ + { + "filesAll": ["Gemfile", "config/routes.rb"], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [ + {"path": "Gemfile", "contains": "gem \"rails\""} + ] + }, + { + "filesAll": ["Gemfile", "config/routes.rb"], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [ + {"path": "Gemfile", "contains": "gem 'rails'"} + ] + }, + { + "filesAll": ["Gemfile.lock", "config/routes.rb"], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [ + {"path": "Gemfile.lock", "contains": " rails ("} + ] + }, + { + "filesAll": ["bin/rails", "config/application.rb", "config/routes.rb"], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [] + }, + { + "filesAll": ["config/routes.rb"], + "filesAny": [], + "pathPatternsAll": ["*.gemspec"], + "pathPatternsAny": [], + "contentMarkers": [], + "contentPatternMarkers": [ + {"pathPattern": "lib/**/engine.rb", "contains": "Rails::Engine"} + ] + } + ] + }, + "entrypoints": { + "java": "org.rostilos.codecrow.plugins.rails.RailsPlugin", + "python": "codecrow_plugin_rails:create_plugin" + } +} diff --git a/analysis-plugins/frameworks/rails/python/codecrow_plugin_rails/__init__.py b/analysis-plugins/frameworks/rails/python/codecrow_plugin_rails/__init__.py new file mode 100644 index 00000000..85958645 --- /dev/null +++ b/analysis-plugins/frameworks/rails/python/codecrow_plugin_rails/__init__.py @@ -0,0 +1,774 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass + +from codecrow_plugins import ( + CandidateClaim, + EvidenceRequest, + FileArtifact, + GraphFact, + PluginDescriptor, + PluginDiagnostic, + PluginOutcome, + ReviewContribution, + TreeSitterDocument, + ValidationDecision, + ValidationResult, +) + + +_MAX_FACTS_PER_FILE = 160 +_MAX_EVIDENCE_REQUESTS = 40 +_HTTP_ROUTES = frozenset({"delete", "get", "head", "options", "patch", "post", "put"}) +_RESOURCE_ROUTES = frozenset({"resource", "resources"}) +_ASSOCIATIONS = { + "belongs_to": "belongs-to", + "has_and_belongs_to_many": "has-and-belongs-to-many", + "has_many": "has-many", + "has_one": "has-one", +} +_CALLBACKS = frozenset({ + "after_action", "after_commit", "after_create", "after_destroy", "after_find", + "after_initialize", "after_rollback", "after_save", "after_touch", "after_update", + "after_validation", "around_action", "around_save", "before_action", "before_create", + "before_destroy", "before_save", "before_update", "before_validation", "prepend_after_action", + "prepend_around_action", "prepend_before_action", +}) +_JOB_POLICIES = frozenset({"discard_on", "retry_on"}) +_FACT_KINDS = frozenset({ + "rails-association", + "rails-callback", + "rails-controller", + "rails-controller-action", + "rails-job", + "rails-job-perform", + "rails-job-policy", + "rails-job-queue", + "rails-model", + "rails-mount", + "rails-route", +}) +_RELATION_LABELS = { + "rails-association": ("association", "relation", "relationship"), + "rails-callback": ("callback",), + "rails-controller": ("controller",), + "rails-controller-action": ("action", "controller action"), + "rails-job": ("job",), + "rails-job-perform": ("job", "perform method"), + "rails-job-policy": ("job policy", "policy"), + "rails-job-queue": ("job queue", "queue"), + "rails-model": ("model",), + "rails-mount": ("mount", "mounted application"), + "rails-route": ("route",), +} +_RELATION_STATES = { + "rails-association": ("is not associated", "is not declared"), + "rails-callback": ("is not registered",), + "rails-controller-action": ("is not exposed",), + "rails-job-policy": ("is not configured",), + "rails-job-queue": ("is not configured",), + "rails-mount": ("is not mounted",), + "rails-route": ("is not declared", "is not registered"), +} +_COMMON_RELATION_STATES = ( + "does not exist", + "doesn't exist", + "is absent", + "is missing", + "is not declared", + "is not defined", +) +_ABSENCE_END = ( + r"(?=$|[.!?,;:]|\s+(?:and|because|despite|even|for|from|in|into|on|" + r"so|therefore|when|while|with|without)\b)" +) + + +def _text(document: TreeSitterDocument, node) -> str: + return document.text(node).strip() if node is not None else "" + + +def _field(node, name: str): + return node.child_by_field_name(name) if node is not None else None + + +def _call_name(document: TreeSitterDocument, node) -> str: + return _text(document, _field(node, "method")) if node is not None and node.type == "call" else "" + + +def _is_implicit_or_self_call(document: TreeSitterDocument, node) -> bool: + if node is None or node.type != "call": + return False + receiver = _field(node, "receiver") + return receiver is None or receiver.type == "self" or _text(document, receiver) == "self" + + +def _is_owned_routes_call(document: TreeSitterDocument, node) -> bool: + if node is None or node.type != "call" or _call_name(document, node) != "routes": + return False + owner = _field(node, "receiver") + if owner is None: + return False + if owner.type in {"constant", "scope_resolution"}: + static_owner = "".join(_text(document, owner).split()) + if re.fullmatch( + r"(?:::)?[A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*", + static_owner, + ) is None: + return False + return static_owner.removeprefix("::").rsplit("::", 1)[-1] in { + "Application", "Engine", + } + if owner.type != "call" or _call_name(document, owner) != "application": + return False + arguments = _field(owner, "arguments") + return ( + _text(document, _field(owner, "receiver")) == "Rails" + and (arguments is None or not arguments.named_children) + ) + + +def _inside_routes_draw(document: TreeSitterDocument, node) -> bool: + """Return whether ``node`` belongs to a statically owned Rails route set.""" + ancestor = node.parent + while ancestor is not None: + if ancestor.type == "call" and _call_name(document, ancestor) == "draw": + receiver = _field(ancestor, "receiver") + block = _field(ancestor, "block") + if ( + receiver is not None + and receiver.type == "call" + and _is_owned_routes_call(document, receiver) + and block is not None + and block.start_byte <= node.start_byte + and node.end_byte <= block.end_byte + ): + return True + ancestor = ancestor.parent + return False + + +def _arguments(node): + return _field(node, "arguments") + + +def _literal(document: TreeSitterDocument, node) -> str: + if node is None: + return "" + if node.type == "string": + if any(child.type in {"interpolation", "subshell"} for child in node.named_children): + return "" + return "".join( + document.text(child) + for child in node.named_children + if child.type == "string_content" + ) + if node.type in {"simple_symbol", "hash_key_symbol", "delimited_symbol"}: + value = _text(document, node) + if value.startswith(":"): + value = value[1:] + return value.strip("'\"") + if node.type in { + "constant", "false", "identifier", "integer", "nil", "scope_resolution", "true", + }: + return "".join(_text(document, node).split()) + return "" + + +def _static_route_literal(document: TreeSitterDocument, node) -> str: + if node is None or node.type not in { + "delimited_symbol", "hash_key_symbol", "simple_symbol", "string", + }: + return "" + return _literal(document, node) + + +def _static_route_literal_list( + document: TreeSitterDocument, + node, +) -> tuple[str, ...]: + if node is None: + return () + if node.type == "array": + values = tuple( + _static_route_literal(document, child) + for child in node.named_children + ) + return values if all(values) else () + value = _static_route_literal(document, node) + return (value,) if value else () + + +def _literal_list(document: TreeSitterDocument, node) -> tuple[str, ...]: + if node is None: + return () + if node.type == "array": + return tuple( + value + for child in node.named_children + if (value := _literal(document, child)) + ) + value = _literal(document, node) + return (value,) if value else () + + +def _positional_arguments(document: TreeSitterDocument, call) -> tuple[object, ...]: + arguments = _arguments(call) + if arguments is None: + return () + return tuple(child for child in arguments.named_children if child.type != "pair") + + +def _keyword_nodes(document: TreeSitterDocument, call) -> dict[str, object]: + arguments = _arguments(call) + if arguments is None: + return {} + values: dict[str, object] = {} + for child in arguments.named_children: + if child.type != "pair": + continue + key = _literal(document, _field(child, "key")) + value = _field(child, "value") + if key and value is not None: + values[key] = value + return values + + +def _first_literal(document: TreeSitterDocument, call) -> str: + positional = _positional_arguments(document, call) + return _literal(document, positional[0]) if positional else "" + + +def _first_route_literal(document: TreeSitterDocument, call) -> str: + positional = _positional_arguments(document, call) + return _static_route_literal(document, positional[0]) if positional else "" + + +def _attributes(**values: str) -> tuple[tuple[str, str], ...]: + return tuple(sorted( + (key, value) + for key, value in values.items() + if value + )) + + +def _option_attributes( + document: TreeSitterDocument, + call, + names: tuple[str, ...], +) -> tuple[tuple[str, str], ...]: + keywords = _keyword_nodes(document, call) + values = { + name: ",".join(_literal_list(document, keywords.get(name))) + for name in names + } + return _attributes(**values) + + +def _declaration_name(document: TreeSitterDocument, declaration) -> str: + raw = "".join(_text(document, _field(declaration, "name")).split()) + if not raw: + return "" + prefixes: list[str] = [] + ancestor = declaration.parent + while ancestor is not None: + if ancestor.type in {"class", "module"}: + name = "".join(_text(document, _field(ancestor, "name")).split()) + if name: + prefixes.append(name) + ancestor = ancestor.parent + return "::".join((*reversed(prefixes), raw)) if prefixes else raw + + +def _superclass(document: TreeSitterDocument, declaration) -> str: + value = _text(document, _field(declaration, "superclass")) + return value.removeprefix("<").strip().replace(" ", "") + + +def _body(declaration): + return _field(declaration, "body") + + +def _direct_calls(body) -> tuple[object, ...]: + if body is None: + return () + return tuple(child for child in body.named_children if child.type == "call") + + +def _join_route(*parts: str) -> str: + components = [part.strip("/") for part in parts if part.strip("/")] + return "/" + "/".join(components) if components else "/" + + +def _route_prefix(document: TreeSitterDocument, route_call) -> str | None: + prefixes: list[str] = [] + ancestor = route_call.parent + while ancestor is not None: + if ancestor.type == "call": + operation = _call_name(document, ancestor) + if operation in _RESOURCE_ROUTES: + # Exact member/nested-resource paths require Rails inflection, + # `param`, and `on` semantics. A per-file extractor must not + # turn those dynamic prefixes into a false absolute route. + return None + if operation == "namespace": + if not _is_implicit_or_self_call(document, ancestor): + return None + positional = _positional_arguments(document, ancestor) + if not positional: + return None + value = _static_route_literal(document, positional[0]) + if not value: + return None + prefixes.append(value) + elif operation == "scope": + if not _is_implicit_or_self_call(document, ancestor): + return None + keywords = _keyword_nodes(document, ancestor) + positional = _positional_arguments(document, ancestor) + path_node = positional[0] if positional else keywords.get("path") + value = _static_route_literal(document, path_node) + if path_node is not None and not value: + return None + if value: + prefixes.append(value) + ancestor = ancestor.parent + return _join_route(*reversed(prefixes)) if prefixes else "" + + +def _bounded_facts(facts: set[GraphFact]) -> tuple[GraphFact, ...]: + by_kind: dict[str, list[GraphFact]] = {} + for fact in sorted(facts): + by_kind.setdefault(fact.kind, []).append(fact) + selected: list[GraphFact] = [] + offset = 0 + kinds = tuple(sorted(by_kind)) + while len(selected) < _MAX_FACTS_PER_FILE: + added = False + for kind in kinds: + values = by_kind[kind] + if offset < len(values): + selected.append(values[offset]) + added = True + if len(selected) == _MAX_FACTS_PER_FILE: + break + if not added: + break + offset += 1 + return tuple(sorted(selected)) + + +def _fact_identifiers(fact: GraphFact) -> frozenset[str]: + values = [fact.source, fact.target, *(value for _, value in fact.attributes)] + identifiers: set[str] = set() + for value in values: + normalized = value.casefold().strip() + if not normalized: + continue + identifiers.add(normalized) + separated = normalized + for separator in ("::", "#", "/", ".", ":"): + separated = separated.replace(separator, " ") + identifiers.update(part.strip(" _-()[],'\"") for part in separated.split()) + return frozenset(value for value in identifiers if len(value) >= 3) + + +def _mentions_identifier(message: str, identifier: str) -> bool: + if identifier.replace("_", "").isalnum(): + return re.search( + rf"(? str: + escaped = re.escape(identifier) + if identifier.replace("_", "").isalnum(): + return rf"(? bool: + labels = _RELATION_LABELS.get(fact.kind, ()) + label_pattern = "|".join(re.escape(label) for label in labels) + optional_label = rf"(?:\s+(?:{label_pattern}))?" if labels else "" + states = (*_COMMON_RELATION_STATES, *_RELATION_STATES.get(fact.kind, ())) + state_pattern = "|".join(re.escape(state) for state in states) + named = r"(?:named\s+)?" + for identifier in _fact_identifiers(fact): + identifier_pattern = _identifier_pattern(identifier) + if re.search( + rf"{identifier_pattern}{optional_label}\s+(?:{state_pattern}){_ABSENCE_END}", + message, + ): + return True + if labels and re.search( + rf"(? None: + for call in document.walk(): + if call.type != "call": + continue + operation = _call_name(document, call) + if operation not in {*_HTTP_ROUTES, *_RESOURCE_ROUTES, "match", "mount", "root"}: + continue + if not _is_implicit_or_self_call(document, call): + continue + if not _inside_routes_draw(document, call): + continue + prefix = _route_prefix(document, call) + if prefix is None: + continue + keywords = _keyword_nodes(document, call) + line = document.line(call) + + if operation in _RESOURCE_ROUTES: + resource = _first_route_literal(document, call) + if not resource: + continue + path_node = keywords.get("path") + route_fragment = resource + if path_node is not None: + if path_node.type not in { + "delimited_symbol", "hash_key_symbol", "simple_symbol", "string", + }: + continue + route_fragment = _static_route_literal(document, path_node) + if not route_fragment: + continue + route = _join_route(prefix, route_fragment) + actions = ",".join(_static_route_literal_list(document, keywords.get("only"))) + excluded = ",".join(_static_route_literal_list(document, keywords.get("except"))) + controller = _static_route_literal(document, keywords.get("controller")) + facts.add(GraphFact( + "rails-route", + path, + "declares", + f"{operation.upper()} {route}", + path, + line, + _attributes(actions=actions, controller=controller, excluded_actions=excluded), + )) + continue + + if operation == "root": + controller_action = ( + _first_route_literal(document, call) + or _static_route_literal(document, keywords.get("to")) + ) + facts.add(GraphFact( + "rails-route", path, "handles", f"GET {_join_route(prefix)}", path, line, + _attributes(controller_action=controller_action), + )) + continue + + if operation == "mount": + application = _first_literal(document, call) + mount_path = _static_route_literal(document, keywords.get("at")) + if application and mount_path: + facts.add(GraphFact( + "rails-mount", path, "mounts", application, path, line, + (("path", _join_route(prefix, mount_path)),), + )) + continue + + route_fragment = _first_route_literal(document, call) + if not route_fragment: + continue + route = _join_route(prefix, route_fragment) + controller_action = _static_route_literal(document, keywords.get("to")) + route_name = _static_route_literal(document, keywords.get("as")) + via_node = keywords.get("via") + via = ",".join(_static_route_literal_list(document, via_node)) + if operation == "match" and (via_node is None or not via): + continue + verb = operation.upper() + if operation == "match" and via: + verb = via.upper() + facts.add(GraphFact( + "rails-route", path, "handles", f"{verb} {route}", path, line, + _attributes(controller_action=controller_action, route_name=route_name), + )) + + @staticmethod + def _classes( + document: TreeSitterDocument, + path: str, + facts: set[GraphFact], + ) -> None: + for declaration in document.walk(): + if declaration.type != "class": + continue + owner = _declaration_name(document, declaration) + parent = _superclass(document, declaration) + if not owner or not parent: + continue + body = _body(declaration) + is_model = parent in {"ApplicationRecord", "ActiveRecord::Base"} + is_controller = ( + parent == "ApplicationController" + or parent.startswith("ActionController::") + ) + is_job = parent in {"ApplicationJob", "ActiveJob::Base"} + + if is_model: + facts.add(GraphFact( + "rails-model", path, "declares", owner, path, + document.line(declaration), (("superclass", parent),), + )) + if is_controller: + facts.add(GraphFact( + "rails-controller", path, "declares", owner, path, + document.line(declaration), (("superclass", parent),), + )) + RailsPlugin._controller_actions(document, body, path, owner, facts) + if is_job: + facts.add(GraphFact( + "rails-job", path, "declares", owner, path, + document.line(declaration), (("superclass", parent),), + )) + RailsPlugin._job_perform(document, body, path, owner, facts) + + if not (is_model or is_controller or is_job): + continue + for call in _direct_calls(body): + if not _is_implicit_or_self_call(document, call): + continue + operation = _call_name(document, call) + if is_model and operation in _ASSOCIATIONS: + association = _first_literal(document, call) + if not association: + continue + keywords = _keyword_nodes(document, call) + facts.add(GraphFact( + "rails-association", owner, _ASSOCIATIONS[operation], association, + path, document.line(call), + _attributes( + class_name=_literal(document, keywords.get("class_name")), + dependent=_literal(document, keywords.get("dependent")), + foreign_key=_literal(document, keywords.get("foreign_key")), + inverse_of=_literal(document, keywords.get("inverse_of")), + optional=_literal(document, keywords.get("optional")), + polymorphic=_literal(document, keywords.get("polymorphic")), + source=_literal(document, keywords.get("source")), + through=_literal(document, keywords.get("through")), + ), + )) + if operation in _CALLBACKS: + callback = _first_literal(document, call) + if callback: + callback_options = dict(_option_attributes( + document, call, ("except", "if", "on", "only", "unless"), + )) + facts.add(GraphFact( + "rails-callback", owner, "registers", callback, + path, document.line(call), + _attributes(macro=operation, **callback_options), + )) + if is_job and operation == "queue_as": + queue = _first_literal(document, call) + if queue: + facts.add(GraphFact( + "rails-job-queue", owner, "queues-on", queue, + path, document.line(call), + )) + if is_job and operation in _JOB_POLICIES: + exception = _first_literal(document, call) + if exception: + facts.add(GraphFact( + "rails-job-policy", owner, operation.replace("_", "-"), exception, + path, document.line(call), + _option_attributes(document, call, ("attempts", "jitter", "wait")), + )) + + @staticmethod + def _controller_actions( + document: TreeSitterDocument, + body, + path: str, + owner: str, + facts: set[GraphFact], + ) -> None: + if body is None: + return + named_visibility: dict[str, str] = {} + for member in body.named_children: + if member.type != "call": + continue + if not _is_implicit_or_self_call(document, member): + continue + operation = _call_name(document, member) + if operation not in {"private", "protected", "public"}: + continue + for argument in _positional_arguments(document, member): + name = _literal(document, argument) + if name: + named_visibility[name] = operation + visibility = "public" + for member in body.named_children: + if member.type == "identifier" and _text(document, member) in {"private", "protected", "public"}: + visibility = _text(document, member) + continue + if member.type == "call" and _call_name(document, member) in {"private", "protected", "public"}: + if not _is_implicit_or_self_call(document, member): + continue + if not _positional_arguments(document, member): + visibility = _call_name(document, member) + continue + if member.type != "method": + continue + name = _text(document, _field(member, "name")) + if name and named_visibility.get(name, visibility) == "public": + facts.add(GraphFact( + "rails-controller-action", owner, "exposes", f"{owner}#{name}", + path, document.line(member), + )) + + @staticmethod + def _job_perform( + document: TreeSitterDocument, + body, + path: str, + owner: str, + facts: set[GraphFact], + ) -> None: + if body is None: + return + for member in body.named_children: + if member.type != "method" or _text(document, _field(member, "name")) != "perform": + continue + facts.add(GraphFact( + "rails-job-perform", owner, "executes", f"{owner}#perform", + path, document.line(member), + )) + + def review(self, paths: tuple[str, ...]): + selected = tuple(sorted( + path for path in paths if path.casefold().endswith(".rb") + ))[:_MAX_EVIDENCE_REQUESTS] + if not selected: + return PluginOutcome.abstained() + rules = tuple(sorted(( + "Resolve Rails endpoint behavior through route DSL, controller actions, namespaces, scopes, and callbacks before judging reachability.", + "Treat model associations, callbacks, and Active Job declarations as topology context; their presence alone is not defect proof.", + ))) + return PluginOutcome.handled(ReviewContribution( + rules=rules, + evidence_requests=tuple(EvidenceRequest( + "rails-topology", + path, + "exact Rails route, controller, model-association, callback, and job facts", + ) for path in selected), + )) + + def validate(self, claim: CandidateClaim): + requested_kind = claim.claim_kind or claim.category + if not requested_kind.startswith("rails-"): + return PluginOutcome.abstained() + if not claim.path.casefold().endswith(".rb"): + return PluginOutcome.abstained() + + if requested_kind == "rails-topology": + expected_kinds = _FACT_KINDS + elif requested_kind in _FACT_KINDS: + expected_kinds = frozenset({requested_kind}) + else: + return PluginOutcome.handled(ValidationResult( + ValidationDecision.INSUFFICIENT_EVIDENCE, + "rails-unknown-fact-kind", + "The Rails claim kind is not owned by an exact validator.", + )) + matching = tuple( + fact for fact in claim.evidence + if fact.kind in expected_kinds + and claim.path in {fact.path, *fact.related_paths} + ) + message = claim.message.casefold() + relevant = tuple( + fact for fact in matching + if any( + _mentions_identifier(message, identifier) + for identifier in _fact_identifiers(fact) + ) + ) + if any(_is_absence_claim(fact, message) for fact in relevant): + return PluginOutcome.handled(ValidationResult( + ValidationDecision.REJECT, + "rails-absence-contradicted", + "The candidate claims Rails topology is absent, but an exact matching framework fact exists.", + )) + if relevant: + return PluginOutcome.handled(ValidationResult( + ValidationDecision.INSUFFICIENT_EVIDENCE, + "rails-topology-not-defect-proof", + "The cited Rails relationship exists, but structural presence alone does not prove defective behavior.", + )) + if matching: + return PluginOutcome.handled(ValidationResult( + ValidationDecision.INSUFFICIENT_EVIDENCE, + "rails-cited-identifier-mismatch", + "Rails topology facts exist for the path, but their identifiers do not match the candidate message.", + )) + return PluginOutcome.handled(ValidationResult( + ValidationDecision.INSUFFICIENT_EVIDENCE, + "rails-evidence-unavailable", + "No exact matching Rails topology evidence was supplied for this framework claim.", + )) + + +def create_plugin(descriptor: PluginDescriptor) -> RailsPlugin: + return RailsPlugin(descriptor) diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/VcsClient.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/VcsClient.java index 06040d22..a0bc87ad 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/VcsClient.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/VcsClient.java @@ -155,6 +155,32 @@ default int getRepositoryCount(String workspaceId) throws IOException { */ String getFileContent(String workspaceId, String repoIdOrSlug, String filePath, String branchOrCommit) throws IOException; + /** + * List every regular repository file at a pinned commit snapshot. + * Symbolic links and submodule/subrepository entries are not regular + * files for this inventory. + * + *

The result must be complete. Implementations must throw instead of + * returning a truncated inventory when the provider or {@code maxFiles} + * cannot represent the whole tree. Callers use this only as optional + * deterministic enrichment and can then fall back to narrower evidence.

+ * + * @param workspaceId the external workspace/org ID + * @param repoIdOrSlug the repository ID or slug + * @param commit immutable commit-object hash + * @param maxFiles maximum complete inventory size accepted by the caller + * @return normalized repository-relative regular-file paths + */ + default List listRepositoryFiles( + String workspaceId, + String repoIdOrSlug, + String commit, + int maxFiles + ) throws IOException { + throw new UnsupportedOperationException( + "Complete repository file listing is not supported by this provider"); + } + /** * Get the latest commit hash for a branch. * @param workspaceId the external workspace/org ID diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudClient.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudClient.java index bfc354cf..5f915374 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudClient.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudClient.java @@ -19,13 +19,16 @@ import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Comparator; +import java.util.Deque; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; /** * VcsClient implementation for Bitbucket Cloud. @@ -36,6 +39,9 @@ public class BitbucketCloudClient implements VcsClient { private static final Logger log = LoggerFactory.getLogger(BitbucketCloudClient.class); private static final String API_BASE = "https://api.bitbucket.org/2.0"; private static final int DEFAULT_PAGE_SIZE = 50; + private static final int MAX_TREE_REQUESTS = 10_000; + private static final long MAX_TREE_ENTRIES = 2_000_000L; + private static final long MIN_TREE_ENTRIES = 256L; private static final MediaType JSON_MEDIA_TYPE = MediaType.parse("application/json"); private final OkHttpClient httpClient; @@ -569,6 +575,107 @@ public String getFileContent(String workspaceId, String repoIdOrSlug, String fil } } + @Override + public List listRepositoryFiles( + String workspaceId, + String repoIdOrSlug, + String commit, + int maxFiles + ) throws IOException { + if (maxFiles <= 0) throw new IllegalArgumentException("maxFiles must be positive"); + TreeSet files = new TreeSet<>(); + Deque directories = new ArrayDeque<>(); + Set visitedDirectories = new LinkedHashSet<>(); + directories.add(""); + long maxEntries = Math.min( + MAX_TREE_ENTRIES, + Math.max(MIN_TREE_ENTRIES, (long) maxFiles * 4L)); + long traversedEntries = 0; + int requests = 0; + + String encodedCommit = URLEncoder.encode( + commit, StandardCharsets.UTF_8).replace("+", "%20"); + while (!directories.isEmpty()) { + String directory = directories.removeFirst(); + if (!visitedDirectories.add(directory)) continue; + String encodedDirectory = URLEncoder.encode( + directory, StandardCharsets.UTF_8) + .replace("+", "%20") + .replace("%2F", "/"); + String url = API_BASE + "/repositories/" + workspaceId + "/" + + repoIdOrSlug + "/src/" + encodedCommit + "/" + + (encodedDirectory.isEmpty() ? "" : encodedDirectory + "/") + + "?pagelen=100"; + Set visitedPages = new LinkedHashSet<>(); + while (url != null) { + if (!visitedPages.add(url)) { + throw new IOException("Repository tree pagination repeated a page"); + } + requests++; + if (requests > MAX_TREE_REQUESTS) { + throw new IOException( + "Repository tree exceeds the " + MAX_TREE_REQUESTS + + "-request traversal limit"); + } + Request request = new Request.Builder() + .url(url) + .header("Accept", "application/json") + .get() + .build(); + try (Response response = httpClient.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw createException("list repository files", response); + } + ResponseBody body = response.body(); + if (body == null) { + throw new IOException("Empty repository tree response"); + } + JsonNode page = objectMapper.readTree(body.string()); + JsonNode entries = page.path("values"); + if (!entries.isArray()) { + throw new IOException("Repository tree response has no values"); + } + traversedEntries += entries.size(); + if (traversedEntries > maxEntries) { + throw new IOException( + "Repository tree exceeds the " + maxEntries + + "-entry traversal limit"); + } + for (JsonNode entry : entries) { + String path = entry.path("path").asText(""); + if (path.isBlank()) continue; + String type = entry.path("type").asText(""); + if (isRegularFile(entry)) { + files.add(path); + if (files.size() > maxFiles) { + throw new IOException( + "Repository tree exceeds the " + maxFiles + + "-file inventory limit"); + } + } else if ("commit_directory".equals(type)) { + directories.addLast(path); + } + } + url = page.hasNonNull("next") + ? page.path("next").asText() + : null; + } + } + } + return List.copyOf(files); + } + + private static boolean isRegularFile(JsonNode entry) { + if (!"commit_file".equals(entry.path("type").asText(""))) return false; + JsonNode attributes = entry.path("attributes"); + if (!attributes.isArray()) return true; + for (JsonNode attribute : attributes) { + String value = attribute.asText(""); + if ("link".equals(value) || "subrepository".equals(value)) return false; + } + return true; + } + @Override public String getLatestCommitHash(String workspaceId, String repoIdOrSlug, String branchName) throws IOException { // Get branch info to get the latest commit diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/GitHubClient.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/GitHubClient.java index fb7017e0..562c3991 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/GitHubClient.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/GitHubClient.java @@ -22,9 +22,12 @@ import java.nio.file.Path; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.List; import java.util.Map; +import java.util.TreeSet; /** * VcsClient implementation for GitHub. @@ -42,6 +45,9 @@ public class GitHubClient implements VcsClient { private static final String GITHUB_API_VERSION_HEADER = "X-GitHub-Api-Version"; private static final String GITHUB_ACCEPT_HEADER = "application/vnd.github+json"; private static final String GITHUB_API_VERSION = "2022-11-28"; + private static final int MAX_TREE_REQUESTS = 10_000; + private static final long MAX_TREE_ENTRIES = 2_000_000L; + private static final long MIN_TREE_ENTRIES = 256L; private final OkHttpClient httpClient; private final ObjectMapper objectMapper; @@ -481,6 +487,182 @@ public String getFileContent(String workspaceId, String repoIdOrSlug, String fil return body.string(); } } + + @Override + public List listRepositoryFiles( + String workspaceId, + String repoIdOrSlug, + String commit, + int maxFiles + ) throws IOException { + if (maxFiles <= 0) throw new IllegalArgumentException("maxFiles must be positive"); + TreeTraversalBudget traversalBudget = new TreeTraversalBudget(maxFiles); + + // The Git Trees endpoint is documented for a tree object ID (or a + // branch/tag ref), while PR selection supplies a pinned commit-object + // ID. Resolve that commit once so every tree request is anchored to + // the exact snapshot and the truncated-response fallback can start + // from the corresponding root tree object. + String rootTreeSha = getCommitTreeSha( + workspaceId, repoIdOrSlug, commit, traversalBudget); + + JsonNode recursive = getGitTree( + workspaceId, repoIdOrSlug, rootTreeSha, true, traversalBudget); + if (!recursive.path("truncated").asBoolean(false)) { + TreeSet files = new TreeSet<>(); + collectGitTreeFiles( + recursive, "", files, maxFiles, null, traversalBudget); + return List.copyOf(files); + } + + // GitHub caps recursive tree responses. Fall back to walking each + // non-recursive tree object so a partial response is never presented + // to repository detection as a complete inventory. + TreeSet files = new TreeSet<>(); + Deque pending = new ArrayDeque<>(); + pending.add(new GitTreeCursor("", rootTreeSha)); + while (!pending.isEmpty()) { + GitTreeCursor cursor = pending.removeFirst(); + JsonNode tree = getGitTree( + workspaceId, + repoIdOrSlug, + cursor.reference(), + false, + traversalBudget); + collectGitTreeFiles( + tree, + cursor.prefix(), + files, + maxFiles, + pending, + traversalBudget); + } + return List.copyOf(files); + } + + private String getCommitTreeSha( + String workspaceId, + String repoIdOrSlug, + String commitSha, + TreeTraversalBudget traversalBudget + ) throws IOException { + traversalBudget.recordRequest(); + String encodedCommit = URLEncoder.encode(commitSha, StandardCharsets.UTF_8); + String url = API_BASE + "/repos/" + workspaceId + "/" + repoIdOrSlug + + "/git/commits/" + encodedCommit; + Request request = createGetRequest(url); + try (Response response = httpClient.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw createException("resolve repository commit tree", response); + } + ResponseBody body = response.body(); + if (body == null) throw new IOException("Empty repository commit response"); + String treeSha = objectMapper.readTree(body.string()) + .path("tree") + .path("sha") + .asText(""); + if (treeSha.isBlank()) { + throw new IOException("Repository commit response has no tree object ID"); + } + return treeSha; + } + } + + private JsonNode getGitTree( + String workspaceId, + String repoIdOrSlug, + String treeReference, + boolean recursive, + TreeTraversalBudget traversalBudget + ) throws IOException { + traversalBudget.recordRequest(); + String encodedReference = URLEncoder.encode( + treeReference, StandardCharsets.UTF_8); + String url = API_BASE + "/repos/" + workspaceId + "/" + repoIdOrSlug + + "/git/trees/" + encodedReference + + (recursive ? "?recursive=1" : ""); + Request request = createGetRequest(url); + try (Response response = httpClient.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw createException("list repository files", response); + } + ResponseBody body = response.body(); + if (body == null) throw new IOException("Empty repository tree response"); + JsonNode tree = objectMapper.readTree(body.string()); + if (!tree.path("tree").isArray()) { + throw new IOException("Repository tree response has no tree entries"); + } + traversalBudget.recordEntries(tree.path("tree").size()); + return tree; + } + } + + private static void collectGitTreeFiles( + JsonNode tree, + String prefix, + TreeSet files, + int maxFiles, + Deque pending, + TreeTraversalBudget traversalBudget + ) throws IOException { + for (JsonNode entry : tree.path("tree")) { + String name = entry.path("path").asText(""); + String type = entry.path("type").asText(""); + if (name.isBlank()) continue; + String path = prefix.isEmpty() ? name : prefix + "/" + name; + if (isRegularFile(entry)) { + files.add(path); + if (files.size() > maxFiles) { + throw new IOException( + "Repository tree exceeds the " + maxFiles + "-file inventory limit"); + } + } else if (pending != null && "tree".equals(type)) { + String sha = entry.path("sha").asText(""); + if (sha.isBlank()) { + throw new IOException("Repository subtree entry has no object ID: " + path); + } + pending.addLast(new GitTreeCursor(path, sha)); + } + } + } + + private static boolean isRegularFile(JsonNode entry) { + if (!"blob".equals(entry.path("type").asText(""))) return false; + String mode = entry.path("mode").asText(""); + return "100644".equals(mode) || "100755".equals(mode); + } + + private record GitTreeCursor(String prefix, String reference) {} + + private static final class TreeTraversalBudget { + private final long maxEntries; + private long entries; + private int requests; + + private TreeTraversalBudget(int maxFiles) { + maxEntries = Math.min( + MAX_TREE_ENTRIES, + Math.max(MIN_TREE_ENTRIES, (long) maxFiles * 4L)); + } + + private void recordRequest() throws IOException { + requests++; + if (requests > MAX_TREE_REQUESTS) { + throw new IOException( + "Repository tree exceeds the " + MAX_TREE_REQUESTS + + "-request traversal limit"); + } + } + + private void recordEntries(int count) throws IOException { + entries += count; + if (entries > maxEntries) { + throw new IOException( + "Repository tree exceeds the " + maxEntries + + "-entry traversal limit"); + } + } + } @Override public String getLatestCommitHash(String workspaceId, String repoIdOrSlug, String branchName) throws IOException { diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/GitLabClient.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/GitLabClient.java index 236e3d05..2c6929aa 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/GitLabClient.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/GitLabClient.java @@ -477,6 +477,17 @@ public String getFileContent(String workspaceId, String repoIdOrSlug, String fil return body.string(); } } + + @Override + public List listRepositoryFiles( + String workspaceId, + String repoIdOrSlug, + String commit, + int maxFiles + ) throws IOException { + return repositoryApi.listFiles( + workspaceId, repoIdOrSlug, commit, maxFiles); + } @Override public String getLatestCommitHash(String workspaceId, String repoIdOrSlug, String branchName) throws IOException { diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabRepositoryApi.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabRepositoryApi.java index 3f543c8a..c304a31b 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabRepositoryApi.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabRepositoryApi.java @@ -1,14 +1,23 @@ package org.rostilos.codecrow.vcsclient.gitlab.api; +import com.fasterxml.jackson.databind.JsonNode; import okhttp3.Response; import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; /** * Focused GitLab repository operations used by the shared client. */ public final class GitLabRepositoryApi { + private static final int MAX_TREE_REQUESTS = 10_000; + private static final long MAX_TREE_ENTRIES = 2_000_000L; + private static final long MIN_TREE_ENTRIES = 256L; + private final GitLabApiContext api; public GitLabRepositoryApi(GitLabApiContext api) { @@ -54,4 +63,90 @@ public String getTree( return api.bodyOr(response, "[]"); } } + + public List listFiles( + String namespace, + String project, + String commit, + int maxFiles + ) throws IOException { + if (maxFiles <= 0) throw new IllegalArgumentException("maxFiles must be positive"); + final int pageSize = 100; + TreeSet files = new TreeSet<>(); + Set visitedPages = new LinkedHashSet<>(); + long maxEntries = Math.min( + MAX_TREE_ENTRIES, + Math.max(MIN_TREE_ENTRIES, (long) maxFiles * 4L)); + long traversedEntries = 0; + int requests = 0; + String treeEndpoint = api.projectUrl(namespace, project) + "/repository/tree"; + String url = treeEndpoint + + "?ref=" + api.encode(commit) + + "&recursive=true&per_page=" + pageSize + + "&pagination=keyset"; + while (url != null) { + if (!url.startsWith(treeEndpoint)) { + throw new IOException("Repository tree pagination left the provider endpoint"); + } + if (!visitedPages.add(url)) { + throw new IOException("Repository tree pagination repeated a page"); + } + requests++; + if (requests > MAX_TREE_REQUESTS) { + throw new IOException( + "Repository tree exceeds the " + MAX_TREE_REQUESTS + + "-request traversal limit"); + } + try (Response response = api.execute(api.get(url))) { + if (!response.isSuccessful()) { + throw api.error("list repository files", response); + } + JsonNode entries = api.objectMapper().readTree( + api.bodyOr(response, "[]")); + if (!entries.isArray()) { + throw new IOException("Repository tree response is not an array"); + } + traversedEntries += entries.size(); + if (traversedEntries > maxEntries) { + throw new IOException( + "Repository tree exceeds the " + maxEntries + + "-entry traversal limit"); + } + for (JsonNode entry : entries) { + if (!isRegularFile(entry)) continue; + String path = entry.path("path").asText(""); + if (path.isBlank()) continue; + files.add(path); + if (files.size() > maxFiles) { + throw new IOException( + "Repository tree exceeds the " + maxFiles + + "-file inventory limit"); + } + } + url = nextPageUrl(response); + } + } + return List.copyOf(files); + } + + private static boolean isRegularFile(JsonNode entry) { + if (!"blob".equals(entry.path("type").asText(""))) return false; + String mode = entry.path("mode").asText(""); + return "100644".equals(mode) || "100755".equals(mode); + } + + private static String nextPageUrl(Response response) throws IOException { + String link = response.header("Link"); + if (link == null || link.isBlank()) return null; + for (String part : link.split(",")) { + if (!part.contains("rel=\"next\"") && !part.contains("rel=next")) continue; + int start = part.indexOf('<'); + int end = part.indexOf('>', start + 1); + if (start < 0 || end <= start + 1) { + throw new IOException("Repository tree returned an invalid next-page link"); + } + return part.substring(start + 1, end); + } + return null; + } } diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudRepositoryFileListingTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudRepositoryFileListingTest.java new file mode 100644 index 00000000..00557384 --- /dev/null +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudRepositoryFileListingTest.java @@ -0,0 +1,89 @@ +package org.rostilos.codecrow.vcsclient.bitbucket.cloud; + +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class BitbucketCloudRepositoryFileListingTest { + + @Mock + private OkHttpClient httpClient; + + @Test + void followsDirectoryAndPageListingsAndReturnsOnlyFiles() throws Exception { + List requests = new ArrayList<>(); + when(httpClient.newCall(any(Request.class))).thenAnswer(invocation -> { + Request request = invocation.getArgument(0); + requests.add(request); + String url = request.url().toString(); + String body; + if (url.endsWith("/src/commit-sha/?pagelen=100")) { + body = """ + { + "values": [ + {"path":"README.md","type":"commit_file","attributes":[]}, + {"path":"README-link","type":"commit_file","attributes":["link"]}, + {"path":"vendor","type":"commit_file","attributes":["subrepository"]}, + {"path":"src folder","type":"commit_directory"}, + {"path":"external","type":"commit"} + ], + "next":"https://api.bitbucket.org/2.0/repositories/workspace/repo/src/commit-sha/?pagelen=100&page=2" + } + """; + } else if (url.endsWith("/src/commit-sha/?pagelen=100&page=2")) { + body = """ + {"values":[{"path":"LICENSE","type":"commit_file"}]} + """; + } else if (url.endsWith("/src/commit-sha/src%20folder/?pagelen=100")) { + body = """ + {"values":[{"path":"src folder/App.java","type":"commit_file","attributes":["executable"]}]} + """; + } else { + throw new AssertionError("Unexpected Bitbucket request: " + url); + } + Response response = jsonResponse(request, body); + Call call = mock(Call.class); + when(call.execute()).thenReturn(response); + return call; + }); + + List files = new BitbucketCloudClient(httpClient) + .listRepositoryFiles("workspace", "repo", "commit-sha", 10); + + assertThat(files).containsExactly( + "LICENSE", "README.md", "src folder/App.java"); + assertThat(requests).extracting(request -> request.url().toString()) + .containsExactly( + "https://api.bitbucket.org/2.0/repositories/workspace/repo/src/commit-sha/?pagelen=100", + "https://api.bitbucket.org/2.0/repositories/workspace/repo/src/commit-sha/?pagelen=100&page=2", + "https://api.bitbucket.org/2.0/repositories/workspace/repo/src/commit-sha/src%20folder/?pagelen=100"); + } + + private static Response jsonResponse(Request request, String body) { + return new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create( + body, MediaType.parse("application/json"))) + .build(); + } +} diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/GitHubRepositoryFileListingTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/GitHubRepositoryFileListingTest.java new file mode 100644 index 00000000..0238ed7d --- /dev/null +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/GitHubRepositoryFileListingTest.java @@ -0,0 +1,99 @@ +package org.rostilos.codecrow.vcsclient.github; + +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class GitHubRepositoryFileListingTest { + + @Mock + private OkHttpClient httpClient; + + @Test + void resolvesCommitToTreeAndWalksSubtreeShasAfterRecursiveTruncation() + throws Exception { + List requests = new ArrayList<>(); + when(httpClient.newCall(any(Request.class))).thenAnswer(invocation -> { + Request request = invocation.getArgument(0); + requests.add(request); + String url = request.url().toString(); + String body; + if (url.endsWith("/git/commits/commit-sha")) { + body = "{\"tree\":{\"sha\":\"root-tree-sha\"}}"; + } else if (url.endsWith("/git/trees/root-tree-sha?recursive=1")) { + body = """ + { + "truncated": true, + "tree": [ + {"path":"partial.txt","type":"blob","mode":"100644","sha":"partial"} + ] + } + """; + } else if (url.endsWith("/git/trees/root-tree-sha")) { + body = """ + { + "tree": [ + {"path":"README.md","type":"blob","mode":"100644","sha":"readme"}, + {"path":"README-link","type":"blob","mode":"120000","sha":"link"}, + {"path":"src","type":"tree","mode":"040000","sha":"src-tree-sha"}, + {"path":"vendor","type":"commit","mode":"160000","sha":"submodule"} + ] + } + """; + } else if (url.endsWith("/git/trees/src-tree-sha")) { + body = """ + { + "tree": [ + {"path":"App.java","type":"blob","mode":"100755","sha":"app"} + ] + } + """; + } else { + throw new AssertionError("Unexpected GitHub request: " + url); + } + Response response = jsonResponse(request, body); + Call call = mock(Call.class); + when(call.execute()).thenReturn(response); + return call; + }); + + List files = new GitHubClient(httpClient) + .listRepositoryFiles("owner", "repo", "commit-sha", 10); + + assertThat(files).containsExactly("README.md", "src/App.java"); + assertThat(requests).extracting(request -> request.url().toString()) + .containsExactly( + "https://api.github.com/repos/owner/repo/git/commits/commit-sha", + "https://api.github.com/repos/owner/repo/git/trees/root-tree-sha?recursive=1", + "https://api.github.com/repos/owner/repo/git/trees/root-tree-sha", + "https://api.github.com/repos/owner/repo/git/trees/src-tree-sha"); + } + + private static Response jsonResponse(Request request, String body) { + return new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create( + body, MediaType.parse("application/json"))) + .build(); + } +} diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabRepositoryApiTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabRepositoryApiTest.java index 08809000..720c7635 100644 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabRepositoryApiTest.java +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabRepositoryApiTest.java @@ -6,12 +6,143 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; class GitLabRepositoryApiTest { + @Test + void listsOnlyBlobsAcrossRepositoryTreePages() throws Exception { + try (MockWebServer gitLab = new MockWebServer()) { + gitLab.start(); + String nextPage = gitLab.url( + "/gitlab/api/v4/projects/my%20group%2Fmy%20project" + + "/repository/tree?ref=commit-sha&recursive=true" + + "&per_page=100&pagination=keyset&page_token=cursor-2") + .toString(); + gitLab.enqueue(new MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "application/json") + .setHeader("Link", "<" + nextPage + ">; rel=\"next\"") + .setBody(""" + [ + {"path":"README.md","type":"blob","mode":"100644"}, + {"path":"README-link","type":"blob","mode":"120000"}, + {"path":"src","type":"tree","mode":"040000"} + ] + """)); + gitLab.enqueue(new MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "application/json") + .setBody(""" + [{"path":"src/App.java","type":"blob","mode":"100755"}] + """)); + + GitLabRepositoryApi repositories = + new GitLabRepositoryApi(new GitLabApiContext( + new OkHttpClient(), + gitLab.url("/gitlab").toString())); + + List files = repositories.listFiles( + "my group", "my project", "commit-sha", 10); + + assertThat(files).containsExactly("README.md", "src/App.java"); + assertThat(gitLab.takeRequest().getPath()) + .isEqualTo("/gitlab/api/v4/projects/my%20group%2Fmy%20project" + + "/repository/tree?ref=commit-sha&recursive=true" + + "&per_page=100&pagination=keyset"); + assertThat(gitLab.takeRequest().getPath()) + .isEqualTo("/gitlab/api/v4/projects/my%20group%2Fmy%20project" + + "/repository/tree?ref=commit-sha&recursive=true" + + "&per_page=100&pagination=keyset&page_token=cursor-2"); + } + } + + @Test + void rejectsARepeatedKeysetPageBeforeIssuingAnotherRequest() throws Exception { + try (MockWebServer gitLab = new MockWebServer()) { + gitLab.start(); + String firstPage = gitLab.url( + "/api/v4/projects/team%2Frepo/repository/tree" + + "?ref=commit-sha&recursive=true&per_page=100" + + "&pagination=keyset") + .toString(); + gitLab.enqueue(new MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "application/json") + .setHeader("Link", "<" + firstPage + ">; rel=\"next\"") + .setBody(""" + [{"path":"one.txt","type":"blob","mode":"100644"}] + """)); + + GitLabRepositoryApi repositories = + new GitLabRepositoryApi(new GitLabApiContext( + new OkHttpClient(), + gitLab.url("/").toString())); + + assertThatThrownBy(() -> repositories.listFiles( + "team", "repo", "commit-sha", 10)) + .isInstanceOf(IOException.class) + .hasMessageContaining("repeated a page"); + assertThat(gitLab.getRequestCount()).isEqualTo(1); + } + } + + @Test + void rejectsAnOverLimitInventoryWhileWalkingTheCurrentPage() throws Exception { + try (MockWebServer gitLab = new MockWebServer()) { + gitLab.enqueue(new MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "application/json") + .setBody(""" + [ + {"path":"one.txt","type":"blob","mode":"100644"}, + {"path":"two.txt","type":"blob","mode":"100644"} + ] + """)); + gitLab.start(); + + GitLabRepositoryApi repositories = + new GitLabRepositoryApi(new GitLabApiContext( + new OkHttpClient(), + gitLab.url("/").toString())); + + assertThatThrownBy(() -> repositories.listFiles( + "team", "repo", "commit-sha", 1)) + .isInstanceOf(IOException.class) + .hasMessageContaining("1-file inventory limit"); + } + } + + @Test + void boundsNonFileTreeEntriesAsWellAsReturnedFiles() throws Exception { + try (MockWebServer gitLab = new MockWebServer()) { + String entries = IntStream.range(0, 257) + .mapToObj(index -> "{\"path\":\"dir-" + index + + "\",\"type\":\"tree\",\"mode\":\"040000\"}") + .collect(Collectors.joining(",", "[", "]")); + gitLab.enqueue(new MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "application/json") + .setBody(entries)); + gitLab.start(); + + GitLabRepositoryApi repositories = + new GitLabRepositoryApi(new GitLabApiContext( + new OkHttpClient(), + gitLab.url("/").toString())); + + assertThatThrownBy(() -> repositories.listFiles( + "team", "repo", "commit-sha", 1)) + .isInstanceOf(IOException.class) + .hasMessageContaining("256-entry traversal limit"); + } + } + @Test void fileExistenceUsesSharedEncodingAndConfiguredBase() throws Exception { try (MockWebServer gitLab = new MockWebServer()) { diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/ProjectCapabilitySelectionService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/ProjectCapabilitySelectionService.java index 7afa7575..58808c12 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/ProjectCapabilitySelectionService.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/ProjectCapabilitySelectionService.java @@ -18,18 +18,23 @@ import org.springframework.stereotype.Service; import java.io.IOException; -import java.util.LinkedHashMap; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeMap; import java.util.TreeSet; @Service public class ProjectCapabilitySelectionService { private static final Logger log = LoggerFactory.getLogger(ProjectCapabilitySelectionService.class); - private static final int MAX_MARKER_FILES = 16; + private static final int MAX_MARKER_FILES = 64; private static final int MAX_MARKER_BYTES = 262_144; + private static final int MAX_REPOSITORY_FILES = 500_000; private final PluginRuntime runtime; private final PluginRegistry registry; @@ -67,31 +72,103 @@ public SelectionPlan plan( String commit, List changedFiles, AnalysisProfileConfig analysisProfile) { - TreeSet paths = new TreeSet<>(); + TreeSet changedPaths = new TreeSet<>(); if (changedFiles != null) { changedFiles.stream().map(ProjectCapabilitySelectionService::normalize) - .forEach(paths::add); + .forEach(changedPaths::add); } + TreeSet paths = new TreeSet<>(); String projectType = analysisProfile != null ? analysisProfile.projectType() : null; String sourceRoot = analysisProfile != null ? analysisProfile.sourceRoot() : null; + boolean completeRepositoryInventory = false; + if (projectType == null && !registry.descriptors().isEmpty()) { + try { + List repositoryFiles = vcsClient.listRepositoryFiles( + workspace, + repository, + commit, + MAX_REPOSITORY_FILES); + if (repositoryFiles != null) { + repositoryFiles.stream() + .filter(path -> path != null && !path.isBlank()) + .map(ProjectCapabilitySelectionService::normalize) + .forEach(paths::add); + completeRepositoryInventory = true; + } else { + log.warn( + "Repository path inventory at commit {} was unavailable; automatic " + + "plugin detection will continue with reduced evidence", + commit); + } + } catch (Exception exception) { + log.warn( + "Cannot list the pinned repository tree at commit {}; automatic plugin " + + "detection will continue with changed paths and exact marker reads: {}", + commit, + exception.getMessage()); + } + } + if (!completeRepositoryInventory) { + // Changed paths are only a fallback existence signal. A complete + // pinned tree is authoritative and deliberately excludes deleted + // paths and old rename sides from repository facts. + paths.addAll(changedPaths); + } + TreeSet markerPaths = new TreeSet<>(); TreeSet patternMarkers = new TreeSet<>(); + List markerReadRules = new ArrayList<>(); if (projectType == null) { for (PluginDescriptor descriptor : registry.descriptors()) { markerPaths.addAll(descriptor.detection().filesAll()); markerPaths.addAll(descriptor.detection().filesAny()); - descriptor.detection().contentMarkers().stream() - .map(ContentMarker::path) - .forEach(markerPaths::add); - descriptor.detection().alternatives().forEach(alternative -> { + for (String path : descriptor.detection().filesAll()) { + markerReadRules.add(MarkerReadRule.exact( + descriptor.id(), "files-all:" + path, path, false)); + } + for (String path : descriptor.detection().filesAny()) { + markerReadRules.add(MarkerReadRule.exact( + descriptor.id(), "files-any:" + path, path, false)); + } + for (ContentMarker marker : descriptor.detection().contentMarkers()) { + markerPaths.add(marker.path()); + markerReadRules.add(MarkerReadRule.exact( + descriptor.id(), + "content:" + marker.path() + ":" + marker.contains(), + marker.path(), + true)); + } + int alternativeIndex = 0; + for (var alternative : descriptor.detection().alternatives()) { + String scope = "alternative-" + alternativeIndex++ + ":"; markerPaths.addAll(alternative.filesAll()); markerPaths.addAll(alternative.filesAny()); - alternative.contentMarkers().stream() - .map(ContentMarker::path) - .forEach(markerPaths::add); - patternMarkers.addAll(alternative.contentPatternMarkers()); - }); + for (String path : alternative.filesAll()) { + markerReadRules.add(MarkerReadRule.exact( + descriptor.id(), scope + "files-all:" + path, path, false)); + } + for (String path : alternative.filesAny()) { + markerReadRules.add(MarkerReadRule.exact( + descriptor.id(), scope + "files-any:" + path, path, false)); + } + for (ContentMarker marker : alternative.contentMarkers()) { + markerPaths.add(marker.path()); + markerReadRules.add(MarkerReadRule.exact( + descriptor.id(), + scope + "content:" + marker.path() + ":" + marker.contains(), + marker.path(), + true)); + } + for (ContentPatternMarker marker : alternative.contentPatternMarkers()) { + patternMarkers.add(marker); + markerReadRules.add(MarkerReadRule.pattern( + descriptor.id(), + scope + "pattern:" + marker.pathPattern() + + ":" + marker.contains(), + marker)); + } + } } } TreeSet candidateRoots = new TreeSet<>(); @@ -99,51 +176,140 @@ public SelectionPlan plan( candidateRoots.add(sourceRoot); } else { candidateRoots.add(""); + Map> rootMarkersByName = new LinkedHashMap<>(); + for (String markerPath : markerPaths) { + rootMarkersByName + .computeIfAbsent(baseName(markerPath), ignored -> new ArrayList<>()) + .add(markerPath); + } + for (String repositoryPath : paths) { + for (String markerPath : rootMarkersByName.getOrDefault( + baseName(repositoryPath), List.of())) { + String root = rootForMarker(repositoryPath, markerPath); + if (root != null) candidateRoots.add(root); + } + } + if (!completeRepositoryInventory) { + // Without a provider tree, ancestor roots at least let an + // ordinary changed framework file discover unchanged exact + // markers. Glob-only context still correctly remains reduced. + changedPaths.forEach(path -> addAncestorRoots(candidateRoots, path)); + } + } + + Comparator candidateOrder = markerCandidateOrder(changedPaths); + Map markerLanes = new LinkedHashMap<>(); + boolean candidateDiscoveryReduced = false; + if (completeRepositoryInventory) { + Map> exactRulesByName = new LinkedHashMap<>(); + List patternRules = new ArrayList<>(); + for (MarkerReadRule rule : markerReadRules) { + if (rule.exactPath() != null && rule.contentRequired()) { + exactRulesByName + .computeIfAbsent(baseName(rule.exactPath()), + ignored -> new ArrayList<>()) + .add(rule); + } else if (rule.pattern() != null) { + patternRules.add(rule); + } + } for (String repositoryPath : paths) { - for (String markerPath : markerPaths) { - if (repositoryPath.equals(markerPath)) { - candidateRoots.add(""); - } else if (repositoryPath.endsWith("/" + markerPath)) { - candidateRoots.add(repositoryPath.substring( - 0, - repositoryPath.length() - markerPath.length() - 1)); + for (MarkerReadRule rule : exactRulesByName.getOrDefault( + baseName(repositoryPath), List.of())) { + String root = rootForMarker(repositoryPath, rule.exactPath()); + if (root == null || !candidateRoots.contains(root)) continue; + candidateDiscoveryReduced |= offerCandidate( + markerLanes, + rule, + new MarkerCandidate(root, repositoryPath), + candidateOrder); + } + + if (candidateRoots.contains("")) { + candidateDiscoveryReduced |= offerPatternCandidates( + markerLanes, + patternRules, + "", + repositoryPath, + repositoryPath, + candidateOrder); + } + int slash = repositoryPath.indexOf('/'); + while (slash > 0) { + String root = repositoryPath.substring(0, slash); + if (candidateRoots.contains(root)) { + candidateDiscoveryReduced |= offerPatternCandidates( + markerLanes, + patternRules, + root, + repositoryPath.substring(slash + 1), + repositoryPath, + candidateOrder); } + slash = repositoryPath.indexOf('/', slash + 1); + } + } + } else { + for (String root : candidateRoots) { + for (MarkerReadRule rule : markerReadRules) { + if (rule.exactPath() == null) continue; + candidateDiscoveryReduced |= offerCandidate( + markerLanes, + rule, + new MarkerCandidate(root, rooted(root, rule.exactPath())), + candidateOrder); } } } - TreeSet resolvedMarkerPaths = candidateRoots.stream() - .flatMap(root -> markerPaths.stream().map(path -> - root.isEmpty() ? path : root + "/" + path)) - .collect(java.util.stream.Collectors.toCollection(TreeSet::new)); - if (resolvedMarkerPaths.size() > MAX_MARKER_FILES) { - throw new IllegalStateException("plugin marker declarations exceed the host budget"); + MarkerSchedule markerSchedule = scheduleMarkerPaths( + markerLanes.values(), candidateOrder, MAX_MARKER_FILES); + List scheduledMarkerPaths = markerSchedule.paths(); + if (candidateDiscoveryReduced || markerSchedule.omittedCandidates()) { + log.warn( + "Some plugin marker candidates were skipped after reaching the {}-file host " + + "budget; automatic plugin detection will continue with reduced evidence", + MAX_MARKER_FILES); } Map markerContents = new LinkedHashMap<>(); int consumed = 0; - for (String markerPath : resolvedMarkerPaths) { + int skippedForBytes = 0; + for (String markerPath : scheduledMarkerPaths) { try { String content = vcsClient.getFileContent( workspace, repository, markerPath, commit); if (content == null) continue; + // A successful pinned read proves path existence even when + // the content cannot fit in the optional marker byte budget. + paths.add(markerPath); int bytes = content.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; if (consumed + bytes > MAX_MARKER_BYTES) { - throw new IllegalStateException("plugin marker contents exceed the host budget"); + skippedForBytes++; + continue; } consumed += bytes; - paths.add(markerPath); markerContents.put(markerPath, content); } catch (IOException exception) { - throw new IllegalStateException( - "Cannot read plugin marker from the pinned repository snapshot: " + markerPath, + log.warn( + "Cannot read plugin marker {} from the pinned repository snapshot; " + + "automatic plugin detection will continue with reduced evidence", + markerPath, exception); } } + if (skippedForBytes > 0) { + log.warn( + "Skipped {} plugin marker content file(s) after reaching the {}-byte host " + + "budget; automatic plugin detection will continue with reduced evidence", + skippedForBytes, + MAX_MARKER_BYTES); + } + RepositoryFacts repositoryFacts = new RepositoryFacts( + commit, List.copyOf(paths), markerContents, + projectType, sourceRoot); ProjectCapabilities preliminary = selector.select( - new RepositoryFacts( - commit, List.copyOf(paths), markerContents, - projectType, sourceRoot)); + repositoryFacts, changedPaths); List enrichmentPaths = filterEnrichmentPaths(preliminary, changedFiles); return new SelectionPlan( commit, @@ -154,7 +320,8 @@ public SelectionPlan plan( preliminary, enrichmentPaths, projectType, - sourceRoot); + sourceRoot, + completeRepositoryInventory); } /** @@ -168,37 +335,73 @@ public ProjectCapabilities complete( TreeSet paths = new TreeSet<>(plan.repositoryPaths()); Map markerContents = new LinkedHashMap<>(plan.markerContents()); int consumed = plan.markerBytes(); + int skippedForBytes = 0; + boolean skippedForFiles = false; if (enrichment != null && enrichment.fileContents() != null) { - enrichment.fileContents().stream() - .map(file -> normalize(file.path())) - .forEach(paths::add); - for (ContentPatternMarker marker : plan.patternMarkers()) { - var matchingFile = enrichment.fileContents().stream() + if (!plan.completeRepositoryInventory()) { + enrichment.fileContents().stream() .filter(file -> !file.skipped() && file.content() != null) - .filter(file -> { - String relative = relativeToRoot( - normalize(file.path()), plan.sourceRoot()); - return relative != null - && PluginGlob.matches(marker.pathPattern(), relative); - }) - .filter(file -> file.content().contains(marker.contains())) - .findFirst(); - if (matchingFile.isEmpty()) continue; - String path = normalize(matchingFile.get().path()); - if (markerContents.containsKey(path)) continue; - String content = matchingFile.get().content(); + .map(file -> normalize(file.path())) + .forEach(paths::add); + } + Map> patternCandidates = + new TreeMap<>(); + for (ContentPatternMarker marker : plan.patternMarkers()) { + TreeMap candidates = patternCandidates.computeIfAbsent( + marker, ignored -> new TreeMap<>()); + for (var file : enrichment.fileContents()) { + if (file.skipped() || file.content() == null) continue; + String path = normalize(file.path()); + if (markerContents.containsKey(path)) continue; + if (plan.completeRepositoryInventory() && !paths.contains(path)) continue; + if (!matchesPatternCandidate(marker, path, plan.sourceRoot()) + || !file.content().contains(marker.contains())) { + continue; + } + candidates.putIfAbsent(path, file.content()); + if (candidates.size() > MAX_MARKER_FILES) { + candidates.pollLastEntry(); + skippedForFiles = true; + } + } + } + PatternMarkerSchedule patternSchedule = schedulePatternMarkerPaths( + patternCandidates, + markerContents.keySet(), + Math.max(0, MAX_MARKER_FILES - markerContents.size())); + skippedForFiles |= patternSchedule.omittedCandidates(); + for (Map.Entry candidate : patternSchedule.files().entrySet()) { + String path = candidate.getKey(); + String content = candidate.getValue(); int bytes = content.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; if (consumed + bytes > MAX_MARKER_BYTES) { - throw new IllegalStateException("plugin marker contents exceed the host byte budget"); + skippedForBytes++; + continue; } consumed += bytes; markerContents.put(path, content); } } + if (skippedForFiles) { + log.warn( + "Some content-pattern marker files were skipped after reaching the {}-file " + + "host budget; automatic plugin detection will continue with reduced " + + "evidence", + MAX_MARKER_FILES); + } + if (skippedForBytes > 0) { + log.warn( + "Skipped {} content-pattern marker file(s) after reaching the {}-byte host " + + "budget; automatic plugin detection will continue with reduced evidence", + skippedForBytes, + MAX_MARKER_BYTES); + } - return selector.select(new RepositoryFacts( - plan.commit(), List.copyOf(paths), markerContents, - plan.projectType(), plan.sourceRoot())); + return selector.select( + new RepositoryFacts( + plan.commit(), List.copyOf(paths), markerContents, + plan.projectType(), plan.sourceRoot()), + plan.enrichmentPaths()); } /** @@ -237,6 +440,163 @@ public PluginRegistry registry() { return registry; } + private static void addAncestorRoots(Set roots, String path) { + int slash = path.lastIndexOf('/'); + while (slash > 0) { + roots.add(path.substring(0, slash)); + slash = path.lastIndexOf('/', slash - 1); + } + } + + private static String rooted(String root, String relative) { + return root == null || root.isBlank() ? relative : root + "/" + relative; + } + + private static String baseName(String path) { + int slash = path.lastIndexOf('/'); + return slash < 0 ? path : path.substring(slash + 1); + } + + private static String rootForMarker(String repositoryPath, String markerPath) { + if (repositoryPath.equals(markerPath)) return ""; + String suffix = "/" + markerPath; + if (!repositoryPath.endsWith(suffix)) return null; + return repositoryPath.substring(0, repositoryPath.length() - suffix.length()); + } + + private static boolean offerPatternCandidates( + Map lanes, + List rules, + String root, + String relativePath, + String repositoryPath, + Comparator candidateOrder) { + boolean reduced = false; + for (MarkerReadRule rule : rules) { + if (PluginGlob.matches(rule.pattern().pathPattern(), relativePath)) { + reduced |= offerCandidate( + lanes, + rule, + new MarkerCandidate(root, repositoryPath), + candidateOrder); + } + } + return reduced; + } + + private static boolean offerCandidate( + Map lanes, + MarkerReadRule rule, + MarkerCandidate candidate, + Comparator candidateOrder) { + MarkerLaneKey key = new MarkerLaneKey(rule.pluginId(), rule.condition()); + BoundedMarkerLane lane = lanes.computeIfAbsent( + key, ignored -> new BoundedMarkerLane(key, candidateOrder)); + return lane.offer(candidate, MAX_MARKER_FILES); + } + + private static Comparator markerCandidateOrder( + Set changedPaths) { + return Comparator + .comparing((MarkerCandidate candidate) -> + !changedPaths.contains(candidate.path())) + .thenComparing(candidate -> + !rootContainsChangedPath(candidate.root(), changedPaths)) + // The repository root contains every changed path. Retain its + // conventional marker before speculative nested ancestors can + // consume the bounded lane. + .thenComparing(candidate -> !candidate.root().isBlank()) + .thenComparing( + (MarkerCandidate candidate) -> candidate.root().length(), + Comparator.reverseOrder()) + .thenComparing(MarkerCandidate::path) + .thenComparing(MarkerCandidate::root); + } + + private static MarkerSchedule scheduleMarkerPaths( + Collection lanes, + Comparator candidateOrder, + int limit) { + Map> lanesByPlugin = new LinkedHashMap<>(); + for (BoundedMarkerLane lane : lanes) { + if (lane.candidates().isEmpty()) continue; + lanesByPlugin + .computeIfAbsent(lane.key().pluginId(), ignored -> new ArrayList<>()) + .add(new MarkerLaneCursor(lane)); + } + List plugins = new ArrayList<>(); + for (Map.Entry> entry : lanesByPlugin.entrySet()) { + entry.getValue().sort(Comparator + .comparing(MarkerLaneCursor::first, candidateOrder) + .thenComparing(cursor -> cursor.key().condition())); + plugins.add(new PluginLaneCursor(entry.getKey(), entry.getValue())); + } + plugins.sort(Comparator + .comparing(PluginLaneCursor::bestCandidate, candidateOrder) + .thenComparing(PluginLaneCursor::pluginId)); + + LinkedHashSet selected = new LinkedHashSet<>(); + boolean progressed = true; + while (selected.size() < limit && progressed) { + progressed = false; + for (PluginLaneCursor plugin : plugins) { + if (selected.size() >= limit) break; + String candidate = plugin.next(selected); + if (candidate == null) continue; + selected.add(candidate); + progressed = true; + } + } + boolean omittedCandidates = lanes.stream() + .flatMap(lane -> lane.candidates().stream()) + .map(MarkerCandidate::path) + .anyMatch(path -> !selected.contains(path)); + return new MarkerSchedule(List.copyOf(selected), omittedCandidates); + } + + private static PatternMarkerSchedule schedulePatternMarkerPaths( + Map> candidatesByPattern, + Set existingPaths, + int limit) { + List>> lanes = candidatesByPattern.values() + .stream() + .map(candidates -> candidates.entrySet().iterator()) + .toList(); + LinkedHashMap selected = new LinkedHashMap<>(); + boolean progressed = true; + while (selected.size() < limit && progressed) { + progressed = false; + for (var lane : lanes) { + while (lane.hasNext()) { + Map.Entry candidate = lane.next(); + if (existingPaths.contains(candidate.getKey()) + || selected.containsKey(candidate.getKey())) { + continue; + } + selected.put(candidate.getKey(), candidate.getValue()); + progressed = true; + break; + } + if (selected.size() >= limit) break; + } + } + boolean omittedCandidates = candidatesByPattern.values().stream() + .flatMap(candidates -> candidates.keySet().stream()) + .anyMatch(path -> !existingPaths.contains(path) && !selected.containsKey(path)); + return new PatternMarkerSchedule( + java.util.Collections.unmodifiableMap(new LinkedHashMap<>(selected)), + omittedCandidates); + } + + private static boolean rootContainsChangedPath( + String root, + Set changedPaths) { + if (root == null || root.isBlank()) return !changedPaths.isEmpty(); + String prefix = root + "/"; + return changedPaths.stream() + .anyMatch(path -> path.equals(root) || path.startsWith(prefix)); + } + private static String normalize(String path) { String normalized = path.replace('\\', '/'); while (normalized.startsWith("./")) normalized = normalized.substring(2); @@ -250,6 +610,155 @@ private static String relativeToRoot(String path, String root) { return path.startsWith(prefix) ? path.substring(prefix.length()) : null; } + private static boolean matchesPatternCandidate( + ContentPatternMarker marker, + String path, + String sourceRoot) { + String relative = relativeToRoot(path, sourceRoot); + if (relative == null) return false; + if (PluginGlob.matches(marker.pathPattern(), relative)) return true; + if (sourceRoot != null && !sourceRoot.isBlank()) return false; + int slash = relative.indexOf('/'); + while (slash >= 0) { + relative = relative.substring(slash + 1); + if (PluginGlob.matches(marker.pathPattern(), relative)) return true; + slash = relative.indexOf('/'); + } + return false; + } + + private record MarkerReadRule( + String pluginId, + String condition, + String exactPath, + ContentPatternMarker pattern, + boolean contentRequired) { + + private static MarkerReadRule exact( + String pluginId, + String condition, + String path, + boolean contentRequired) { + return new MarkerReadRule( + pluginId, condition, path, null, contentRequired); + } + + private static MarkerReadRule pattern( + String pluginId, + String condition, + ContentPatternMarker pattern) { + return new MarkerReadRule( + pluginId, condition, null, pattern, true); + } + } + + private record MarkerLaneKey(String pluginId, String condition) {} + + private record MarkerCandidate(String root, String path) {} + + private record MarkerSchedule( + List paths, + boolean omittedCandidates) {} + + private record PatternMarkerSchedule( + Map files, + boolean omittedCandidates) {} + + private static final class BoundedMarkerLane { + private final MarkerLaneKey key; + private final Comparator order; + private final TreeSet candidates; + private final Map candidatesByPath = new LinkedHashMap<>(); + + private BoundedMarkerLane( + MarkerLaneKey key, + Comparator order) { + this.key = key; + this.order = order; + candidates = new TreeSet<>(order); + } + + private boolean offer(MarkerCandidate candidate, int limit) { + MarkerCandidate existing = candidatesByPath.get(candidate.path()); + if (existing != null) { + if (order.compare(candidate, existing) >= 0) return false; + candidates.remove(existing); + } + candidates.add(candidate); + candidatesByPath.put(candidate.path(), candidate); + if (candidates.size() <= limit) return false; + MarkerCandidate removed = candidates.pollLast(); + if (removed != null) candidatesByPath.remove(removed.path()); + return true; + } + + private MarkerLaneKey key() { + return key; + } + + private List candidates() { + return List.copyOf(candidates); + } + } + + private static final class MarkerLaneCursor { + private final MarkerLaneKey key; + private final List candidates; + private int offset; + + private MarkerLaneCursor(BoundedMarkerLane lane) { + key = lane.key(); + candidates = lane.candidates(); + } + + private MarkerLaneKey key() { + return key; + } + + private MarkerCandidate first() { + return candidates.get(0); + } + + private String next(Set selected) { + while (offset < candidates.size()) { + String path = candidates.get(offset++).path(); + if (!selected.contains(path)) return path; + } + return null; + } + } + + private static final class PluginLaneCursor { + private final String pluginId; + private final List lanes; + private int laneOffset; + + private PluginLaneCursor( + String pluginId, + List lanes) { + this.pluginId = pluginId; + this.lanes = lanes; + } + + private String pluginId() { + return pluginId; + } + + private MarkerCandidate bestCandidate() { + return lanes.get(0).first(); + } + + private String next(Set selected) { + for (int attempts = 0; attempts < lanes.size(); attempts++) { + MarkerLaneCursor lane = lanes.get(laneOffset); + laneOffset = (laneOffset + 1) % lanes.size(); + String candidate = lane.next(selected); + if (candidate != null) return candidate; + } + return null; + } + } + public record SelectionPlan( String commit, List repositoryPaths, @@ -259,7 +768,8 @@ public record SelectionPlan( ProjectCapabilities preliminaryCapabilities, List enrichmentPaths, String projectType, - String sourceRoot) { + String sourceRoot, + boolean completeRepositoryInventory) { public SelectionPlan { if (commit == null || commit.isBlank()) { throw new IllegalArgumentException("selection commit is required"); diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/ProjectCapabilitySelectionServiceTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/ProjectCapabilitySelectionServiceTest.java index 18deaa81..bb3a2f75 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/ProjectCapabilitySelectionServiceTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/ProjectCapabilitySelectionServiceTest.java @@ -1,7 +1,13 @@ package org.rostilos.codecrow.pipelineagent.generic.service; import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.analysisengine.dto.request.ai.enrichment.FileContentDto; +import org.rostilos.codecrow.analysisengine.dto.request.ai.enrichment.PrEnrichmentDataDto; import org.rostilos.codecrow.core.model.project.config.AnalysisProfileConfig; +import org.rostilos.codecrow.plugins.CodeCrowPlugin; +import org.rostilos.codecrow.plugins.ContentMarker; +import org.rostilos.codecrow.plugins.ContentPatternMarker; +import org.rostilos.codecrow.plugins.DetectionAlternative; import org.rostilos.codecrow.plugins.DetectionRules; import org.rostilos.codecrow.plugins.FileDisposition; import org.rostilos.codecrow.plugins.FilePolicyPlugin; @@ -12,11 +18,15 @@ import org.rostilos.codecrow.plugins.PluginRuntime; import org.rostilos.codecrow.vcsclient.VcsClient; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -29,12 +39,13 @@ void application_host_starts_with_no_plugin_implementations_on_its_classpath() { } @Test - void plugin_policy_filters_generated_paths_before_enrichment() { + void plugin_policy_filters_generated_paths_before_enrichment() throws Exception { var runtime = new PluginRuntime(List.of(new FixturePolicyPlugin())); var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = unavailableInventoryClient(); var plan = service.plan( - mock(VcsClient.class), + vcsClient, "workspace", "repository", "0123456789abcdef", @@ -76,7 +87,7 @@ void automatic_selection_reads_exact_markers_below_configured_source_root() throws Exception { var runtime = new PluginRuntime(List.of(new RootedMarkerPlugin())); var service = new ProjectCapabilitySelectionService(runtime); - var vcsClient = mock(VcsClient.class); + var vcsClient = unavailableInventoryClient(); when(vcsClient.getFileContent( "workspace", "repository", @@ -109,7 +120,7 @@ void automatic_selection_reads_markers_below_inferred_nested_root() throws Exception { var runtime = new PluginRuntime(List.of(new RootedMarkerPlugin())); var service = new ProjectCapabilitySelectionService(runtime); - var vcsClient = mock(VcsClient.class); + var vcsClient = unavailableInventoryClient(); when(vcsClient.getFileContent( "workspace", "repository", @@ -138,6 +149,438 @@ void automatic_selection_reads_markers_below_inferred_nested_root() "0123456789abcdef"); } + @Test + void complete_inventory_selects_nested_django_from_unchanged_path_patterns() + throws Exception { + var runtime = new PluginRuntime(List.of( + languagePlugin("python", ".py"), + djangoPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = mock(VcsClient.class); + when(vcsClient.listRepositoryFiles( + "workspace", "repository", "0123456789abcdef", 500_000)) + .thenReturn(List.of( + "services/store/manage.py", + "services/store/store/settings.py", + "services/store/store/urls.py", + "services/store/catalog/views.py", + "unrelated/tool.py")); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("services/store/catalog/views.py")); + + assertThat(plan.preliminaryCapabilities().repositoryPlugins()) + .containsExactly("python", "django"); + assertThat(plan.preliminaryCapabilities().detectionEvidence().get("django")) + .contains( + "root:services/store", + "file:services/store/manage.py", + "pattern:**/settings.py:services/store/store/settings.py", + "pattern:**/urls.py:services/store/store/urls.py"); + assertThat(plan.preliminaryCapabilities().filePlugins()) + .containsOnlyKeys("services/store/catalog/views.py") + .containsEntry("services/store/catalog/views.py", List.of("python")); + assertThat(plan.enrichmentPaths()) + .containsExactly("services/store/catalog/views.py"); + } + + @Test + void complete_inventory_does_not_restore_a_deleted_changed_marker() + throws Exception { + var runtime = new PluginRuntime(List.of(new RootedMarkerPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = mock(VcsClient.class); + when(vcsClient.listRepositoryFiles( + "workspace", "repository", "0123456789abcdef", 500_000)) + .thenReturn(List.of("src/Thing.fixture")); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("framework.marker", "src/Thing.fixture")); + var enrichment = new PrEnrichmentDataDto( + List.of(FileContentDto.skipped("framework.marker", "deleted")), + List.of(), + List.of(), + PrEnrichmentDataDto.EnrichmentStats.empty()); + + assertThat(plan.repositoryPaths()).containsExactly("src/Thing.fixture"); + assertThat(plan.preliminaryCapabilities().repositoryPlugins()).isEmpty(); + assertThat(service.complete(plan, enrichment).repositoryPlugins()).isEmpty(); + verify(vcsClient, org.mockito.Mockito.never()).getFileContent( + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyString()); + } + + @Test + void complete_inventory_reads_only_existing_nested_quarkus_content_marker() + throws Exception { + var runtime = new PluginRuntime(List.of( + languagePlugin("java", ".java"), + quarkusPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = mock(VcsClient.class); + when(vcsClient.listRepositoryFiles( + "workspace", "repository", "0123456789abcdef", 500_000)) + .thenReturn(List.of( + "services/orders/pom.xml", + "services/orders/src/main/java/example/OrderResource.java", + "services/legacy/pom.xml")); + when(vcsClient.getFileContent( + "workspace", "repository", "services/orders/pom.xml", "0123456789abcdef")) + .thenReturn("io.quarkus"); + when(vcsClient.getFileContent( + "workspace", "repository", "services/legacy/pom.xml", "0123456789abcdef")) + .thenReturn("example"); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("services/orders/src/main/java/example/OrderResource.java")); + + assertThat(plan.preliminaryCapabilities().repositoryPlugins()) + .containsExactly("java", "quarkus"); + assertThat(plan.preliminaryCapabilities().detectionEvidence().get("quarkus")) + .contains("root:services/orders"); + assertThat(plan.preliminaryCapabilities().filePlugins()) + .containsOnlyKeys("services/orders/src/main/java/example/OrderResource.java"); + } + + @Test + void many_repository_roots_do_not_starve_the_changed_path_root() + throws Exception { + var runtime = new PluginRuntime(List.of( + languagePlugin("java", ".java"), + quarkusPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = mock(VcsClient.class); + var repositoryFiles = new ArrayList(); + IntStream.range(0, 100) + .mapToObj(index -> "service-%03d/pom.xml".formatted(index)) + .forEach(repositoryFiles::add); + repositoryFiles.add("zz-target/pom.xml"); + repositoryFiles.add("zz-target/src/main/java/example/OrderResource.java"); + when(vcsClient.listRepositoryFiles( + "workspace", "repository", "0123456789abcdef", 500_000)) + .thenReturn(repositoryFiles); + when(vcsClient.getFileContent( + "workspace", "repository", "zz-target/pom.xml", "0123456789abcdef")) + .thenReturn("io.quarkus"); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("zz-target/src/main/java/example/OrderResource.java")); + + assertThat(plan.preliminaryCapabilities().repositoryPlugins()) + .containsExactly("java", "quarkus"); + assertThat(plan.markerContents()) + .containsEntry("zz-target/pom.xml", "io.quarkus"); + verify(vcsClient).getFileContent( + "workspace", "repository", "zz-target/pom.xml", "0123456789abcdef"); + verify(vcsClient, times(64)).getFileContent( + org.mockito.ArgumentMatchers.eq("workspace"), + org.mockito.ArgumentMatchers.eq("repository"), + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.eq("0123456789abcdef")); + } + + @Test + void complete_inventory_selects_nested_rails_from_unchanged_markers() + throws Exception { + var runtime = new PluginRuntime(List.of( + languagePlugin("ruby", ".rb"), + railsPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = mock(VcsClient.class); + when(vcsClient.listRepositoryFiles( + "workspace", "repository", "0123456789abcdef", 500_000)) + .thenReturn(List.of( + "apps/billing/Gemfile", + "apps/billing/config/routes.rb", + "apps/billing/app/models/invoice.rb")); + when(vcsClient.getFileContent( + "workspace", "repository", "apps/billing/Gemfile", "0123456789abcdef")) + .thenReturn("gem \"rails\""); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("apps/billing/app/models/invoice.rb")); + + assertThat(plan.preliminaryCapabilities().repositoryPlugins()) + .containsExactly("ruby", "rails"); + assertThat(plan.preliminaryCapabilities().detectionEvidence().get("rails")) + .contains("root:apps/billing"); + assertThat(plan.preliminaryCapabilities().filePlugins()) + .containsOnlyKeys("apps/billing/app/models/invoice.rb"); + } + + @Test + void repository_inventory_failure_degrades_to_changed_file_detection() + throws Exception { + var runtime = new PluginRuntime(List.of( + languagePlugin("python", ".py"), + djangoPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = mock(VcsClient.class); + when(vcsClient.listRepositoryFiles( + "workspace", "repository", "0123456789abcdef", 500_000)) + .thenThrow(new IOException("tree temporarily unavailable")); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("services/store/catalog/views.py")); + + assertThat(plan.preliminaryCapabilities().repositoryPlugins()) + .containsExactly("python"); + assertThat(plan.preliminaryCapabilities().filePlugins()) + .containsOnlyKeys("services/store/catalog/views.py"); + assertThat(plan.enrichmentPaths()) + .containsExactly("services/store/catalog/views.py"); + } + + @Test + void degraded_inventory_retains_the_repository_root_quarkus_marker() + throws Exception { + var runtime = new PluginRuntime(List.of( + languagePlugin("java", ".java"), + quarkusPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = unavailableInventoryClient(); + when(vcsClient.getFileContent( + "workspace", "repository", "pom.xml", "0123456789abcdef")) + .thenReturn("io.quarkus"); + String deeplyNestedJava = String.join( + "/", + IntStream.range(0, 80) + .mapToObj(index -> "directory-%02d".formatted(index)) + .toList()) + "/OrderResource.java"; + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of(deeplyNestedJava)); + + assertThat(plan.preliminaryCapabilities().repositoryPlugins()) + .containsExactly("java", "quarkus"); + assertThat(plan.preliminaryCapabilities().detectionEvidence().get("quarkus")) + .contains("root:."); + verify(vcsClient).getFileContent( + "workspace", "repository", "pom.xml", "0123456789abcdef"); + } + + @Test + void degraded_inventory_retains_content_pattern_evidence_for_multiple_roots() + throws Exception { + var runtime = new PluginRuntime(List.of( + languagePlugin("ruby", ".rb"), + railsEnginePlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = unavailableInventoryClient(); + var changedFiles = List.of( + "engines/accounts/config/routes.rb", + "engines/accounts/accounts.gemspec", + "engines/accounts/lib/accounts/engine.rb", + "engines/billing/config/routes.rb", + "engines/billing/billing.gemspec", + "engines/billing/lib/billing/engine.rb"); + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + changedFiles); + var enrichment = new PrEnrichmentDataDto( + List.of( + FileContentDto.of( + "engines/billing/lib/billing/engine.rb", + "class BillingEngine < Rails::Engine; end"), + FileContentDto.of( + "engines/accounts/lib/accounts/engine.rb", + "class AccountsEngine < Rails::Engine; end")), + List.of(), + List.of(), + PrEnrichmentDataDto.EnrichmentStats.empty()); + + var capabilities = service.complete(plan, enrichment); + + assertThat(capabilities.repositoryPlugins()) + .containsExactly("ruby", "rails-engine"); + assertThat(capabilities.detectionEvidence().get("rails-engine")) + .contains( + "root:engines/accounts", + "root:engines/billing", + "content-pattern:lib/**/engine.rb:engines/accounts/lib/accounts/engine.rb:Rails::Engine", + "content-pattern:lib/**/engine.rb:engines/billing/lib/billing/engine.rb:Rails::Engine"); + } + + @Test + void automatic_selection_degrades_instead_of_failing_when_marker_file_budget_is_exceeded() + throws Exception { + var runtime = new PluginRuntime(List.of(new ManyMarkerPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = unavailableInventoryClient(); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("src/Thing.fixture")); + + assertThat(plan.preliminaryCapabilities().repositoryPlugins()).isEmpty(); + assertThat(plan.markerContents()).isEmpty(); + verify(vcsClient, times(64)).getFileContent( + org.mockito.ArgumentMatchers.eq("workspace"), + org.mockito.ArgumentMatchers.eq("repository"), + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.eq("0123456789abcdef")); + } + + @Test + void changed_marker_paths_are_prioritized_inside_the_bounded_schedule() + throws Exception { + var runtime = new PluginRuntime(List.of(new ManyMarkerPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = unavailableInventoryClient(); + when(vcsClient.getFileContent( + "workspace", + "repository", + "marker-69.file", + "0123456789abcdef")) + .thenReturn("present"); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("marker-69.file", "src/Thing.fixture")); + + assertThat(plan.preliminaryCapabilities().repositoryPlugins()) + .containsExactly("many-markers"); + assertThat(plan.markerContents()).containsKey("marker-69.file"); + } + + @Test + void automatic_selection_skips_oversized_marker_content_and_keeps_reading() + throws Exception { + var runtime = new PluginRuntime(List.of(new TwoMarkerPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = unavailableInventoryClient(); + when(vcsClient.getFileContent( + "workspace", "repository", "a.marker", "0123456789abcdef")) + .thenReturn("framework=true" + "x".repeat(300_000)); + when(vcsClient.getFileContent( + "workspace", "repository", "b.marker", "0123456789abcdef")) + .thenReturn("framework=true"); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("src/Thing.fixture")); + + assertThat(plan.markerContents()) + .containsOnlyKeys("b.marker") + .containsEntry("b.marker", "framework=true"); + assertThat(plan.markerBytes()).isEqualTo("framework=true".length()); + assertThat(plan.preliminaryCapabilities().repositoryPlugins()) + .containsExactly("two-markers"); + } + + @Test + void automatic_selection_degrades_when_an_optional_marker_cannot_be_read() + throws Exception { + var runtime = new PluginRuntime(List.of(new RootedMarkerPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = unavailableInventoryClient(); + when(vcsClient.getFileContent( + "workspace", + "repository", + "framework.marker", + "0123456789abcdef")) + .thenThrow(new IOException("temporary marker read failure")); + + var plan = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("src/Thing.fixture")); + + assertThat(plan.preliminaryCapabilities().repositoryPlugins()).isEmpty(); + assertThat(plan.markerContents()).isEmpty(); + } + + @Test + void completion_skips_pattern_content_that_exceeds_the_remaining_byte_budget() + throws Exception { + var runtime = new PluginRuntime(List.of(new PatternMarkerPlugin())); + var service = new ProjectCapabilitySelectionService(runtime); + var vcsClient = unavailableInventoryClient(); + var preliminary = service.plan( + vcsClient, + "workspace", + "repository", + "0123456789abcdef", + List.of("src/Thing.fixture")); + var marker = new ContentPatternMarker("**/*.fixture", "framework=true"); + var nearlyFull = new ProjectCapabilitySelectionService.SelectionPlan( + preliminary.commit(), + preliminary.repositoryPaths(), + Map.of("seed.marker", "x"), + List.of(marker), + 262_140, + preliminary.preliminaryCapabilities(), + preliminary.enrichmentPaths(), + preliminary.projectType(), + preliminary.sourceRoot(), + preliminary.completeRepositoryInventory()); + var enrichment = new PrEnrichmentDataDto( + List.of(FileContentDto.of("src/Thing.fixture", "framework=true")), + List.of(), + List.of(), + PrEnrichmentDataDto.EnrichmentStats.empty()); + + var capabilities = service.complete(nearlyFull, enrichment); + + assertThat(capabilities.repositoryPlugins()).isEmpty(); + } + + private static VcsClient unavailableInventoryClient() throws IOException { + var vcsClient = mock(VcsClient.class); + when(vcsClient.listRepositoryFiles( + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyString(), + org.mockito.ArgumentMatchers.anyInt())) + .thenThrow(new UnsupportedOperationException( + "repository inventory unavailable")); + return vcsClient; + } + private static final class FixturePolicyPlugin implements FilePolicyPlugin { private final PluginDescriptor descriptor = new PluginDescriptor( "fixture-policy", @@ -162,6 +605,104 @@ public PluginOutcome fileDisposition(String normalizedPath) { } } + private static CodeCrowPlugin languagePlugin(String id, String extension) { + return descriptorPlugin(new PluginDescriptor( + id, + PluginKind.LANGUAGE, + List.of(), + List.of(PluginCapability.CONTEXT), + new DetectionRules( + List.of(extension), List.of(), List.of(), List.of(), List.of()), + Map.of())); + } + + private static CodeCrowPlugin djangoPlugin() { + return descriptorPlugin(new PluginDescriptor( + "django", + PluginKind.FRAMEWORK, + List.of("python"), + List.of(PluginCapability.CONTEXT), + new DetectionRules( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new DetectionAlternative( + List.of("manage.py"), + List.of(), + List.of("**/settings.py", "**/urls.py"), + List.of(), + List.of()))), + Map.of())); + } + + private static CodeCrowPlugin quarkusPlugin() { + return descriptorPlugin(new PluginDescriptor( + "quarkus", + PluginKind.FRAMEWORK, + List.of("java"), + List.of(PluginCapability.CONTEXT), + new DetectionRules( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new DetectionAlternative( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new ContentMarker("pom.xml", "io.quarkus"))))), + Map.of())); + } + + private static CodeCrowPlugin railsPlugin() { + return descriptorPlugin(new PluginDescriptor( + "rails", + PluginKind.FRAMEWORK, + List.of("ruby"), + List.of(PluginCapability.CONTEXT), + new DetectionRules( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new DetectionAlternative( + List.of("Gemfile", "config/routes.rb"), + List.of(), + List.of(), + List.of(), + List.of(new ContentMarker("Gemfile", "gem \"rails\""))))), + Map.of())); + } + + private static CodeCrowPlugin railsEnginePlugin() { + return descriptorPlugin(new PluginDescriptor( + "rails-engine", + PluginKind.FRAMEWORK, + List.of("ruby"), + List.of(PluginCapability.CONTEXT), + new DetectionRules( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new DetectionAlternative( + List.of("config/routes.rb"), + List.of(), + List.of("*.gemspec"), + List.of(), + List.of(), + List.of(new ContentPatternMarker( + "lib/**/engine.rb", + "Rails::Engine"))))), + Map.of())); + } + + private static CodeCrowPlugin descriptorPlugin(PluginDescriptor descriptor) { + return () -> descriptor; + } + private static final class RootedMarkerPlugin implements FilePolicyPlugin { private final PluginDescriptor descriptor = new PluginDescriptor( "rooted-marker", @@ -186,4 +727,89 @@ public PluginOutcome fileDisposition(String normalizedPath) { return PluginOutcome.handled(FileDisposition.FULL); } } + + private static final class ManyMarkerPlugin implements FilePolicyPlugin { + private final PluginDescriptor descriptor = new PluginDescriptor( + "many-markers", + PluginKind.LANGUAGE, + List.of(), + List.of(PluginCapability.FILE_POLICY), + new DetectionRules( + List.of(), + List.of(), + IntStream.range(0, 70) + .mapToObj(index -> "marker-%02d.file".formatted(index)) + .toList(), + List.of(), + List.of()), + Map.of()); + + @Override + public PluginDescriptor descriptor() { + return descriptor; + } + + @Override + public PluginOutcome fileDisposition(String normalizedPath) { + return PluginOutcome.handled(FileDisposition.FULL); + } + } + + private static final class TwoMarkerPlugin implements FilePolicyPlugin { + private final PluginDescriptor descriptor = new PluginDescriptor( + "two-markers", + PluginKind.LANGUAGE, + List.of(), + List.of(PluginCapability.FILE_POLICY), + new DetectionRules( + List.of(), + List.of("a.marker", "b.marker"), + List.of(), + List.of(), + List.of()), + Map.of()); + + @Override + public PluginDescriptor descriptor() { + return descriptor; + } + + @Override + public PluginOutcome fileDisposition(String normalizedPath) { + return PluginOutcome.handled(FileDisposition.FULL); + } + } + + private static final class PatternMarkerPlugin implements FilePolicyPlugin { + private final PluginDescriptor descriptor = new PluginDescriptor( + "pattern-marker", + PluginKind.LANGUAGE, + List.of(), + List.of(PluginCapability.FILE_POLICY), + new DetectionRules( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new DetectionAlternative( + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new ContentPatternMarker( + "**/*.fixture", + "framework=true"))))), + Map.of()); + + @Override + public PluginDescriptor descriptor() { + return descriptor; + } + + @Override + public PluginOutcome fileDisposition(String normalizedPath) { + return PluginOutcome.handled(FileDisposition.FULL); + } + } } diff --git a/python-ecosystem/inference-orchestrator/src/Dockerfile b/python-ecosystem/inference-orchestrator/src/Dockerfile index 70a4d265..91cf28a3 100644 --- a/python-ecosystem/inference-orchestrator/src/Dockerfile +++ b/python-ecosystem/inference-orchestrator/src/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && \ # Install Python dependencies RUN pip install --no-cache-dir -r requirements.txt && \ pip check && \ - python -c "from mcp.shared.context import RequestContext; from mcp_use import MCPClient; import tree_sitter_typescript" + python -c "from mcp.shared.context import RequestContext; from mcp_use import MCPClient; import tree_sitter_python; import tree_sitter_ruby; import tree_sitter_typescript" # --- Builder Stage 2: Copy and Install Application Modules --- diff --git a/python-ecosystem/inference-orchestrator/src/Dockerfile.observable b/python-ecosystem/inference-orchestrator/src/Dockerfile.observable index 7b998e36..4198a15b 100644 --- a/python-ecosystem/inference-orchestrator/src/Dockerfile.observable +++ b/python-ecosystem/inference-orchestrator/src/Dockerfile.observable @@ -18,7 +18,7 @@ RUN apt-get update && \ # when the MCP client or a runtime plugin parser cannot be imported. RUN pip install --no-cache-dir -r requirements.txt && \ pip check && \ - python -c "from mcp.shared.context import RequestContext; from mcp_use import MCPClient; import tree_sitter_typescript" + python -c "from mcp.shared.context import RequestContext; from mcp_use import MCPClient; import tree_sitter_python; import tree_sitter_ruby; import tree_sitter_typescript" # --- Builder Stage 2: Copy and Install Application Modules --- diff --git a/python-ecosystem/inference-orchestrator/src/requirements.txt b/python-ecosystem/inference-orchestrator/src/requirements.txt index 2c2209ab..ce14fbec 100644 --- a/python-ecosystem/inference-orchestrator/src/requirements.txt +++ b/python-ecosystem/inference-orchestrator/src/requirements.txt @@ -19,8 +19,10 @@ newrelic==11.5.0 # Runtime validation executes installed analysis plugins against # exact enriched source. These native wheels must match the RAG parser ABI. tree-sitter==0.25.2 +tree-sitter-python==0.25.0 tree-sitter-java==0.23.5 tree-sitter-javascript==0.25.0 tree-sitter-typescript==0.23.2 tree-sitter-go==0.25.0 tree-sitter-php==0.24.1 +tree-sitter-ruby==0.23.1 diff --git a/python-ecosystem/inference-orchestrator/tests/test_plugin_context.py b/python-ecosystem/inference-orchestrator/tests/test_plugin_context.py index 045b55ce..ef6bf8d4 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_plugin_context.py +++ b/python-ecosystem/inference-orchestrator/tests/test_plugin_context.py @@ -135,7 +135,7 @@ def _capabilities_dto( file_plugins, ) -> ProjectCapabilitiesDto: catalog, _, selector = plugin_context._plugin_host() - from codecrow_plugins import ProjectCapabilities + from codecrow_plugins import PluginKind, ProjectCapabilities repository_plugins = tuple(repository_plugins) normalized_files = { @@ -143,7 +143,13 @@ def _capabilities_dto( for path, plugin_ids in file_plugins.items() } evidence = { - plugin_id: (f"fixture:{plugin_id}",) + plugin_id: tuple(sorted(( + f"fixture:{plugin_id}", + *(("root:.",) if ( + catalog.registry.descriptor(plugin_id).kind + is PluginKind.FRAMEWORK + ) else ()), + ))) for plugin_id in repository_plugins } fingerprint = selector._fingerprint( diff --git a/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py b/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py index df716b57..b2c442a7 100644 --- a/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py +++ b/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py @@ -1062,6 +1062,107 @@ def test_plugin_selection_transition_fails_before_qdrant_mutation(tmp_path): point_ops.client.get_collection(collection) +@pytest.mark.parametrize("degradation", ("byte-budget", "invalid-utf8")) +def test_degraded_marker_inspection_preserves_incremental_plugin_selection( + tmp_path, + degradation, +): + catalog = discover_builtin_plugins() + selector = ProjectSelector(catalog.registry) + (tmp_path / "app.py").write_text( + "from fastapi import FastAPI\n", + encoding="utf-8", + ) + marker = tmp_path / "requirements.txt" + marker.write_text("fastapi\n", encoding="utf-8") + facts = build_repository_facts( + tmp_path, + "base", + ("app.py", "requirements.txt"), + catalog.registry, + ) + capabilities = selector.select(facts) + assert "fastapi" in capabilities.repository_plugins + + client = QdrantClient(":memory:") + collection = "repository" + client.create_collection( + collection_name=collection, + vectors_config=VectorParams(size=4, distance=Distance.COSINE), + ) + point_ops = PointOperations( + client, + _StaticEmbedding(), + batch_size=50, + embedding_dim=4, + ) + state_nodes = RepositoryIndexer._repository_facts_nodes( + facts, + capabilities, + "ws", + "project", + "main", + "base", + catalog.implementation_fingerprint(capabilities.repository_plugins), + ) + point_ops.process_and_upsert_chunks( + state_nodes, + collection, + "ws", + "project", + "main", + ) + + if degradation == "byte-budget": + marker.write_text("fastapi\n" + "#" * 300_000, encoding="utf-8") + else: + marker.write_bytes(b"\xff\xfe") + + plugin_runtime = MagicMock(spec=PluginRuntime) + plugin_runtime.repository_analysis_plugins.return_value = () + plugin_runtime.file_disposition.return_value = FileDisposition.FULL + splitter = _splitter_mock() + splitter.split_documents.side_effect = lambda documents, capabilities: [ + TextNode(text=document.text, metadata=dict(document.metadata)) + for document in documents + ] + operations = FileOperations( + client, + point_ops, + CollectionManager(client, 4), + MagicMock(), + splitter, + DocumentLoader(SimpleNamespace( + excluded_patterns=(), + max_file_size_bytes=1_000_000, + )), + plugin_catalog=catalog, + plugin_runtime=plugin_runtime, + plugin_selector=selector, + ) + + operations.apply_changes( + ["requirements.txt"], + [], + str(tmp_path), + "ws", + "project", + "main", + "changed", + collection, + ) + + stored_facts, plugin_ids, *_identity = load_repository_facts( + client, + collection, + "main", + ) + assert stored_facts.marker_contents == { + "requirements.txt": "fastapi\n", + } + assert "fastapi" in plugin_ids + + def test_plugin_deactivation_from_mixed_update_and_delete_requires_full_reindex( tmp_path, ): From 4b596494a77b6eff19dac7d9755d5de4547b2ed9 Mon Sep 17 00:00:00 2001 From: rostislav Date: Fri, 21 Aug 2026 10:18:59 +0300 Subject: [PATCH 2/2] Correcting a Stuck Primary RAG Index & related plugins --- README.md | 14 +- .../python/codecrow_plugins/runtime.py | 130 ++++++- .../tests/test_magento_repository_analysis.py | 45 +++ .../test_repository_runtime_resilience.py | 70 ++++ .../codecrow_plugin_magento/repository.py | 132 ++++++- .../java-shared/application.properties.sample | 16 +- deployment/config/rag-pipeline/.env.sample | 11 + ...BranchIndexBuildExecutorConfiguration.java | 2 +- .../branch/BranchIndexMaintenanceService.java | 9 +- .../RagIndexOperationHeartbeatService.java | 24 +- .../ragengine/client/RagPipelineClient.java | 83 ++++- .../client/RagPipelineClientTest.java | 36 ++ .../src/service/rag/rag_client.py | 29 +- .../tests/test_rag_client.py | 12 + .../tests/test_rag_index_timeout.py | 4 +- .../src/rag_pipeline/api/routers/index.py | 51 +++ .../src/rag_pipeline/api/routers/system.py | 6 +- .../core/index_manager/indexer.py | 347 ++++++++++++------ .../core/index_manager/manager.py | 107 +++++- .../core/index_manager/point_operations.py | 114 ++++++ .../src/rag_pipeline/models/config.py | 17 + .../rag-pipeline/tests/test_config.py | 18 + .../rag-pipeline/tests/test_index_manager.py | 41 +++ .../rag-pipeline/tests/test_indexer.py | 75 +++- .../tests/test_point_operations.py | 64 ++++ .../rag-pipeline/tests/test_router_index.py | 64 ++++ .../rag-pipeline/tests/test_routers.py | 4 +- 27 files changed, 1373 insertions(+), 152 deletions(-) diff --git a/README.md b/README.md index 99bd36fa..f2f7f334 100644 --- a/README.md +++ b/README.md @@ -115,8 +115,9 @@ below. | Go (`.go`) | ✅ | ✅ | ✅ | ✅ | | PHP / PHTML (`.php`, `.inc`, `.phtml`) | ✅ | ✅ | ✅ | ✅ | | C# (`.cs`), Rust (`.rs`) | ✅ | ✅ | ✅ | — | -| TSX (`.tsx`) | ✅ | ✅ | ✅ | — | -| Bash / Shell, C, C++, CSS, Haskell, HTML, JSON, Ruby, Scala | ✅ | ✅ | generic | — | +| TSX (`.tsx`) | ✅ | ✅ | ✅ | framework-dependent | +| Ruby (`.rb`) | ✅ | ✅ | generic | framework-dependent | +| Bash / Shell, C, C++, CSS, Haskell, HTML, JSON, Scala | ✅ | ✅ | generic | — | | Kotlin, Swift, Lua, Perl, COBOL, Objective-C, SQL, R, SCSS, Vue/Svelte SFCs, YAML/TOML/XML, Markdown/RST, and other text | ✅ | fallback | generic | — | `generic` means language-aware or text chunking without a dedicated semantic RAG @@ -124,6 +125,9 @@ query. TSX uses the TypeScript RAG parser/query compatibility path but is not included in the TypeScript repository-fact session. C, C++, and Ruby ship RAG parser packages but currently have no dedicated semantic query, so the table reports their resulting generic chunk behavior rather than package availability. +`framework-dependent` means the base language tier does not emit those facts, +but a selected framework plugin does. Ember can additionally enrich conservative +`.hbs` template structure; this is not a general Handlebars language plugin. Exact plugin facts are bounded, typed declarations and relationships. JavaScript, TypeScript, and PHP maintain repository-scoped resolution sessions; Go, Java, and @@ -135,9 +139,15 @@ replaces model review with a preset defect-rule engine. | Plugin | Requires | Deterministic Context | | :------------- | :--------------- | :--------------------------------------------------------------------------------------------------------------------------------------------- | | Spring | Java | Components, combined controller routes, dependency injection, beans, configuration, and Spring Data repository inheritance | +| Quarkus | Java | CDI beans/injection, JAX-RS resources/routes, configuration-property uses and key/profile metadata, schedules, channels, and Panache topology | | FastAPI | Python | Applications, routers and route prefixes, HTTP/WebSocket routes, `Depends`, middleware, lifespan handlers, and exception handlers | +| Django | Python | AppConfig, installed apps/middleware/root URL configuration, URL paths/includes, views, models/relations, middleware hooks, and signal receivers | +| Ember.js | `json` (auto-detected via `package.json`) | Router maps, route/controller/component/service/model roles, service injection, Ember Data relationships, and `.hbs` ownership/invocations | +| Express.js | `json` (auto-detected via `package.json`) | Applications and routers, HTTP routes, router mounts, middleware, and error-handler topology from JS/TS source | +| Next.js | `json` (auto-detected via `package.json`) | File-system pages and routes, HTTP handlers, layouts, middleware, client/server boundaries, Server Actions, and data loaders from JS/TS source | | Magento 2 | PHP | Module topology, DI and plugins, events, routes and ACLs, layouts, blocks, templates and themes, Web APIs, queues, schemas, and related source | | Hyvä | Magento 2 | ViewModel registry, layout/template, Alpine state/event, REST/Web API, DI, and bounded PHP call-chain relations | +| Ruby on Rails | Ruby | Routes/mounts, controllers/actions, models, associations/callbacks, and Active Job queues, `perform`, retry, and discard declarations | | Data contracts | Language-neutral | Exact GraphQL, Protocol Buffers, JSON Schema, and explicit contract-path field declarations and references across languages | Plugins are selected automatically from bounded facts at the pinned repository diff --git a/analysis-plugins/contracts/python/codecrow_plugins/runtime.py b/analysis-plugins/contracts/python/codecrow_plugins/runtime.py index 0ad5b7fa..2af1d647 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/runtime.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/runtime.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import time +from typing import Callable from .api import ( ArchitecturePacket, @@ -624,7 +626,26 @@ def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None: retained.append((plugin_id, session)) self._sessions = retained - def finish(self) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: + @staticmethod + def _report_progress( + callback: Callable[[dict[str, object]], None] | None, + event: dict[str, object], + ) -> None: + if callback is None: + return + try: + callback(event) + except Exception: + # Repository progress is optional host observability. A broken + # observer must not change the plugin composition result. + return + + def finish( + self, + *, + progress_callback: Callable[[dict[str, object]], None] | None = None, + deadline: float | None = None, + ) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: if self._finished: raise RuntimeError("repository analysis is already finished") self._finished = True @@ -634,18 +655,125 @@ def finish(self) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: contexts: dict[tuple[str, str, str], RepositoryContext] = {} current = RepositoryAnalysis() for plugin_id, session in self._sessions: + if deadline is not None and time.monotonic() >= deadline: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-finalization-timeout", + message=( + "repository analysis time budget was exhausted before " + f"finalizing {plugin_id}" + ), + plugin_id=plugin_id, + recoverable=True, + )) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "timed_out", + "message": ( + f"Architecture finalization timed out before {plugin_id}" + ), + }) + break + + plugin_started = time.monotonic() + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "started", + "message": f"Finalizing {plugin_id} repository architecture", + }) try: + configure_progress = getattr( + session, + "set_progress_callback", + None, + ) + if callable(configure_progress): + configure_progress(progress_callback) + configure_deadline = getattr( + session, + "set_analysis_deadline", + None, + ) + if callable(configure_deadline): + configure_deadline(deadline) outcome = session.finish(current) + except TimeoutError as exception: + duration_ms = round((time.monotonic() - plugin_started) * 1000) + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-finalization-timeout", + message=str(exception), + plugin_id=plugin_id, + recoverable=True, + )) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "timed_out", + "durationMs": duration_ms, + "message": ( + f"Architecture finalization timed out in {plugin_id} " + f"after {duration_ms} ms" + ), + }) + break except Exception as exception: + duration_ms = round((time.monotonic() - plugin_started) * 1000) self._diagnostics.append(PluginDiagnostic( code="plugin-repository-finish-exception", message=f"{type(exception).__name__}: {exception}", plugin_id=plugin_id, + recoverable=True, )) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "failed", + "durationMs": duration_ms, + "message": ( + f"Architecture finalization failed in {plugin_id} " + f"after {duration_ms} ms" + ), + }) continue + duration_ms = round((time.monotonic() - plugin_started) * 1000) + if deadline is not None and time.monotonic() >= deadline: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-finalization-timeout", + message=( + f"{plugin_id} repository analysis exceeded the shared " + "time budget" + ), + plugin_id=plugin_id, + recoverable=True, + )) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "timed_out", + "durationMs": duration_ms, + "message": ( + f"Architecture finalization timed out in {plugin_id} " + f"after {duration_ms} ms" + ), + }) + break if outcome.status is OutcomeStatus.FAILED: self._diagnostics.append(outcome.diagnostic) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "failed", + "durationMs": duration_ms, + "message": ( + f"Architecture finalization failed in {plugin_id} " + f"after {duration_ms} ms" + ), + }) continue + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "completed", + "durationMs": duration_ms, + "message": ( + f"Finalized {plugin_id} repository architecture in " + f"{duration_ms} ms" + ), + }) if outcome.status is not OutcomeStatus.HANDLED: continue contribution = outcome.value diff --git a/analysis-plugins/contracts/python/tests/test_magento_repository_analysis.py b/analysis-plugins/contracts/python/tests/test_magento_repository_analysis.py index 8ae3cac5..42bc4150 100644 --- a/analysis-plugins/contracts/python/tests/test_magento_repository_analysis.py +++ b/analysis-plugins/contracts/python/tests/test_magento_repository_analysis.py @@ -390,6 +390,51 @@ def _resolve( return outcome.value +def test_magento_repository_reports_timed_substages(): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + plugin = catalog.implementation("magento") + session = plugin.start_repository_analysis("progress-test").value + session.ingest((FileArtifact( + "app/code/Acme/Checkout/etc/module.xml", + '', + ),)) + events = [] + session.set_progress_callback(events.append) + + outcome = session.finish(RepositoryAnalysis()) + + assert outcome.status is OutcomeStatus.HANDLED + assert any( + event.get("substage") == "module discovery" + and event.get("status") == "started" + for event in events + ) + assert any( + event.get("substage") == "packet materialization" + and event.get("status") == "completed" + and isinstance(event.get("durationMs"), int) + for event in events + ) + + +def test_magento_repository_honors_host_finalization_deadline(): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + plugin = catalog.implementation("magento") + session = plugin.start_repository_analysis("timeout-test").value + session.ingest((FileArtifact( + "app/code/Acme/Checkout/etc/module.xml", + '', + ),)) + session.set_analysis_deadline(0.0) + + try: + session.finish(RepositoryAnalysis()) + except TimeoutError as exception: + assert "module discovery" in str(exception) + else: + raise AssertionError("expired architecture deadline was not enforced") + + def _factory_artifacts( *, checkout_enabled: bool = True, diff --git a/analysis-plugins/contracts/python/tests/test_repository_runtime_resilience.py b/analysis-plugins/contracts/python/tests/test_repository_runtime_resilience.py index 78b4f7ef..81e88765 100644 --- a/analysis-plugins/contracts/python/tests/test_repository_runtime_resilience.py +++ b/analysis-plugins/contracts/python/tests/test_repository_runtime_resilience.py @@ -1,4 +1,5 @@ from types import SimpleNamespace +import time from codecrow_plugins import ( FileArtifact, @@ -57,3 +58,72 @@ def test_repository_runtime_quarantines_ingest_failure_and_keeps_session(): ("plugin-repository-file-skipped", "bad.xml", True), ("project-warning", "warning.xml", True), ] + + +class _TimedRepositorySession: + def __init__(self): + self.progress_callback = None + self.deadline = None + + def set_progress_callback(self, callback): + self.progress_callback = callback + + def set_analysis_deadline(self, deadline): + self.deadline = deadline + + def finish(self, _dependencies): + raise TimeoutError("test plugin exhausted the architecture budget") + + +def test_repository_runtime_reports_timeout_as_recoverable_and_stops(): + session = _TimedRepositorySession() + later = _FileIsolatingSession() + runtime = SimpleNamespace( + MAX_REPOSITORY_SYMBOLS=10, + MAX_ARCHITECTURE_PACKETS=10, + ) + events = [] + deadline = time.monotonic() + 60 + handle = RepositoryAnalysisHandle( + runtime, + [("timed-plugin", session), ("later-plugin", later)], + [], + ) + + analysis, diagnostics = handle.finish( + progress_callback=events.append, + deadline=deadline, + ) + + assert analysis == RepositoryAnalysis() + assert session.progress_callback is not None + assert session.deadline == deadline + assert [(item.code, item.recoverable) for item in diagnostics] == [ + ("plugin-repository-finalization-timeout", True), + ] + assert [event["status"] for event in events] == ["started", "timed_out"] + assert later.ingested == [] + + +def test_repository_runtime_discards_result_that_returns_after_deadline( + monkeypatch, +): + session = _FileIsolatingSession() + runtime = SimpleNamespace( + MAX_REPOSITORY_SYMBOLS=10, + MAX_ARCHITECTURE_PACKETS=10, + ) + handle = RepositoryAnalysisHandle( + runtime, + [("slow-plugin", session)], + [], + ) + readings = iter((0.0, 1.0, 10.0, 10.0)) + monkeypatch.setattr(time, "monotonic", lambda: next(readings)) + + analysis, diagnostics = handle.finish(deadline=5.0) + + assert analysis == RepositoryAnalysis() + assert [(item.code, item.recoverable) for item in diagnostics] == [ + ("plugin-repository-finalization-timeout", True), + ] diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/repository.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/repository.py index c855b35f..ef72c702 100644 --- a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/repository.py +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/repository.py @@ -8,6 +8,7 @@ import time from dataclasses import dataclass, field, replace from pathlib import PurePosixPath +from typing import Callable from codecrow_plugins import ( ArchitecturePacket, @@ -186,10 +187,14 @@ def __init__( plugin_id: str, artifacts: dict[str, str], symbols: tuple[SymbolDefinition, ...], + progress_callback: Callable[[dict[str, object]], None] | None = None, + deadline: float | None = None, ) -> None: self.plugin_id = plugin_id self.artifacts = artifacts self.symbols = symbols + self.progress_callback = progress_callback + self.deadline = deadline self.symbols_by_name: dict[str, tuple[SymbolDefinition, ...]] = {} self.symbols_by_casefold: dict[str, tuple[SymbolDefinition, ...]] = {} for symbol in symbols: @@ -216,15 +221,90 @@ def __init__( self._acl_sources: dict[str, set[str]] = {} self._admin_controller_sources: dict[str, set[str]] = {} + def _report_progress(self, event: dict[str, object]) -> None: + if self.progress_callback is None: + return + try: + self.progress_callback(event) + except Exception: + # Repository progress is optional host observability. + return + + def _check_deadline(self, stage_name: str) -> None: + if self.deadline is not None and time.monotonic() >= self.deadline: + raise TimeoutError( + "Magento repository architecture exceeded its time budget " + f"during {stage_name}" + ) + + def _run_stage(self, stage_name: str, stage): + self._check_deadline(stage_name) + started = time.monotonic() + self._report_progress({ + "pluginId": self.plugin_id, + "substage": stage_name, + "status": "started", + "message": f"Building Magento architecture: {stage_name}", + }) + try: + result = stage() + self._check_deadline(stage_name) + except Exception as exception: + duration_ms = round((time.monotonic() - started) * 1000) + status = "timed_out" if isinstance(exception, TimeoutError) else "failed" + logger.warning( + "Magento architecture substage %s status=%s duration_ms=%s: %s", + stage_name, + status, + duration_ms, + exception, + ) + self._report_progress({ + "pluginId": self.plugin_id, + "substage": stage_name, + "status": status, + "durationMs": duration_ms, + "message": ( + f"Magento architecture {stage_name} {status.replace('_', ' ')} " + f"after {duration_ms} ms" + ), + }) + raise + duration_ms = round((time.monotonic() - started) * 1000) + logger.info( + "Magento architecture substage %s status=completed duration_ms=%s", + stage_name, + duration_ms, + ) + self._report_progress({ + "pluginId": self.plugin_id, + "substage": stage_name, + "status": "completed", + "durationMs": duration_ms, + "message": ( + f"Built Magento architecture: {stage_name} in {duration_ms} ms" + ), + }) + return result + def resolve(self) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: started = time.monotonic() - modules = self._modules() + modules = self._run_stage("module discovery", self._modules) if not modules: return RepositoryAnalysis(), tuple(self._diagnostics) try: - self._module_packets(modules) - themes = self._themes(modules) - di_states = self._di(modules) + self._run_stage( + "module packets", + lambda: self._module_packets(modules), + ) + themes = self._run_stage( + "theme discovery", + lambda: self._themes(modules), + ) + di_states = self._run_stage( + "dependency injection", + lambda: self._di(modules), + ) stages = ( ("constructor/DI", lambda: self._constructor_packets( modules, @@ -277,7 +357,9 @@ def resolve(self) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: ) for stage_name, stage in stages: try: - stage() + self._run_stage(stage_name, stage) + except TimeoutError: + raise except Exception as exception: raise RuntimeError( f"Magento {stage_name} enrichment failed: " @@ -285,7 +367,7 @@ def resolve(self) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: ) from exception except RuntimeError: raise - packets = self.graph.build() + packets = self._run_stage("packet materialization", self.graph.build) logger.info( "Magento repository resolution: modules=%s packets=%s elapsed=%.3fs", len(modules), @@ -7084,6 +7166,16 @@ class MagentoRepositorySession: revision: str artifacts: dict[str, str] = field(default_factory=dict) source_root: str | None = None + progress_callback: Callable[[dict[str, object]], None] | None = field( + default=None, + repr=False, + compare=False, + ) + analysis_deadline: float | None = field( + default=None, + repr=False, + compare=False, + ) @classmethod def restore(cls, plugin_id: str, revision: str, snapshots) -> "MagentoRepositorySession": @@ -7121,6 +7213,27 @@ def _snapshot(self) -> RepositorySnapshot: def set_source_root(self, source_root: str | None) -> None: self.source_root = source_root + def set_progress_callback( + self, + progress_callback: Callable[[dict[str, object]], None] | None, + ) -> None: + self.progress_callback = progress_callback + + def set_analysis_deadline(self, deadline: float | None) -> None: + self.analysis_deadline = deadline + + def _report_scoped_progress( + self, + root: str, + event: dict[str, object], + ) -> None: + if self.progress_callback is None: + return + self.progress_callback({ + **event, + "sourceRoot": root or ".", + }) + def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None: for artifact in artifacts: path = artifact.path @@ -7183,6 +7296,13 @@ def finish(self, dependencies: RepositoryAnalysis): self.plugin_id, scoped, scoped_symbols, + progress_callback=( + lambda event, root=root: self._report_scoped_progress( + root, + event, + ) + ), + deadline=self.analysis_deadline, ) analysis, scoped_diagnostics = resolver.resolve() analyses.append(self._prefix_analysis(analysis, root, scoped_paths)) diff --git a/deployment/config/java-shared/application.properties.sample b/deployment/config/java-shared/application.properties.sample index 245f789c..f92211a6 100644 --- a/deployment/config/java-shared/application.properties.sample +++ b/deployment/config/java-shared/application.properties.sample @@ -181,14 +181,14 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF #codecrow.rag.api.timeout.read=120 # RAG indexing timeout - 4 hours for large repositories #codecrow.rag.api.timeout.indexing=14400 -# Per-project fan-out for an explicit Refresh all. Two independent branch -# snapshots (for example main and develop) run in parallel without allowing a -# project with many retained branches to occupy all service slots. -#codecrow.rag.branch-build.parallelism=2 -# Dedicated service-wide capacity for full branch snapshot builds. This pool is -# separate from PR, branch-analysis, webhook and inference executors. Size it to -# the available RAG replicas and memory; it does not cap ordinary analyses. -#codecrow.rag.branch-build.global-parallelism=4 +# SSE exact-branch streams emit heartbeats every 15 seconds; fail a silent +# transport after one minute while allowing an active stream to run indefinitely. +#codecrow.rag.api.timeout.stream-idle=60 +# Exact configured-branch builds are serialized by default because architecture +# finalization shares CPU, memory, Qdrant, and embedding clients in one RAG process. +# Raise these only after provisioning independent capacity for concurrent builds. +#codecrow.rag.branch-build.parallelism=1 +#codecrow.rag.branch-build.global-parallelism=1 # Repair interval for readable Qdrant current-branch aliases of active generations. #codecrow.rag.operator-alias.reconcile-interval-ms=300000 #codecrow.rag.operator-alias.reconcile-initial-delay-ms=15000 diff --git a/deployment/config/rag-pipeline/.env.sample b/deployment/config/rag-pipeline/.env.sample index 936b514f..b9f5466e 100644 --- a/deployment/config/rag-pipeline/.env.sample +++ b/deployment/config/rag-pipeline/.env.sample @@ -52,10 +52,21 @@ SERVICE_SECRET=change-me-to-a-random-secret # QDRANT_TIMEOUT_SECONDS=30 # QDRANT_VECTORS_ON_DISK=true # QDRANT_UPSERT_BATCH_SIZE=128 +# Cap each serialized Qdrant request well below the server's 32 MiB default. +# QDRANT_UPSERT_MAX_PAYLOAD_BYTES=8388608 # === Queue and Server Runtime === # REDIS_URL=redis://redis:6379/1 # MAX_CONCURRENT_RAG_JOBS=2 +# Full repository builds from Redis and HTTP/SSE share this process-wide slot. +# Keep it at one unless each concurrent build has independent RAG capacity. +# RAG_FULL_INDEX_CONCURRENCY=1 +# Heartbeat frames keep quiet HTTP/SSE exact-branch builds observable and reset +# the Java client's read-idle timeout without changing durable job progress. +# RAG_INDEX_STREAM_HEARTBEAT_SECONDS=15 +# Repository-plugin finalization is optional enrichment. At this deadline it is +# omitted and semantic indexing continues with a recoverable diagnostic. +# RAG_ARCHITECTURE_FINALIZATION_TIMEOUT_SECONDS=600 # Keep one API process by default: every worker loads its own embedding/indexing state. # Increase only when the host has enough memory for another complete runtime. # UVICORN_WORKERS=1 diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java index b014abc3..5c9b9ead 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java @@ -13,7 +13,7 @@ public class BranchIndexBuildExecutorConfiguration { @Bean(name = "branchIndexBuildExecutor") public Executor branchIndexBuildExecutor( - @Value("${codecrow.rag.branch-build.global-parallelism:4}") int parallelism) { + @Value("${codecrow.rag.branch-build.global-parallelism:1}") int parallelism) { int workers = Math.max(1, parallelism); ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(workers); diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java index 42361f43..9cb7be5a 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java @@ -63,7 +63,7 @@ public BranchIndexMaintenanceService( AnalysisLockService lockService, AnalysisJobService jobService, @Qualifier("branchIndexBuildExecutor") Executor branchIndexBuildExecutor, - @Value("${codecrow.rag.branch-build.parallelism:2}") int perProjectParallelism) { + @Value("${codecrow.rag.branch-build.parallelism:1}") int perProjectParallelism) { this.ragOperationsService = ragOperationsService; this.vcsClientProvider = vcsClientProvider; this.generationBuildService = generationBuildService; @@ -83,9 +83,10 @@ public Map rebuild(Project project, String requestedBranch, bool // Obtaining a provider client can refresh a shared installation token. Do // that small VCS preparation phase once at a time, then let the expensive - // archive download and RAG mutation for every resolved branch run in - // parallel. Concurrent token refreshes previously allowed one branch to - // disappear before it had a durable job or operation to report. + // archive download and RAG mutation for every resolved branch run under + // the bounded branch-build capacity. Concurrent token refreshes previously + // allowed one branch to disappear before it had a durable job or operation + // to report. List plans = new ArrayList<>(); for (String branch : branches) { try { diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java index 220e8552..e7a00d47 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java @@ -2,6 +2,8 @@ import jakarta.annotation.PreDestroy; import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -10,11 +12,14 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; /** Keeps a live exact-generation operation owned while remote work is running. */ @Service public class RagIndexOperationHeartbeatService { + private static final Logger log = LoggerFactory.getLogger( + RagIndexOperationHeartbeatService.class); private static final long HEARTBEAT_INTERVAL_SECONDS = 15; private static final int HEARTBEAT_THREADS = 4; @@ -41,20 +46,33 @@ HEARTBEAT_THREADS, new HeartbeatThreadFactory()), } public HeartbeatScope start(long operationId) { + AtomicBoolean degraded = new AtomicBoolean(false); ScheduledFuture heartbeat = executor.scheduleAtFixedRate( - () -> heartbeat(operationId), + () -> heartbeat(operationId, degraded), heartbeatIntervalSeconds, heartbeatIntervalSeconds, TimeUnit.SECONDS); return () -> heartbeat.cancel(false); } - private void heartbeat(long operationId) { + private void heartbeat(long operationId, AtomicBoolean degraded) { try { registryService.heartbeatBuild(operationId); - } catch (Exception ignored) { + if (degraded.getAndSet(false)) { + log.info("RAG generation heartbeat recovered: operation={}", operationId); + } + } catch (Exception failure) { // A later heartbeat may still succeed. If the producer stops, // durable operation recovery owns the terminal transition. + if (degraded.compareAndSet(false, true)) { + log.warn( + "RAG generation heartbeat failed; retrying: operation={} detail={}", + operationId, failure.getMessage()); + } else { + log.debug( + "RAG generation heartbeat remains degraded: operation={} detail={}", + operationId, failure.getMessage()); + } } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java index d13a4d4a..8e5853ca 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java @@ -5,6 +5,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.rostilos.codecrow.ragengine.source.RepositorySourceTreeIdentity; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -23,6 +24,7 @@ public class RagPipelineClient { private final OkHttpClient httpClient; private final OkHttpClient longRunningHttpClient; + private final OkHttpClient streamingHttpClient; private final ObjectMapper objectMapper; private final String ragApiUrl; private final boolean ragEnabled; @@ -50,12 +52,32 @@ public boolean isServiceFailure() { } } + public RagPipelineClient( + String ragApiUrl, + boolean ragEnabled, + int connectTimeout, + int readTimeout, + int indexingTimeout, + String serviceSecret + ) { + this( + ragApiUrl, + ragEnabled, + connectTimeout, + readTimeout, + indexingTimeout, + Math.min(indexingTimeout, 60), + serviceSecret); + } + + @Autowired public RagPipelineClient( @Value("${codecrow.rag.api.url:http://rag-pipeline:8001}") String ragApiUrl, @Value("${codecrow.rag.api.enabled:true}") boolean ragEnabled, @Value("${codecrow.rag.api.timeout.connect:30}") int connectTimeout, @Value("${codecrow.rag.api.timeout.read:120}") int readTimeout, @Value("${codecrow.rag.api.timeout.indexing:14400}") int indexingTimeout, + @Value("${codecrow.rag.api.timeout.stream-idle:60}") int streamIdleTimeout, @Value("${codecrow.rag.api.secret:}") String serviceSecret ) { this.ragApiUrl = normalizeBaseUrl(ragApiUrl); @@ -73,6 +95,12 @@ public RagPipelineClient( .readTimeout(indexingTimeout, java.util.concurrent.TimeUnit.SECONDS) .writeTimeout(indexingTimeout, java.util.concurrent.TimeUnit.SECONDS) .build(); + + this.streamingHttpClient = this.longRunningHttpClient.newBuilder() + .readTimeout( + Math.max(1, streamIdleTimeout), + java.util.concurrent.TimeUnit.SECONDS) + .build(); this.objectMapper = new ObjectMapper(); } @@ -1139,6 +1167,11 @@ private Map postLongRunningSse( Runnable ownershipAdmissionConsumer, Consumer> progressConsumer ) throws IOException { + Object workspace = payload.get("workspace"); + Object project = payload.get("project"); + Object branch = payload.get("branch"); + Object commit = payload.get("commit"); + long startedNanos = System.nanoTime(); RequestBody body = RequestBody.create(objectMapper.writeValueAsString(payload), JSON); Request.Builder builder = new Request.Builder() .url(url) @@ -1146,7 +1179,10 @@ private Map postLongRunningSse( .post(body); addAuthHeader(builder); - try (Response response = longRunningHttpClient.newCall(builder.build()).execute()) { + log.info( + "RAG index stream starting workspace={} project={} branch={} commit={}", + workspace, project, branch, commit); + try (Response response = streamingHttpClient.newCall(builder.build()).execute()) { if (!response.isSuccessful()) { String detail = response.body() != null ? response.body().string() : "{}"; throw new RagApiException(response.code(), detail); @@ -1166,6 +1202,12 @@ private Map postLongRunningSse( Map event = objectMapper.readValue(json, Map.class); String type = String.valueOf(event.get("type")); if ("admitted".equals(type)) { + log.info( + "RAG index stream admitted workspace={} project={} branch={} " + + "ownership_transferred={} elapsed_ms={}", + workspace, project, branch, + event.get("repositoryOwnershipTransferred"), + elapsedMillis(startedNanos)); if (Boolean.TRUE.equals( event.get("repositoryOwnershipTransferred")) && ownershipAdmissionConsumer != null) { @@ -1174,7 +1216,21 @@ private Map postLongRunningSse( } continue; } + if ("heartbeat".equals(type)) { + log.info( + "RAG index stream heartbeat workspace={} project={} branch={} " + + "stage={} elapsed_ms={} idle_ms={}", + workspace, project, branch, event.get("stage"), + event.get("elapsedMs"), event.get("idleMs")); + continue; + } if ("progress".equals(type)) { + log.info( + "RAG index stream progress workspace={} project={} branch={} " + + "stage={} progress={} message={} elapsed_ms={}", + workspace, project, branch, event.get("stage"), + event.get("progress"), event.get("message"), + elapsedMillis(startedNanos)); if (progressConsumer != null) { progressConsumer.accept(new LinkedHashMap<>(event)); } @@ -1183,6 +1239,13 @@ private Map postLongRunningSse( if ("complete".equals(type)) { Object result = event.get("result"); if (result instanceof Map resultMap) { + log.info( + "RAG index stream completed workspace={} project={} branch={} " + + "commit={} documents={} chunks={} elapsed_ms={}", + workspace, project, branch, commit, + resultMap.get("document_count"), + resultMap.get("chunk_count"), + elapsedMillis(startedNanos)); return new LinkedHashMap<>((Map) resultMap); } throw new IOException("RAG progress stream completed without index result"); @@ -1191,10 +1254,26 @@ private Map postLongRunningSse( throw new IOException("RAG API error: " + event.getOrDefault("message", "unknown error")); } } - } + } catch (IOException | RuntimeException failure) { + log.warn( + "RAG index stream failed workspace={} project={} branch={} " + + "commit={} elapsed_ms={}: {}", + workspace, project, branch, commit, + elapsedMillis(startedNanos), failure.getMessage()); + throw failure; + } + log.warn( + "RAG index stream ended without terminal result workspace={} " + + "project={} branch={} commit={} elapsed_ms={}", + workspace, project, branch, commit, elapsedMillis(startedNanos)); throw new IOException("RAG progress stream ended without a terminal result"); } + private static long elapsedMillis(long startedNanos) { + return java.util.concurrent.TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - startedNanos); + } + private static String truncateDetail(String detail) { if (detail == null) { return "no detail"; diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java index 226b8e25..1e5d1b2d 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java @@ -1,5 +1,8 @@ package org.rostilos.codecrow.ragengine.client; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import com.fasterxml.jackson.databind.ObjectMapper; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -8,6 +11,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Path; @@ -545,6 +549,38 @@ void testIndexRepository_StreamForwardsProgressAndReturnsTerminalResult() throws assertThat(request.getHeader("Accept")).isEqualTo("text/event-stream"); } + @Test + void testIndexRepository_StreamLogsLifecycleAndHeartbeat() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("data: {\"type\":\"heartbeat\",\"stage\":\"architecture\"," + + "\"elapsedMs\":15000,\"idleMs\":15000}\n\n" + + "data: {\"type\":\"complete\",\"result\":{" + + "\"document_count\":2,\"chunk_count\":5}}\n\n") + .addHeader("Content-Type", "text/event-stream")); + Logger logger = (Logger) LoggerFactory.getLogger(RagPipelineClient.class); + ListAppender logs = new ListAppender<>(); + logs.start(); + logger.addAppender(logs); + try { + client.indexRepository( + repositoryPath.toString(), "ws", "proj", "develop", "abc123", + List.of(), List.of(), "generation-target", ignored -> { }); + } finally { + logger.detachAppender(logs); + } + + List messages = logs.list.stream() + .map(ILoggingEvent::getFormattedMessage) + .toList(); + assertThat(messages).anyMatch(message -> message.contains( + "RAG index stream starting workspace=ws project=proj branch=develop")); + assertThat(messages).anyMatch(message -> message.contains( + "RAG index stream heartbeat workspace=ws project=proj branch=develop " + + "stage=architecture")); + assertThat(messages).anyMatch(message -> message.contains( + "RAG index stream completed workspace=ws project=proj branch=develop")); + } + @Test void testIndexRepository_StreamAcknowledgesSnapshotOwnershipTransfer() throws Exception { mockWebServer.enqueue(new MockResponse() diff --git a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py index 901968b1..b1557925 100644 --- a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py +++ b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py @@ -79,6 +79,7 @@ def __init__(self, base_url: Optional[str] = None, enabled: Optional[bool] = Non self.enabled = enabled if enabled is not None else os.environ.get("RAG_ENABLED", "true").lower() == "true" self.timeout = 30.0 self._client: Optional[httpx.AsyncClient] = None + self._mutation_client: Optional[httpx.AsyncClient] = None self._service_secret = ( os.environ.get("SERVICE_SECRET") or os.environ.get("CODECROW_RAG_API_SECRET", "") @@ -91,7 +92,7 @@ def __init__(self, base_url: Optional[str] = None, enabled: Optional[bool] = Non logger.info("RAG client disabled") async def _get_client(self) -> httpx.AsyncClient: - """Get or create an HTTP client for connection pooling (instance-level).""" + """Get the query/health pool, isolated from long PR mutations.""" if self._client is None or self._client.is_closed: headers = {} if self._service_secret: @@ -102,12 +103,34 @@ async def _get_client(self) -> httpx.AsyncClient: headers=headers, ) return self._client + + async def _get_mutation_client(self) -> httpx.AsyncClient: + """Get the bounded PR overlay/cleanup connection pool.""" + if self._mutation_client is None or self._mutation_client.is_closed: + headers = {} + if self._service_secret: + headers["x-service-secret"] = self._service_secret + self._mutation_client = httpx.AsyncClient( + timeout=self.timeout, + limits=httpx.Limits( + max_connections=4, + max_keepalive_connections=2, + ), + headers=headers, + ) + return self._mutation_client async def close(self): """Close this instance's HTTP client.""" if self._client is not None and not self._client.is_closed: await self._client.aclose() self._client = None + if ( + self._mutation_client is not None + and not self._mutation_client.is_closed + ): + await self._mutation_client.aclose() + self._mutation_client = None def _record_cleanup_failure(self, detail: str) -> None: if not self._cleanup_degraded: @@ -715,7 +738,7 @@ async def index_pr_files( if collection_target: payload["collection_target"] = collection_target - client = await self._get_client() + client = await self._get_mutation_client() response = await client.post( f"{self.base_url}/index/pr-files", json=payload, @@ -779,7 +802,7 @@ async def delete_pr_files( return True try: - client = await self._get_client() + client = await self._get_mutation_client() response = await client.delete( f"{self.base_url}/index/pr-files/{workspace}/{project}/{pr_number}", params=( diff --git a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py index ec264777..62de59ed 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py +++ b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py @@ -392,6 +392,18 @@ async def test_get_client_reuses(self): assert client1 is client2 await c.close() + @pytest.mark.asyncio(loop_scope="function") + async def test_query_and_mutation_connection_pools_are_isolated(self): + c = RagClient(base_url="http://rag:8001", enabled=True) + + query_client = await c._get_client() + mutation_client = await c._get_mutation_client() + + assert query_client is not mutation_client + await c.close() + assert query_client.is_closed + assert mutation_client.is_closed + @pytest.mark.asyncio(loop_scope="function") async def test_empty_queries_duplicates(self, enabled_client): r = await enabled_client.search_for_duplicates("ws", "proj", "main", []) diff --git a/python-ecosystem/inference-orchestrator/tests/test_rag_index_timeout.py b/python-ecosystem/inference-orchestrator/tests/test_rag_index_timeout.py index c8893f50..165307b1 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_rag_index_timeout.py +++ b/python-ecosystem/inference-orchestrator/tests/test_rag_index_timeout.py @@ -29,7 +29,7 @@ def test_pr_index_timeout_is_configurable(monkeypatch): http_client = AsyncMock() http_client.post.return_value = response client = RagClient(base_url="http://rag-pipeline:8001", enabled=True) - client._get_client = AsyncMock(return_value=http_client) + client._get_mutation_client = AsyncMock(return_value=http_client) result = asyncio.run(client.index_pr_files( "workspace", @@ -55,7 +55,7 @@ def test_pr_index_timeout_default_covers_measured_large_overlay(monkeypatch): http_client = AsyncMock() http_client.post.return_value = response client = RagClient(base_url="http://rag-pipeline:8001", enabled=True) - client._get_client = AsyncMock(return_value=http_client) + client._get_mutation_client = AsyncMock(return_value=http_client) asyncio.run(client.index_pr_files( "workspace", diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py index 35adca29..82f76117 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py @@ -32,6 +32,21 @@ router = APIRouter(tags=["index"]) +def _stream_heartbeat_interval_seconds() -> float: + raw = os.environ.get("RAG_INDEX_STREAM_HEARTBEAT_SECONDS", "15") + try: + return max(1.0, float(raw)) + except (TypeError, ValueError): + logger.warning( + "Invalid RAG_INDEX_STREAM_HEARTBEAT_SECONDS=%r; using 15 seconds", + raw, + ) + return 15.0 + + +INDEX_STREAM_HEARTBEAT_SECONDS = _stream_heartbeat_interval_seconds() + + class _IndexStreamWorkerRegistry: """Track synchronous HTTP-stream indexing beyond request cancellation.""" @@ -462,6 +477,12 @@ def run_index() -> None: raise async def event_stream(): + stream_started = time.monotonic() + last_payload_at = stream_started + next_heartbeat_at = ( + stream_started + INDEX_STREAM_HEARTBEAT_SECONDS + ) + last_stage = "admitted" if repository_ownership_transferred else "starting" try: if repository_ownership_transferred: admitted = { @@ -469,6 +490,10 @@ async def event_stream(): "repositoryOwnershipTransferred": True, } yield f"data: {json.dumps(admitted)}\n\n" + last_payload_at = time.monotonic() + next_heartbeat_at = ( + last_payload_at + INDEX_STREAM_HEARTBEAT_SECONDS + ) while True: try: payload = progress_events.get_nowait() @@ -491,15 +516,41 @@ async def event_stream(): else: # Polling thread-safe queues avoids nesting a # blocking consumer inside Starlette's thread pool. + now = time.monotonic() + if now >= next_heartbeat_at: + heartbeat = { + "type": "heartbeat", + "stage": last_stage, + "message": "RAG indexing is still processing", + "elapsedMs": round( + (now - stream_started) * 1000 + ), + "idleMs": round( + (now - last_payload_at) * 1000 + ), + } + yield ( + "data: " + f"{json.dumps(heartbeat, ensure_ascii=False)}" + "\n\n" + ) + next_heartbeat_at = ( + now + INDEX_STREAM_HEARTBEAT_SECONDS + ) await asyncio.sleep(0.05) continue if event_type == "progress": event = {"type": "progress", **payload} + last_stage = str(payload.get("stage", last_stage)) elif event_type == "complete": event = {"type": "complete", "result": payload} else: event = {"type": "error", **payload} yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n" + last_payload_at = time.monotonic() + next_heartbeat_at = ( + last_payload_at + INDEX_STREAM_HEARTBEAT_SECONDS + ) if event_type in {"complete", "error"}: break finally: diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/system.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/system.py index 38d8ee45..ab2b3d8f 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/system.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/system.py @@ -13,7 +13,11 @@ def root(): @router.get("/health") -def health(): +async def health(): + # Keep liveness on the event loop. Synchronous FastAPI handlers share a + # finite AnyIO worker pool, so a burst of slow indexing/Qdrant calls must + # not make Docker declare an otherwise running process unhealthy merely + # because every worker token is occupied. return {"status": "healthy"} diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py index e6d55178..d8c11b04 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py @@ -47,6 +47,7 @@ # Memory-efficient batch sizes DOCUMENT_BATCH_SIZE = 50 INSERT_BATCH_SIZE = 128 +OPAQUE_STATE_PART_SIZE = 100_000 def _plugin_identity_metadata( @@ -264,7 +265,7 @@ def _snapshot_nodes( ) -> List[TextNode]: """Store opaque plugin snapshots in bounded zero-vector payload nodes.""" nodes: List[TextNode] = [] - part_size = 400_000 + part_size = OPAQUE_STATE_PART_SIZE for snapshot in analysis.snapshots: identity = f"{snapshot.plugin_id}\0{snapshot.kind}" digest = hashlib.sha256(identity.encode("utf-8")).hexdigest() @@ -327,7 +328,7 @@ def _repository_facts_nodes( ensure_ascii=False, ) content_digest = hashlib.sha256(content.encode("utf-8")).hexdigest() - part_size = 400_000 + part_size = OPAQUE_STATE_PART_SIZE parts = [ content[offset:offset + part_size] for offset in range(0, len(content), part_size) @@ -775,6 +776,7 @@ def report_progress( # for admission control, so using it as the iteration bound would # silently truncate the resolver input whenever a plugin removes # files from semantic indexing. + completed_batch_duration_ms = 0 for i in range(0, len(file_list), DOCUMENT_BATCH_SIZE): batch_num += 1 file_batch = file_list[i:i + DOCUMENT_BATCH_SIZE] @@ -803,10 +805,11 @@ def report_progress( path, ) skipped_file_paths.update(missing_paths) - if not documents: - continue - - if analysis_handle is not None and analysis_handle.active: + if ( + documents + and analysis_handle is not None + and analysis_handle.active + ): from codecrow_plugins import FileArtifact artifacts = tuple(sorted( @@ -826,82 +829,97 @@ def report_progress( for document in documents if document.metadata["path"] in semantic_paths ] - if not semantic_documents: - del documents - continue - - split_started = time.perf_counter() - chunks, split_skipped_paths = ( - self.splitter.split_documents_resilient( - semantic_documents, - capabilities=capabilities, + architecture_only_count = len(documents) - len(semantic_documents) + split_duration_ms = 0 + point_pipeline_duration_ms = 0 + batch_chunk_count = 0 + chunks = [] + + if semantic_documents: + split_started = time.perf_counter() + chunks, split_skipped_paths = ( + self.splitter.split_documents_resilient( + semantic_documents, + capabilities=capabilities, + ) ) - ) - split_duration_ms = round( - (time.perf_counter() - split_started) * 1000 - ) - skipped_file_paths.update(split_skipped_paths) - document_count += ( - len(semantic_documents) - len(split_skipped_paths) - ) - identity_metadata = _plugin_identity_metadata( - capabilities, - implementation_fingerprint, - self.representation_fingerprint, - ) - for chunk in chunks: - chunk.metadata.update(identity_metadata) - batch_chunk_count = len(chunks) - chunk_count += batch_chunk_count - - # Check chunk limit - if self.config.max_chunks_per_index > 0 and chunk_count > self.config.max_chunks_per_index: - self.collection_manager.delete_collection(pending_collection_name) - raise ValueError(f"Repository exceeds chunk limit: {chunk_count}+ chunks.") - - # Process and upsert - point_pipeline_started = time.perf_counter() - success, failed = self.point_ops.process_and_upsert_chunks( - chunks, - pending_collection_name, - workspace, - project, - branch, - reuse_collection_name=vector_reuse_collection, - operation_id=operation_id, - metrics=embedding_metrics, - ) - point_pipeline_duration_ms = round( - (time.perf_counter() - point_pipeline_started) * 1000 - ) - successful_chunks += success - skipped_chunk_count += failed - if failed: - logger.warning( - "Skipped %s rejected chunks in batch %s; " - "continuing repository indexing", - failed, - batch_num, + split_duration_ms = round( + (time.perf_counter() - split_started) * 1000 + ) + skipped_file_paths.update(split_skipped_paths) + document_count += ( + len(semantic_documents) - len(split_skipped_paths) + ) + identity_metadata = _plugin_identity_metadata( + capabilities, + implementation_fingerprint, + self.representation_fingerprint, + ) + for chunk in chunks: + chunk.metadata.update(identity_metadata) + batch_chunk_count = len(chunks) + chunk_count += batch_chunk_count + + # Check chunk limit + if ( + self.config.max_chunks_per_index > 0 + and chunk_count > self.config.max_chunks_per_index + ): + self.collection_manager.delete_collection( + pending_collection_name + ) + raise ValueError( + f"Repository exceeds chunk limit: {chunk_count}+ chunks." + ) + + # Process and upsert + point_pipeline_started = time.perf_counter() + success, failed = self.point_ops.process_and_upsert_chunks( + chunks, + pending_collection_name, + workspace, + project, + branch, + reuse_collection_name=vector_reuse_collection, + operation_id=operation_id, + metrics=embedding_metrics, ) - + point_pipeline_duration_ms = round( + (time.perf_counter() - point_pipeline_started) * 1000 + ) + successful_chunks += success + skipped_chunk_count += failed + if failed: + logger.warning( + "Skipped %s rejected chunks in batch %s; " + "continuing repository indexing", + failed, + batch_num, + ) + + batch_duration_ms = round( + (time.perf_counter() - batch_started) * 1000 + ) logger.info( "RAG document batch completed operation_id=%s batch=%s/%s " - "semantic_files=%s chunks=%s load_duration_ms=%s " + "semantic_files=%s architecture_only_files=%s chunks=%s " + "load_duration_ms=%s " "split_duration_ms=%s point_pipeline_duration_ms=%s " "duration_ms=%s", operation_id, batch_num, total_batches, len(semantic_documents), + architecture_only_count, batch_chunk_count, load_duration_ms, split_duration_ms, point_pipeline_duration_ms, - round((time.perf_counter() - batch_started) * 1000), + batch_duration_ms, ) batch_progress = 18 + round(67 * batch_num / max(total_batches, 1)) - elapsed_ms = round((time.perf_counter() - operation_started) * 1000) - average_batch_ms = elapsed_ms / batch_num + completed_batch_duration_ms += batch_duration_ms + average_batch_ms = completed_batch_duration_ms / batch_num estimated_remaining_ms = round( average_batch_ms * max(total_batches - batch_num, 0) ) @@ -916,17 +934,18 @@ def report_progress( total_batches, indexedChunks=successful_chunks, estimatedChunks=estimated_chunks, + semanticFiles=len(semantic_documents), + architectureOnlyFiles=architecture_only_count, completedBatches=batch_num, totalBatches=total_batches, - batchDurationMs=round( - (time.perf_counter() - batch_started) * 1000 - ), + batchDurationMs=batch_duration_ms, estimatedRemainingMs=estimated_remaining_ms, + remainingEstimateScope="file_batches", ) - + del documents del chunks - + if batch_num % 5 == 0: gc.collect() @@ -935,52 +954,151 @@ def report_progress( context_nodes = [] snapshot_nodes = [] if analysis_handle is not None: + configured_architecture_timeout = getattr( + self.config, + "architecture_finalization_timeout_seconds", + 600, + ) + if not isinstance(configured_architecture_timeout, (int, float)): + configured_architecture_timeout = 600 + architecture_timeout_seconds = max( + 1.0, + float(configured_architecture_timeout), + ) + architecture_deadline = ( + time.monotonic() + architecture_timeout_seconds + ) report_progress( "architecture", "Building deterministic architecture context", 88, + architectureTimeoutSeconds=architecture_timeout_seconds, + estimatedRemainingMs=0, + remainingEstimateScope="file_batches_complete", + ) + + def report_architecture_progress(event: dict) -> None: + report_progress( + "architecture", + str(event.get( + "message", + "Building deterministic architecture context", + )), + 88 if event.get("status") == "started" else 89, + architectureTimeoutSeconds=architecture_timeout_seconds, + architecturePlugin=event.get("pluginId"), + architectureSubstage=event.get("substage"), + architectureStatus=event.get("status"), + sourceRoot=event.get("sourceRoot"), + substageDurationMs=event.get("durationMs"), + ) + + repository_analysis, diagnostics = analysis_handle.finish( + progress_callback=report_architecture_progress, + deadline=architecture_deadline, ) - repository_analysis, diagnostics = analysis_handle.finish() skipped_file_paths.update( self.accept_recoverable_repository_diagnostics( diagnostics, "repository architecture analysis", ) ) - - architecture_nodes = self._architecture_nodes( - repository_analysis, - capabilities, - workspace, - project, - branch, - commit, - implementation_fingerprint, - self.representation_fingerprint, - ) - snapshot_nodes = self._snapshot_nodes( - repository_analysis, - capabilities, - workspace, - project, - branch, - commit, - implementation_fingerprint, - self.representation_fingerprint, - ) - context_nodes = self._repository_context_nodes( - repository_analysis, - capabilities, - workspace, - project, - branch, - commit, - implementation_fingerprint, - self.representation_fingerprint, - ) - analysis_nodes.extend( - (*architecture_nodes, *context_nodes, *snapshot_nodes) + architecture_timed_out = any( + diagnostic.code + == "plugin-repository-finalization-timeout" + for diagnostic in diagnostics ) + if architecture_timed_out: + logger.warning( + "Repository architecture finalization exceeded %.1fs; " + "continuing operation_id=%s without deterministic " + "architecture context", + architecture_timeout_seconds, + operation_id, + ) + report_progress( + "architecture", + ( + "Architecture time budget exhausted; continuing " + "with semantic indexing" + ), + 90, + architectureStatus="degraded", + degraded=True, + architectureTimeoutSeconds=architecture_timeout_seconds, + ) + else: + try: + report_progress( + "architecture", + "Materializing deterministic architecture records", + 89, + architectureStatus="materializing", + ) + architecture_nodes = self._architecture_nodes( + repository_analysis, + capabilities, + workspace, + project, + branch, + commit, + implementation_fingerprint, + self.representation_fingerprint, + ) + snapshot_nodes = self._snapshot_nodes( + repository_analysis, + capabilities, + workspace, + project, + branch, + commit, + implementation_fingerprint, + self.representation_fingerprint, + ) + context_nodes = self._repository_context_nodes( + repository_analysis, + capabilities, + workspace, + project, + branch, + commit, + implementation_fingerprint, + self.representation_fingerprint, + ) + analysis_nodes.extend( + (*architecture_nodes, *context_nodes, *snapshot_nodes) + ) + except Exception as exception: + architecture_nodes = [] + snapshot_nodes = [] + context_nodes = [] + logger.warning( + "Repository architecture materialization failed; " + "continuing operation_id=%s with semantic indexing: %s", + operation_id, + exception, + exc_info=True, + ) + report_progress( + "architecture", + ( + "Architecture materialization failed; continuing " + "with semantic indexing" + ), + 90, + architectureStatus="degraded", + degraded=True, + architectureFailure=type(exception).__name__, + ) + + # Repository sessions retain the architecture-only source + # artifacts used to build snapshots, while RepositoryAnalysis + # retains the unsliced snapshot strings. Once bounded storage + # nodes exist, release both before PointStruct/Qdrant payloads + # are materialized to avoid holding three copies concurrently. + analysis_handle = None + del repository_analysis + gc.collect() facts_nodes = [] if repository_facts is not None: @@ -1006,6 +1124,13 @@ def report_progress( raise ValueError( f"Repository exceeds chunk limit after architecture analysis: {chunk_count} chunks." ) + report_progress( + "architecture", + f"Persisting {architecture_count} deterministic context records", + 90, + architectureStatus="persisting", + architectureRecords=architecture_count, + ) success, failed = self.point_ops.process_and_upsert_chunks( analysis_nodes, pending_collection_name, @@ -1032,6 +1157,16 @@ def report_progress( len(snapshot_nodes), len(facts_nodes), ) + report_progress( + "architecture", + ( + f"Persisted {success} deterministic context records" + ), + 91, + architectureStatus="completed", + architectureRecords=success, + skippedArchitectureRecords=failed, + ) generation_manifest_sha256 = None generation_manifest_points = 0 @@ -1039,7 +1174,7 @@ def report_progress( report_progress( "sealing", "Sealing persisted vectors for generation integrity", - 91, + 92, indexedChunks=successful_chunks, estimatedChunks=estimated_chunks, ) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py index 4f618212..edd8c054 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py @@ -9,6 +9,9 @@ import hashlib import json import re +import threading +import time +from contextlib import contextmanager from typing import Callable, Optional, List from llama_index.core import Settings @@ -108,6 +111,9 @@ class RAGIndexManager: def __init__(self, config: RAGConfig): self.config = config + self._full_index_capacity = threading.BoundedSemaphore( + _config_int(config, "full_index_concurrency", 1) + ) self._mutation_coordinator = ProjectMutationCoordinator( os.getenv("REDIS_URL", "redis://redis:6379/1"), lease_seconds=_config_int( @@ -202,6 +208,11 @@ def __init__(self, config: RAGConfig): self.qdrant_client, self.embed_model, batch_size=_config_int(config, "qdrant_upsert_batch_size", 128), + max_upsert_payload_bytes=_config_int( + config, + "qdrant_upsert_max_payload_bytes", + 8 * 1024 * 1024, + ), embedding_batch_size=( _config_int(config, "openrouter_batch_size", 50) if str(config.embedding_provider).lower() == "openrouter" @@ -312,6 +323,52 @@ def _publication_scope( # Repository indexing + @contextmanager + def _admit_full_index( + self, + workspace: str, + project: str, + branch: str, + progress_callback: Optional[Callable[[dict], None]], + ): + """Serialize memory-heavy full builds inside one RAG worker process.""" + wait_started = time.monotonic() + acquired = self._full_index_capacity.acquire(blocking=False) + if not acquired: + logger.info( + "RAG full index waiting for process capacity " + "workspace=%s project=%s branch=%s", + workspace, + project, + branch, + ) + if progress_callback is not None: + try: + progress_callback({ + "stage": "waiting_capacity", + "message": "Waiting for RAG full-index capacity", + "progress": 0, + }) + except Exception as exception: + logger.warning( + "RAG capacity progress callback failed: %s", + exception, + ) + self._full_index_capacity.acquire() + waited_ms = round((time.monotonic() - wait_started) * 1000) + logger.info( + "RAG full index admitted workspace=%s project=%s branch=%s " + "wait_ms=%s", + workspace, + project, + branch, + waited_ms, + ) + try: + yield + finally: + self._full_index_capacity.release() + def estimate_repository_size( self, repo_path: str, @@ -347,6 +404,51 @@ def index_repository( raise ValueError( "readable generation aliases require an immutable collection target" ) + with self._admit_full_index( + workspace, + project, + branch, + progress_callback, + ): + return self._index_repository_admitted( + repo_path=repo_path, + workspace=workspace, + project=project, + branch=branch, + commit=commit, + preserve_other_branches=preserve_other_branches, + include_patterns=include_patterns, + exclude_patterns=exclude_patterns, + source_tree_sha256=source_tree_sha256, + collection_target=collection_target, + reuse_collection_target=reuse_collection_target, + publish_branch_alias=publish_branch_alias, + publish_legacy_project_alias=publish_legacy_project_alias, + progress_callback=progress_callback, + project_type=project_type, + source_root=source_root, + ) + + def _index_repository_admitted( + self, + repo_path: str, + workspace: str, + project: str, + branch: str, + commit: str, + preserve_other_branches: bool = False, + include_patterns: Optional[List[str]] = None, + exclude_patterns: Optional[List[str]] = None, + source_tree_sha256: Optional[str] = None, + collection_target: Optional[str] = None, + reuse_collection_target: Optional[str] = None, + publish_branch_alias: bool = False, + publish_legacy_project_alias: bool = False, + progress_callback: Optional[Callable[[dict], None]] = None, + project_type: Optional[str] = None, + source_root: Optional[str] = None, + ) -> IndexStats: + """Run a full build after process-wide capacity has been acquired.""" alias_name = collection_target or self._get_project_collection_name( workspace, project ) @@ -383,7 +485,10 @@ def index_repository( project, "full-index", collection_target=collection_target, - publication_scope=self._publication_scope(branch, publication_aliases), + publication_scope=self._publication_scope( + branch, + publication_aliases, + ), ) as lease: return self._indexer.index_repository( repo_path=repo_path, diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py index e7640a49..1e850d6a 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py @@ -52,6 +52,7 @@ def __init__( client: QdrantClient, embed_model, batch_size: int = 50, + max_upsert_payload_bytes: int = 8 * 1024 * 1024, embedding_batch_size: Optional[int] = None, max_embedding_workers: int = 1, embedding_dim: int | None = None, @@ -61,6 +62,8 @@ def __init__( ): if batch_size <= 0: raise ValueError("batch_size must be positive") + if max_upsert_payload_bytes <= 0: + raise ValueError("max_upsert_payload_bytes must be positive") if embedding_batch_size is not None and embedding_batch_size <= 0: raise ValueError("embedding_batch_size must be positive") if max_embedding_workers <= 0: @@ -73,6 +76,7 @@ def __init__( self.embed_model = embed_model # ``batch_size`` remains the public/legacy Qdrant write batch setting. self.batch_size = batch_size + self.max_upsert_payload_bytes = max_upsert_payload_bytes self.embedding_batch_size = embedding_batch_size or batch_size self.max_embedding_workers = max_embedding_workers self.embedding_dim = embedding_dim @@ -420,6 +424,49 @@ def upsert_points_detailed( return PointWriteResult(successful, tuple(skipped_points)) + @staticmethod + def _serialized_point_size(point: PointStruct) -> int: + """Return the exact compact JSON size of one point on the REST wire.""" + payload = point.model_dump(mode="json", exclude_none=True) + return len(json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8")) + + def _payload_bounded_batches( + self, + points: List[PointStruct], + ) -> list[tuple[List[PointStruct], int]]: + """Pack points below a conservative request-body budget. + + Qdrant enforces a byte limit independently of the configured point-count + batch. Opaque repository snapshots can be hundreds of kilobytes each, + so a count-only batch may otherwise allocate and transmit a 30-50 MB + request only to have Qdrant reject and recursively retry it. + """ + if not points: + return [] + # Reserve space for the REST request envelope and JSON separators. + request_overhead = 256 + batches: list[tuple[List[PointStruct], int]] = [] + current: List[PointStruct] = [] + current_size = request_overhead + for point in points: + point_size = self._serialized_point_size(point) + 1 + if ( + current + and current_size + point_size > self.max_upsert_payload_bytes + ): + batches.append((current, current_size)) + current = [] + current_size = request_overhead + current.append(point) + current_size += point_size + if current: + batches.append((current, current_size)) + return batches + @staticmethod def _status_code(exception: Exception) -> int | None: status_code = getattr(exception, "status_code", None) @@ -485,6 +532,30 @@ def _upsert_resilient( if not points: return PointWriteResult() + bounded_batches = self._payload_bounded_batches(points) + if len(bounded_batches) > 1: + logger.info( + "Splitting %s Qdrant points into %s byte-bounded requests " + "before write (estimated_bytes=%s limit=%s)", + len(points), + len(bounded_batches), + sum(size for _, size in bounded_batches), + self.max_upsert_payload_bytes, + ) + successful = 0 + skipped_points: list[PointStruct] = [] + offset = batch_offset + for bounded_batch, _estimated_bytes in bounded_batches: + batch_result = self._upsert_resilient( + collection_name, + bounded_batch, + batch_offset=offset, + ) + successful += batch_result.successful + skipped_points.extend(batch_result.skipped_points) + offset += len(bounded_batch) + return PointWriteResult(successful, tuple(skipped_points)) + error = None for attempt in range(1, self.upsert_max_attempts + 1): try: @@ -660,6 +731,49 @@ def process_and_upsert_chunks( write_buffer: list[PointStruct] = [] started = time.perf_counter() + # Deterministic context uses zero vectors and can contain large opaque + # snapshot payloads. Running those batches through the embedding pool + # creates several complete PointStruct batches concurrently for no + # benefit and was a major source of transient memory pressure. Build + # and persist one bounded slice at a time instead. + if chunk_data and all( + self._is_architecture_chunk(chunk) + for _, chunk in chunk_data + ): + for offset in range(0, len(chunk_data), self.embedding_batch_size): + point_batch = self.embed_and_create_points( + chunk_data[offset:offset + self.embedding_batch_size], + reuse_collection_name=reuse_collection_name, + metrics=operation_metrics, + ) + batch_result = self.upsert_points_detailed( + collection_name, + point_batch, + ) + successful += batch_result.successful + failed += batch_result.failed + logger.info( + "RAG deterministic-context batch completed " + "operation_id=%s points=%s skipped=%s", + operation_id, + len(point_batch), + batch_result.failed, + ) + logger.info( + "RAG point pipeline completed operation_id=%s chunks=%s " + "successful=%s failed=%s reused=0 embedded=0 duration_ms=%s " + "embedding_concurrency=0 embedding_batch_size=%s " + "qdrant_batch_size=%s", + operation_id, + len(chunk_data), + successful, + failed, + round((time.perf_counter() - started) * 1000), + self.embedding_batch_size, + self.batch_size, + ) + return successful, failed + def submit_available(executor: ThreadPoolExecutor) -> None: nonlocal next_batch while ( diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py index 7b886e29..56edb883 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py @@ -107,6 +107,17 @@ class RAGConfig(BaseModel): default_factory=lambda: int(os.getenv("QDRANT_UPSERT_BATCH_SIZE", "128")), ge=1, ) + qdrant_upsert_max_payload_bytes: int = Field( + default_factory=lambda: int(os.getenv( + "QDRANT_UPSERT_MAX_PAYLOAD_BYTES", + str(8 * 1024 * 1024), + )), + ge=1024, + ) + full_index_concurrency: int = Field( + default_factory=lambda: int(os.getenv("RAG_FULL_INDEX_CONCURRENCY", "1")), + ge=1, + ) rag_mutation_lease_seconds: int = Field( default_factory=lambda: int(os.getenv("RAG_MUTATION_LEASE_SECONDS", "300")), ge=30, @@ -117,6 +128,12 @@ class RAGConfig(BaseModel): ), ge=0, ) + architecture_finalization_timeout_seconds: float = Field( + default_factory=lambda: float( + os.getenv("RAG_ARCHITECTURE_FINALIZATION_TIMEOUT_SECONDS", "600") + ), + ge=1, + ) revision_preflight_cache_entries: int = Field( default_factory=lambda: int( os.getenv("RAG_REVISION_PREFLIGHT_CACHE_ENTRIES", "512") diff --git a/python-ecosystem/rag-pipeline/tests/test_config.py b/python-ecosystem/rag-pipeline/tests/test_config.py index b3d4be90..d151412e 100644 --- a/python-ecosystem/rag-pipeline/tests/test_config.py +++ b/python-ecosystem/rag-pipeline/tests/test_config.py @@ -65,11 +65,29 @@ def test_default_values(self): assert config.similarity_threshold == 0.7 assert config.max_file_size_bytes == 1024 * 1024 assert config.qdrant_timeout_seconds == 30 + assert config.qdrant_upsert_max_payload_bytes == 8 * 1024 * 1024 + assert config.full_index_concurrency == 1 + assert config.architecture_finalization_timeout_seconds == 600 def test_qdrant_timeout_is_configurable(self): with patch.dict(os.environ, {"QDRANT_TIMEOUT_SECONDS": "45"}): assert RAGConfig().qdrant_timeout_seconds == 45 + def test_rag_resource_limits_are_configurable(self): + with patch.dict(os.environ, { + "QDRANT_UPSERT_MAX_PAYLOAD_BYTES": "4194304", + "RAG_FULL_INDEX_CONCURRENCY": "2", + }): + config = RAGConfig() + assert config.qdrant_upsert_max_payload_bytes == 4194304 + assert config.full_index_concurrency == 2 + + def test_architecture_finalization_timeout_is_configurable(self): + with patch.dict(os.environ, { + "RAG_ARCHITECTURE_FINALIZATION_TIMEOUT_SECONDS": "90", + }): + assert RAGConfig().architecture_finalization_timeout_seconds == 90 + def test_auto_detect_embedding_dim_ollama(self): config = RAGConfig(embedding_provider="ollama", ollama_model="all-minilm", embedding_dim=0) assert config.embedding_dim == 384 diff --git a/python-ecosystem/rag-pipeline/tests/test_index_manager.py b/python-ecosystem/rag-pipeline/tests/test_index_manager.py index 9823655b..8346d358 100644 --- a/python-ecosystem/rag-pipeline/tests/test_index_manager.py +++ b/python-ecosystem/rag-pipeline/tests/test_index_manager.py @@ -3,6 +3,7 @@ CollectionManager, BranchManager, PointOperations, StatsManager, RAGIndexManager. """ import pytest +import threading import uuid from httpx import Headers from qdrant_client.http.exceptions import ( @@ -597,6 +598,7 @@ def test_exact_snapshot_forwards_resolved_prior_generation_for_vector_reuse( ) manager._indexer = MagicMock() manager._indexer.index_repository.return_value = MagicMock() + manager._full_index_capacity = threading.BoundedSemaphore(1) manager._mutation_coordinator = MagicMock() lease = MagicMock(token="operation-token") lease.assert_owned = MagicMock() @@ -626,6 +628,45 @@ def test_exact_snapshot_forwards_resolved_prior_generation_for_vector_reuse( "reuse_collection_name" ] == "prior-generation-physical" + def test_full_index_capacity_serializes_heavy_builds(self): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + manager._full_index_capacity = threading.BoundedSemaphore(1) + first_entered = threading.Event() + release_first = threading.Event() + second_waiting = threading.Event() + second_entered = threading.Event() + + def first_build(): + with manager._admit_full_index("ws", "one", "main", None): + first_entered.set() + release_first.wait(timeout=2) + + def second_build(): + def progress(event): + if event["stage"] == "waiting_capacity": + second_waiting.set() + + with manager._admit_full_index("ws", "two", "main", progress): + second_entered.set() + + first = threading.Thread(target=first_build) + second = threading.Thread(target=second_build) + first.start() + assert first_entered.wait(timeout=1) + second.start() + assert second_waiting.wait(timeout=1) + assert not second_entered.is_set() + + release_first.set() + first.join(timeout=1) + second.join(timeout=1) + + assert not first.is_alive() + assert not second.is_alive() + assert second_entered.is_set() + @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") @patch("rag_pipeline.core.index_manager.manager.QdrantClient") diff --git a/python-ecosystem/rag-pipeline/tests/test_indexer.py b/python-ecosystem/rag-pipeline/tests/test_indexer.py index 3918d0e9..9e84e0fa 100644 --- a/python-ecosystem/rag-pipeline/tests/test_indexer.py +++ b/python-ecosystem/rag-pipeline/tests/test_indexer.py @@ -28,6 +28,7 @@ def _mock_config(**overrides): cfg = MagicMock() cfg.max_files_per_index = 0 cfg.max_chunks_per_index = 0 + cfg.architecture_finalization_timeout_seconds = 600 for k, v in overrides.items(): setattr(cfg, k, v) return cfg @@ -173,8 +174,20 @@ def test_progress_callback_failure_does_not_fail_indexing(self): assert result.document_count == 0 - def test_architecture_files_are_ingested_while_generated_files_are_not_loaded(self, tmp_path): - from codecrow_plugins import FileDisposition, ProjectCapabilities, RepositoryAnalysis + def test_architecture_only_batch_is_ingested_and_reports_completion( + self, + tmp_path, + monkeypatch, + ): + from codecrow_plugins import ( + FileDisposition, + PluginDiagnostic, + ProjectCapabilities, + RepositoryAnalysis, + ) + from rag_pipeline.core.index_manager import indexer as indexer_module + + monkeypatch.setattr(indexer_module, "DOCUMENT_BATCH_SIZE", 1) config = _mock_config() coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() @@ -184,9 +197,19 @@ def test_architecture_files_are_ingested_while_generated_files_are_not_loaded(se Path("generated/code/Proxy.php"), ] loader.iter_repository_files.return_value = iter(paths) - loader.load_file_batch.return_value = [ - SimpleNamespace(text="", metadata={"path": "etc/di.xml"}), + documents_by_path = { + "source.php": SimpleNamespace( + text="", + metadata={"path": "etc/di.xml"}, + ), + } + loader.load_file_batch.side_effect = lambda batch, *_args, **_kwargs: [ + documents_by_path[Path(path).as_posix()] + for path in batch ] splitter.split_documents.return_value = [MagicMock()] coll_mgr.create_pending_collection.return_value = "pending" @@ -204,7 +227,15 @@ def test_architecture_files_are_ingested_while_generated_files_are_not_loaded(se fingerprint="sha256:" + "0" * 64, ) handle = MagicMock(active=True) - handle.finish.return_value = (RepositoryAnalysis(), ()) + handle.finish.return_value = ( + RepositoryAnalysis(), + (PluginDiagnostic( + code="plugin-repository-finalization-timeout", + message="Magento architecture exceeded its time budget", + plugin_id="magento", + recoverable=True, + ),), + ) runtime = MagicMock() runtime.start_repository_analysis.return_value = handle runtime.file_disposition.side_effect = lambda path, _capabilities: { @@ -217,16 +248,38 @@ def test_architecture_files_are_ingested_while_generated_files_are_not_loaded(se plugin_catalog=MagicMock(), plugin_runtime=runtime, plugin_selector=selector, ) + progress_events = [] indexer.index_repository( - str(tmp_path), "ws", "proj", "main", "abc", "alias" + str(tmp_path), "ws", "proj", "main", "abc", "alias", + progress_callback=progress_events.append, ) - artifacts = handle.ingest.call_args.args[0] - assert [artifact.path for artifact in artifacts] == ["etc/di.xml", "source.php"] - loaded_paths = loader.load_file_batch.call_args.args[0] - assert loaded_paths == [Path("source.php"), Path("etc/di.xml")] + ingested_paths = [ + artifact.path + for invocation in handle.ingest.call_args_list + for artifact in invocation.args[0] + ] + assert ingested_paths == ["source.php", "etc/di.xml"] + loaded_paths = [ + invocation.args[0] + for invocation in loader.load_file_batch.call_args_list + ] + assert loaded_paths == [[Path("source.php")], [Path("etc/di.xml")]] semantic_documents = splitter.split_documents.call_args.args[0] assert [document.metadata["path"] for document in semantic_documents] == ["source.php"] + batch_events = [ + event for event in progress_events + if event["stage"] == "indexing" and "completedBatches" in event + ] + assert [event["completedBatches"] for event in batch_events] == [1, 2] + assert batch_events[-1]["architectureOnlyFiles"] == 1 + assert batch_events[-1]["estimatedRemainingMs"] == 0 + assert batch_events[-1]["remainingEstimateScope"] == "file_batches" + assert any( + event.get("architectureStatus") == "degraded" + and event.get("degraded") is True + for event in progress_events + ) def test_exceeds_file_limit(self): config = _mock_config(max_files_per_index=5) diff --git a/python-ecosystem/rag-pipeline/tests/test_point_operations.py b/python-ecosystem/rag-pipeline/tests/test_point_operations.py index e07692a9..a4e9b4b1 100644 --- a/python-ecosystem/rag-pipeline/tests/test_point_operations.py +++ b/python-ecosystem/rag-pipeline/tests/test_point_operations.py @@ -387,6 +387,70 @@ def reject_multi_point_request(**kwargs): assert len(accepted) == 4 +def test_upsert_splits_by_serialized_payload_before_qdrant_rejects_request(): + client = MagicMock() + operations = PointOperations( + client, + MagicMock(), + batch_size=4, + max_upsert_payload_bytes=900, + upsert_max_attempts=1, + ) + points = [ + PointStruct( + id=str(uuid.uuid4()), + vector=[0.1, 0.2, 0.3], + payload={"path": f"snapshot-{index}", "text": "x" * 400}, + ) + for index in range(4) + ] + + successful, failed = operations.upsert_points("pending", points) + + assert (successful, failed) == (4, 0) + assert client.upsert.call_count == 4 + assert all( + len(call.kwargs["points"]) == 1 + for call in client.upsert.call_args_list + ) + + +def test_deterministic_context_avoids_concurrent_embedding_workers(): + client = MagicMock() + embed_model = MagicMock() + operations = PointOperations( + client, + embed_model, + batch_size=4, + embedding_batch_size=2, + max_embedding_workers=4, + embedding_dim=3, + upsert_max_attempts=1, + ) + chunks = [ + TextNode( + text="opaque-state-" + str(index), + metadata={"path": f"state-{index}", "repository_snapshot": True}, + ) + for index in range(5) + ] + + successful, failed = operations.process_and_upsert_chunks( + chunks, + "pending", + "workspace", + "project", + "main", + ) + + assert (successful, failed) == (5, 0) + embed_model.get_text_embedding_batch.assert_not_called() + assert [ + len(call.kwargs["points"]) + for call in client.upsert.call_args_list + ] == [2, 2, 1] + + def test_upsert_isolates_one_rejected_point_without_losing_valid_siblings(): class InvalidPoint(RuntimeError): status_code = 400 diff --git a/python-ecosystem/rag-pipeline/tests/test_router_index.py b/python-ecosystem/rag-pipeline/tests/test_router_index.py index 3e98b33b..01ac00d1 100644 --- a/python-ecosystem/rag-pipeline/tests/test_router_index.py +++ b/python-ecosystem/rag-pipeline/tests/test_router_index.py @@ -271,6 +271,70 @@ def test_stream_progress_is_bounded_and_keeps_latest_event(self): assert events.qsize() == 1 assert events.get_nowait() == {"completedBatches": 99} + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_stream_emits_heartbeat_while_indexer_is_quiet( + self, + mock_get, + monkeypatch, + ): + _, im = _mock_singletons() + release = threading.Event() + stats = IndexStats( + namespace="ns", document_count=10, chunk_count=50, + last_updated="2024-01-01", workspace="ws", project="proj", + branch="main", + ) + + def quiet_index(**_kwargs): + release.wait(timeout=1) + return stats + + im.index_repository.side_effect = quiet_index + mock_get.return_value = (_, im) + import rag_pipeline.api.routers.index as index_router + + monkeypatch.setattr( + index_router, + "INDEX_STREAM_HEARTBEAT_SECONDS", + 0.01, + ) + req = MagicMock() + req.repo_path = "/tmp/repo" + req.workspace = "ws" + req.project = "proj" + req.branch = "main" + req.commit = "abc" + req.preserve_other_branches = False + req.include_patterns = None + req.exclude_patterns = None + req.project_type = None + req.source_root = None + req.source_tree_sha256 = None + req.collection_target = "target" + req.reuse_collection_target = None + + response = index_router.index_repository_stream(req) + + async def consume(): + items = [] + iterator = response.body_iterator.__aiter__() + first = await iterator.__anext__() + items.append(first) + release.set() + async for item in iterator: + items.append(item) + return items + + events = [json.loads( + (item.decode() if isinstance(item, bytes) else item) + .removeprefix("data: ").strip() + ) for item in asyncio.run(consume())] + + assert events[0]["type"] == "heartbeat" + assert events[0]["stage"] == "starting" + assert events[0]["elapsedMs"] >= 0 + assert events[-1]["type"] == "complete" + def test_orphan_cleanup_removes_only_old_owned_stream_directories( self, tmp_path, diff --git a/python-ecosystem/rag-pipeline/tests/test_routers.py b/python-ecosystem/rag-pipeline/tests/test_routers.py index ade565f7..59fdee2a 100644 --- a/python-ecosystem/rag-pipeline/tests/test_routers.py +++ b/python-ecosystem/rag-pipeline/tests/test_routers.py @@ -2,6 +2,8 @@ Tests for rag_pipeline.api.routers — system, parse, index, query, pr. Tests individual route handlers and helper functions. """ +import asyncio + import pytest from unittest.mock import patch, MagicMock from fastapi.testclient import TestClient @@ -20,7 +22,7 @@ def test_root(self): def test_health(self): from rag_pipeline.api.routers.system import health - result = health() + result = asyncio.run(health()) assert result["status"] == "healthy" @patch("rag_pipeline.api.routers.system.gc")