[fix] RAG index generation stuck

analysis-plugins/contracts/python/codecrow_plugins/graphql.py

@@ -9,7 +9,10 @@
 _TOKEN = re.compile(
     r"(?P<space>\s+)"
     r"|(?P<comment>\#[^\r\n]*)"
-    r"|(?P<block>\"\"\"(?:.|\n)*?\"\"\")"
+    # DOTALL already makes ``.`` consume newlines.  ``(?:.|\n)`` gave the
+    # regex engine two ways to consume every newline and caused exponential
+    # backtracking on Java text blocks that were not GraphQL documents.
+    r"|(?P<block>\"\"\".*?\"\"\")"
     r"|(?P<template>`(?:\\.|[^`\\])*`)"
     r"|(?P<single>'(?:\\.|[^'\\])*')"
     r"|(?P<string>\"(?:\\.|[^\"\\])*\")"

analysis-plugins/contracts/python/codecrow_plugins/runtime.py

@@ -603,7 +603,12 @@ def __init__(
     def active(self) -> bool:
         return bool(self._sessions)
 
-    def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None:
+    def ingest(
+        self,
+        artifacts: tuple[FileArtifact, ...],
+        *,
+        progress_callback: Callable[[dict[str, object]], None] | None = None,
+    ) -> None:
         if self._finished:
             raise RuntimeError("repository analysis is already finished")
         if tuple(sorted(artifact.path for artifact in artifacts)) != tuple(
@@ -612,6 +617,20 @@ def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None:
             raise ValueError("repository artifacts must be path-sorted")
         retained: list[tuple[str, object]] = []
         for plugin_id, session in self._sessions:
+            plugin_started = time.monotonic()
+            progress_details: dict[str, object] = {
+                "pluginId": plugin_id,
+                "substage": "ingest",
+                "status": "started",
+                "files": len(artifacts),
+                "message": f"Ingesting repository files with {plugin_id}",
+            }
+            if artifacts:
+                progress_details.update({
+                    "firstPath": artifacts[0].path,
+                    "lastPath": artifacts[-1].path,
+                })
+            self._report_progress(progress_callback, progress_details)
             for artifact in artifacts:
                 try:
                     session.ingest((artifact,))
@@ -623,6 +642,16 @@ def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None:
                         path=artifact.path,
                         recoverable=True,
                     ))
+            duration_ms = round((time.monotonic() - plugin_started) * 1000)
+            self._report_progress(progress_callback, {
+                **progress_details,
+                "status": "completed",
+                "durationMs": duration_ms,
+                "message": (
+                    f"Ingested {len(artifacts)} repository files with "
+                    f"{plugin_id} in {duration_ms} ms"
+                ),
+            })
             retained.append((plugin_id, session))
         self._sessions = retained
 

analysis-plugins/contracts/python/tests/test_data_contracts_repository_analysis.py

@@ -206,6 +206,53 @@ def test_host_language_query_identifier_is_not_graphql():
     ) == ()
 
 
+def test_java_sql_text_blocks_do_not_stall_or_enter_contract_snapshot():
+    java_repository = '''
+public interface JobRepository {
+    @Query("SELECT j FROM Job j WHERE j.id = :jobId")
+    Optional<Job> findById(long jobId);
+
+    @Query(value = """
+            UPDATE job j
+               SET updated_at = :renewedAt
+             WHERE j.id = :jobId
+               AND j.status = 'RUNNING'
+               AND NOT EXISTS (
+                    SELECT 1 FROM rag_index_operation o WHERE o.job_id = j.id
+               )
+            """, nativeQuery = true)
+    int renewLease(long jobId);
+}
+'''
+    assert parse_operations(java_repository, embedded_only=True) == ()
+
+    files = {
+        "schema/job.graphqls": "type Query { job: Job } type Job { id: ID }",
+        "src/JobRepository.java": java_repository,
+    }
+    catalog = PluginCatalog.discover(PLUGINS_ROOT)
+    runtime = PluginRuntime(catalog)
+    capabilities = ProjectSelector(catalog.registry).select(RepositoryFacts(
+        revision=REVISION,
+        paths=tuple(sorted(files)),
+    ))
+    handle = runtime.start_repository_analysis(capabilities, REVISION)
+    handle.ingest(tuple(
+        FileArtifact(path, content)
+        for path, content in sorted(files.items())
+    ))
+
+    analysis, diagnostics = handle.finish()
+
+    assert diagnostics == ()
+    snapshot = next(
+        item for item in analysis.snapshots
+        if item.kind == "data-contract-reference-graph"
+    )
+    records = json.loads(gzip.decompress(base64.b64decode(snapshot.content)))
+    assert [record["path"] for record in records] == ["schema/job.graphqls"]
+
+
 def test_json_reference_uses_its_actual_source_line():
     files = {
         "schemas/base.schema.json": '{"type":"object"}',

analysis-plugins/contracts/python/tests/test_repository_runtime_resilience.py

@@ -43,14 +43,24 @@ def test_repository_runtime_quarantines_ingest_failure_and_keeps_session():
         [("test-plugin", session)],
         [],
     )
+    progress_events = []
 
     handle.ingest((
         FileArtifact("bad.xml", "<invalid>"),
         FileArtifact("good.xml", "<valid />"),
-    ))
+    ), progress_callback=progress_events.append)
     _analysis, diagnostics = handle.finish()
 
     assert session.ingested == ["good.xml"]
+    assert [event["status"] for event in progress_events] == [
+        "started",
+        "completed",
+    ]
+    assert all(event["pluginId"] == "test-plugin" for event in progress_events)
+    assert all(event["files"] == 2 for event in progress_events)
+    assert all(event["firstPath"] == "bad.xml" for event in progress_events)
+    assert all(event["lastPath"] == "good.xml" for event in progress_events)
+    assert progress_events[-1]["durationMs"] >= 0
     assert [
         (diagnostic.code, diagnostic.path, diagnostic.recoverable)
         for diagnostic in diagnostics

analysis-plugins/domains/data-contracts/python/codecrow_plugin_data_contracts/__init__.py

@@ -45,6 +45,12 @@
 _JSON_REF = re.compile(
     r'(?<!\\)"\$ref"\s*:\s*(?P<value>"(?:\\.|[^"\\])*")',
 )
+_EMBEDDED_GRAPHQL_SIGNAL = re.compile(
+    r"(?:\"\"\"|[\"'`])\s*"
+    r"(?:\{|query\b|mutation\b|subscription\b)"
+    r"|<script\b[^>]*\btype\s*=\s*['\"]application/(?:graphql|gql)['\"]",
+    re.IGNORECASE,
+)
 
 
 @dataclass(frozen=True, order=True)
@@ -145,14 +151,25 @@ def _record(artifact: FileArtifact) -> ContractFileRecord | None:
         if lowered.endswith((".graphqls", ".graphql")) and contract
         else ()
     )
+    graphql_source = lowered.endswith((".graphql", ".graphqls"))
     references: tuple[ReferenceOccurrence, ...] = ()
-    if not lowered.endswith(".graphqls"):
+    if lowered.endswith(".graphql"):
         references = _graphql_references(
             artifact.content,
-            embedded_only=not lowered.endswith((".graphql", ".graphqls")),
+            embedded_only=False,
+        )
+    elif (
+        not graphql_source
+        and _EMBEDDED_GRAPHQL_SIGNAL.search(artifact.content) is not None
+    ):
+        references = _graphql_references(
+            artifact.content,
+            embedded_only=True,
         )
     if lowered.endswith(".json"):
         references = tuple(sorted({*references, *_json_references(artifact.content)}))
+    if not contract and not references:
+        return None
     return ContractFileRecord(
         path=artifact.path,
         is_contract=contract,

java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java

@@ -48,6 +48,7 @@ interface ActiveGenerationCoordinates {
         String getRepresentationFingerprint();
         Integer getFileCount();
         Integer getChunkCount();
+        OffsetDateTime getActivatedAt();
     }
 
     interface TransientCleanupCandidate {
@@ -71,7 +72,8 @@ interface TransientCleanupCandidate {
                    g.collectionName AS collectionName,
                    g.representationFingerprint AS representationFingerprint,
                    g.fileCount AS fileCount,
-                   g.chunkCount AS chunkCount
+                   g.chunkCount AS chunkCount,
+                   g.activatedAt AS activatedAt
             FROM RagBranchIndex b
             JOIN b.activeGeneration g
             WHERE b.project.id = :projectId

java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java

@@ -107,7 +107,8 @@ public AdmittedBuild admit(
                     branch,
                     activeSource.getRevision(),
                     activeSource.getFileCount(),
-                    activeSource.getChunkCount());
+                    activeSource.getChunkCount(),
+                    activeSource.getActivatedAt());
         }
         Job job = jobService.createRagIndexJob(
                 project,

java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java

@@ -296,7 +296,8 @@ public void preparePublishedGenerationForUpdate(
             String branchName,
             String commitHash,
             Integer fileCount,
-            Integer chunkCount) {
+            Integer chunkCount,
+            OffsetDateTime activatedAt) {
         RagIndexStatus status = ragIndexStatusRepository
                 .findByProjectIdForUpdate(project.getId())
                 .orElseGet(() -> {
@@ -316,7 +317,13 @@ public void preparePublishedGenerationForUpdate(
         if (chunkCount != null) {
             status.setChunkCount(chunkCount);
         }
-        status.setLastIndexedAt(OffsetDateTime.now());
+        if (activatedAt != null) {
+            status.setLastIndexedAt(activatedAt);
+        } else if (status.getLastIndexedAt() == null) {
+            // Legacy generations may predate activation timestamps. Preserve
+            // an existing completed checkpoint; only initialize a missing one.
+            status.setLastIndexedAt(OffsetDateTime.now());
+        }
         status.setErrorMessage(null);
         status.setActiveJobId(null);
         status.resetFailedIncrementalCount();

java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java

@@ -463,7 +463,8 @@ && isRagEnabled(project)
                             ragIndexTrackingService.preparePublishedGenerationForUpdate(
                                     project, branchName, commitHash,
                                     sourceGeneration.getFileCount(),
-                                    sourceGeneration.getChunkCount());
+                                    sourceGeneration.getChunkCount(),
+                                    sourceGeneration.getActivatedAt());
                         }
                         emitEvent(eventConsumer, Map.of(
                                 "type", "info",

java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java

@@ -18,6 +18,8 @@
 import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService;
 import org.springframework.test.util.ReflectionTestUtils;
 
+import java.time.OffsetDateTime;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.*;
@@ -33,6 +35,7 @@ class BranchIndexBuildAdmissionServiceTest {
     private BranchIndexBuildAdmissionService service;
     private Project project;
     private RagBranchIndexRegistryService.BuildRegistration registration;
+    private OffsetDateTime sourceActivatedAt;
 
     @BeforeEach
     void setUp() {
@@ -47,6 +50,8 @@ void setUp() {
                 branchIndex, "revision-a", "source-target", null, null, null);
         source.setId(19L);
         source.activate("source-manifest", 120, 240);
+        sourceActivatedAt = OffsetDateTime.parse("2026-08-17T15:27:17Z");
+        source.setActivatedAt(sourceActivatedAt);
         RagBranchIndexGeneration generation = new RagBranchIndexGeneration(
                 branchIndex, "revision-b", "physical-target", source, null, null);
         generation.setId(20L);
@@ -90,7 +95,7 @@ void registersThenAtomicallyLinksAndStartsJobAndOperation() {
                 eq(project), eq("main"), eq(RagBranchIndexKind.PRIMARY),
                 isNull(), eq("revision-b"), startsWith("exact-full-snapshot:automatic:"));
         order.verify(trackingService).preparePublishedGenerationForUpdate(
-                project, "main", "revision-a", 120, 240);
+                project, "main", "revision-a", 120, 240, sourceActivatedAt);
         order.verify(jobService).createRagIndexJob(
                 project, false, JobTriggerSource.WEBHOOK, "main", "revision-b");
         order.verify(registryService).startBuild(30L, 77L, "lock-owner-123");
@@ -148,7 +153,7 @@ void initialPrimaryAdmissionStartsIndexingWithoutInventingASourceCheckpoint() {
         verify(trackingService).markIndexingStarted(
                 project, "main", "revision-first", 78L);
         verify(trackingService, never()).preparePublishedGenerationForUpdate(
-                any(), anyString(), anyString(), any(), any());
+                any(), anyString(), anyString(), any(), any(), any());
         verify(trackingService, never()).markUpdatingStarted(
                 any(), anyString(), anyString(), any());
     }

java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java

@@ -370,6 +370,33 @@ void testMarkUpdatingCompleted_NonBaseBranchPreservesProjectCheckpoint() {
         assertThat(result.getIndexedCommitHash()).isEqualTo("main-commit");
     }
 
+    @Test
+    void preparePublishedGenerationUsesItsActualActivationTime() {
+        OffsetDateTime previousTimestamp = OffsetDateTime.parse(
+                "2026-08-10T12:00:00Z");
+        OffsetDateTime generationActivatedAt = OffsetDateTime.parse(
+                "2026-08-17T15:27:17Z");
+        RagIndexStatus existing = new RagIndexStatus();
+        existing.setProject(testProject);
+        existing.setStatus(RagIndexingStatus.INDEXED);
+        existing.setLastIndexedAt(previousTimestamp);
+        when(ragIndexStatusRepository.findByProjectIdForUpdate(100L))
+                .thenReturn(Optional.of(existing));
+
+        service.preparePublishedGenerationForUpdate(
+                testProject,
+                "master",
+                "cf74934b6c7e",
+                4277,
+                39323,
+                generationActivatedAt);
+
+        assertThat(existing.getIndexedBranch()).isEqualTo("master");
+        assertThat(existing.getIndexedCommitHash()).isEqualTo("cf74934b6c7e");
+        assertThat(existing.getLastIndexedAt()).isEqualTo(generationActivatedAt);
+        verify(ragIndexStatusRepository).save(existing);
+    }
+
     // ── markIncrementalUpdateFailed ──────────────────────────────────────────
 
     @Test

java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java

@@ -38,6 +38,7 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.test.util.ReflectionTestUtils;
 
+import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -484,7 +485,10 @@ void exactIncrementalAlreadyAtTargetCompletesJobAndRepairsPrimaryCheckpoint()
                 .thenReturn(1);
         RagBranchIndexRepository.ActiveGenerationCoordinates source =
                 mock(RagBranchIndexRepository.ActiveGenerationCoordinates.class);
+        OffsetDateTime activatedAt = OffsetDateTime.parse(
+                "2026-08-17T15:27:17Z");
         when(source.getRevision()).thenReturn("target-revision");
+        when(source.getActivatedAt()).thenReturn(activatedAt);
         when(ragBranchIndexRepository.findActiveGenerationCoordinates(100L, "main"))
                 .thenReturn(Optional.of(source));
         @SuppressWarnings("unchecked")
@@ -495,7 +499,7 @@ void exactIncrementalAlreadyAtTargetCompletesJobAndRepairsPrimaryCheckpoint()
 
         assertThat(result).isTrue();
         verify(ragIndexTrackingService).preparePublishedGenerationForUpdate(
-                testProject, "main", "target-revision", 0, 0);
+                testProject, "main", "target-revision", 0, 0, activatedAt);
         verifyNoInteractions(analysisJobService);
         verify(analysisLockService).releaseLock("rag-lock");
         verifyNoInteractions(vcsClientProvider, registry, builder);