diff --git a/deployment/config/inference-orchestrator/.env.sample b/deployment/config/inference-orchestrator/.env.sample index 95f6ef62..d8e365e2 100644 --- a/deployment/config/inference-orchestrator/.env.sample +++ b/deployment/config/inference-orchestrator/.env.sample @@ -16,6 +16,7 @@ SERVICE_SECRET=change-me-to-a-random-secret # ANALYSIS_CONSUMER_HEARTBEAT_SECONDS=5 # MAX_CONCURRENT_COMMANDS=10 # COMMAND_TIMEOUT_SECONDS=600 +# COMMAND_EVENT_TTL_SECONDS=3600 # REVIEW_TIMEOUT_SECONDS=1500 # REVIEW_GLOBAL_RAG_QUERY_TIMEOUT_SECONDS=5 diff --git a/deployment/config/java-shared/application.properties.sample b/deployment/config/java-shared/application.properties.sample index c788fb31..245f789c 100644 --- a/deployment/config/java-shared/application.properties.sample +++ b/deployment/config/java-shared/application.properties.sample @@ -192,6 +192,19 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF # Repair interval for readable Qdrant current-branch aliases of active generations. #codecrow.rag.operator-alias.reconcile-interval-ms=300000 #codecrow.rag.operator-alias.reconcile-initial-delay-ms=15000 +# Exact-generation operation recovery. A live build heartbeats its persisted +# operation; a published operation also repairs an interrupted job/status handoff. +#codecrow.rag.generation.stale-after-minutes=30 +#codecrow.rag.generation.recovery-interval-ms=300000 +#codecrow.rag.generation.recovery-initial-delay-ms=60000 +# Legacy unsealed incremental updates use a separate database job lease because +# they do not own an immutable generation operation. +#codecrow.rag.legacy-job.lease-seconds=120 +#codecrow.rag.legacy-job.heartbeat-interval-seconds=15 +#codecrow.rag.legacy-job.heartbeat-threads=4 +#codecrow.rag.legacy-job.recovery-interval-ms=30000 +#codecrow.rag.legacy-job.recovery-initial-delay-ms=30000 +#codecrow.rag.legacy-job.recovery-batch-size=100 # Shared VCS acquisition threshold for incremental RAG and reconciliation # fallback. The legacy codecrow.rag.incremental.archive-file-threshold property # remains a fallback when this property is not set. @@ -200,6 +213,10 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF # Analysis Lock Configuration # Lock timeout - maximum time for a single analysis to hold a lock (in minutes) #analysis.lock.timeout.minutes=30 +# Independent lease-renewal scheduler capacity (default 4, minimum 2 threads) +#analysis.lock.heartbeat.threads=4 +# Independent lease-renewal interval (seconds); bounded to one-third of the lease +#analysis.lock.heartbeat.interval.seconds=60 # RAG indexing lock timeout - longer timeout for RAG indexing operations (in minutes, default 6 hours) #analysis.lock.rag.timeout.minutes=360 # Lock wait timeout - maximum time to wait for a lock to be released (in minutes) diff --git a/deployment/config/rag-pipeline/.env.sample b/deployment/config/rag-pipeline/.env.sample index 8d678290..936b514f 100644 --- a/deployment/config/rag-pipeline/.env.sample +++ b/deployment/config/rag-pipeline/.env.sample @@ -49,6 +49,7 @@ SERVICE_SECRET=change-me-to-a-random-secret # QDRANT_URL=http://qdrant:6333 # QDRANT_API_KEY= # QDRANT_COLLECTION_PREFIX=codecrow +# QDRANT_TIMEOUT_SECONDS=30 # QDRANT_VECTORS_ON_DISK=true # QDRANT_UPSERT_BATCH_SIZE=128 diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java index 58642f6d..5b91670f 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java @@ -12,11 +12,13 @@ import org.rostilos.codecrow.queue.RedisQueueService; import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import java.io.IOException; import java.security.GeneralSecurityException; +import java.time.OffsetDateTime; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -40,6 +42,9 @@ public class AiAnalysisClient { @Autowired(required = false) private RagBranchIndexGenerationRepository branchGenerationRepository; + @Autowired(required = false) + private RagBranchIndexRepository branchIndexRepository; + static final String INACTIVITY_TIMEOUT_MINUTES_KEY = "ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES"; static final String ADMISSION_TIMEOUT_MINUTES_KEY = @@ -353,11 +358,16 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re payload.put("previousCommitHash", request.getPreviousCommitHash()); payload.put("currentCommitHash", request.getCurrentCommitHash()); payload.put("baseCommitHash", request.getBaseCommitHash()); - if (branchGenerationRepository != null + if (request.getRagEnabled() + && branchGenerationRepository != null + && branchIndexRepository != null && request.getProjectId() != null && request.getTargetBranchName() != null && request.getBaseCommitHash() != null) { - branchGenerationRepository.findAvailableExactGeneration( + int accessed = branchIndexRepository.markAccessedIfUnclaimed( + request.getProjectId(), request.getTargetBranchName(), OffsetDateTime.now()); + if (accessed > 0) { + branchGenerationRepository.findAvailableExactGeneration( request.getProjectId(), request.getTargetBranchName(), request.getBaseCommitHash(), @@ -371,6 +381,7 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re payload.put("ragBaseGenerationManifestSha256", generation.getManifestDigest()); }); + } } payload.put("previousCodeAnalysisIssues", request.getPreviousCodeAnalysisIssues()); payload.put("reconciliationFileContents", request.getReconciliationFileContents()); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java index 784a2012..3fc411dc 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java @@ -4,6 +4,7 @@ import org.rostilos.codecrow.queue.RedisQueueService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.IOException; @@ -24,10 +25,35 @@ public class AiCommandClient { private final RedisQueueService queueService; private final ObjectMapper objectMapper; private static final int COMMAND_TIMEOUT_MINUTES = 30; - + static final String CONSUMER_HEARTBEAT_KEY = + "codecrow:commands:consumer:heartbeat"; + private static final long DEFAULT_ADMISSION_TIMEOUT_MINUTES = 5L; + private static final String ADMISSION_TIMEOUT_MINUTES_KEY = + "COMMAND_QUEUE_ADMISSION_TIMEOUT_MINUTES"; + private final long commandTimeoutMillis; + private final long admissionTimeoutMillis; + + @Autowired public AiCommandClient(RedisQueueService queueService, ObjectMapper objectMapper) { + this( + queueService, + objectMapper, + TimeUnit.MINUTES.toMillis(COMMAND_TIMEOUT_MINUTES), + resolveAdmissionTimeoutMillis()); + } + + AiCommandClient( + RedisQueueService queueService, + ObjectMapper objectMapper, + long commandTimeoutMillis, + long admissionTimeoutMillis) { this.queueService = queueService; this.objectMapper = objectMapper; + if (commandTimeoutMillis <= 0 || admissionTimeoutMillis <= 0) { + throw new IllegalArgumentException("Command queue timeouts must be positive"); + } + this.commandTimeoutMillis = commandTimeoutMillis; + this.admissionTimeoutMillis = admissionTimeoutMillis; } /** @@ -85,24 +111,44 @@ private Map executeAsyncJob( Consumer> eventHandler) throws IOException { String eventQueueKey = "codecrow:analysis:events:" + jobId; String jobsQueueKey = "codecrow:queue:commands"; + String jsonPayload = null; try { + if (!queueService.hasKey(CONSUMER_HEARTBEAT_KEY)) { + throw new IOException( + "Inference Orchestrator is unavailable: no live command queue consumer"); + } Map jobPayload = Map.of( "job_id", jobId, "command_type", commandType, "request", request); - String jsonPayload = objectMapper.writeValueAsString(jobPayload); + jsonPayload = objectMapper.writeValueAsString(jobPayload); queueService.leftPush(jobsQueueKey, jsonPayload); queueService.setExpiry(eventQueueKey, COMMAND_TIMEOUT_MINUTES + 1); long startTime = System.currentTimeMillis(); - long timeoutMillis = TimeUnit.MINUTES.toMillis(COMMAND_TIMEOUT_MINUTES); + boolean workerAcknowledged = false; while (true) { - if (System.currentTimeMillis() - startTime > timeoutMillis) { + long now = System.currentTimeMillis(); + if (!workerAcknowledged) { + if (!queueService.hasKey(CONSUMER_HEARTBEAT_KEY)) { + throw new IOException( + "Inference Orchestrator became unavailable before " + + "the command job was admitted"); + } + if (now - startTime > admissionTimeoutMillis) { + throw new IOException( + "AI command was not admitted by Inference Orchestrator within " + + TimeUnit.MILLISECONDS.toSeconds(admissionTimeoutMillis) + + " seconds"); + } + } + if (now - startTime > commandTimeoutMillis) { throw new IOException( - "AI command timed out after " + COMMAND_TIMEOUT_MINUTES + " minutes for Job: " + jobId); + "AI command timed out after " + commandTimeoutMillis + + "ms for Job: " + jobId); } String eventJson = queueService.rightPop(eventQueueKey, 5); @@ -110,6 +156,7 @@ private Map executeAsyncJob( if (eventJson == null) { continue; // Timeout on rightPop, continue to check overall timeout } + workerAcknowledged = true; try { Map event = objectMapper.readValue(eventJson, Map.class); @@ -150,6 +197,14 @@ private Map executeAsyncJob( log.error("Failed to communicate with AI async queue", e); throw new IOException("AI queue communication failed: " + e.getMessage(), e); } finally { + if (jsonPayload != null) { + try { + queueService.removeFromList(jobsQueueKey, jsonPayload); + } catch (Exception cleanupError) { + log.warn("Failed to remove command job {} from pending queue: {}", + jobId, cleanupError.getMessage()); + } + } try { queueService.deleteKey(eventQueueKey); } catch (Exception ignored) { @@ -157,6 +212,27 @@ private Map executeAsyncJob( } } + private static long resolveAdmissionTimeoutMillis() { + String configured = System.getProperty(ADMISSION_TIMEOUT_MINUTES_KEY); + if (configured == null || configured.isBlank()) { + configured = System.getenv(ADMISSION_TIMEOUT_MINUTES_KEY); + } + if (configured == null || configured.isBlank()) { + return TimeUnit.MINUTES.toMillis(DEFAULT_ADMISSION_TIMEOUT_MINUTES); + } + try { + long minutes = Long.parseLong(configured.trim()); + if (minutes <= 0) { + throw new IllegalArgumentException( + ADMISSION_TIMEOUT_MINUTES_KEY + " must be a positive integer"); + } + return TimeUnit.MINUTES.toMillis(minutes); + } catch (NumberFormatException invalid) { + throw new IllegalArgumentException( + ADMISSION_TIMEOUT_MINUTES_KEY + " must be a positive integer", invalid); + } + } + /** * Request object for summarize endpoint. */ diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java index ea3e8f93..cb735f3c 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java @@ -195,13 +195,19 @@ private Map process( } List unanalyzedCommits = Collections.emptyList(); + AnalysisLockService.LockLease lockLease = null; try { + lockLease = analysisLockService.maintainLockLease( + lockKey.get(), + analysisLockService.getLeaseMinutes(AnalysisLockType.BRANCH_ANALYSIS)); + requireActiveLease(lockLease); + requireConfirmedLease(lockLease); Optional existingBranchOpt = branchRepository.findByProjectIdAndBranchName( project.getId(), request.getTargetBranchName()); if (request.getSourcePrNumber() == null - && matchCache(request, existingBranchOpt, project, consumer)) { + && matchCache(request, existingBranchOpt, project, consumer, lockLease)) { return Map.of( "status", "skipped", "reason", "commit_already_analyzed", @@ -295,6 +301,7 @@ && matchCache(request, existingBranchOpt, project, consumer)) { // This keeps PullRequestState accurate for commit coverage checks. Set mergedPrNumbers = new LinkedHashSet<>(); if (prNumber != null) { + requireConfirmedLease(lockLease); try { pullRequestService.markPullRequestMerged(project.getId(), prNumber); mergedPrNumbers.add(prNumber); @@ -305,6 +312,7 @@ && matchCache(request, existingBranchOpt, project, consumer)) { } if (prNumber != null || isMergeCommit) { + requireConfirmedLease(lockLease); PullRequestStatusSyncService.SyncResult syncResult = pullRequestStatusSyncService .syncOpenPullRequestStates( project, request.getTargetBranchName(), consumer); @@ -320,6 +328,7 @@ && matchCache(request, existingBranchOpt, project, consumer)) { && prNumber == null && !isMergeCommit && mergedPrNumbers.isEmpty()) { + requireConfirmedLease(lockLease); return skipAlreadyAnalyzedRange(project, request, existingBranchOpt, consumer); } @@ -381,12 +390,16 @@ && matchCache(request, existingBranchOpt, project, consumer)) { } if (changedFiles.isEmpty()) { + requireConfirmedLease(lockLease); branchFileOperationsService.createOrUpdateProjectBranch( project, request, existingBranchOpt.orElse(null)); EventNotificationEmitter.emitStatus(consumer, "skipped", "No changed files match the project analysis scope"); + requireConfirmedLease(lockLease); performIncrementalRagUpdate(request, project, repositoryDiff, consumer, directPushLimited); + requireConfirmedLease(lockLease); branchHealthService.markBranchHealthy(project, request); + requireConfirmedLease(lockLease); branchHealthService.recordCommitsAnalyzed(project, unanalyzedCommits, request.getTargetBranchName()); return Map.of("status", "accepted", "cached", false, @@ -403,7 +416,8 @@ && matchCache(request, existingBranchOpt, project, consumer)) { if (!isFirstAnalysis && !directPushLimited) { performDirectPushAnalysisIfNeeded( project, request, unanalyzedCommits, rawDiff, - changedFiles, provider, consumer, prNumber, isMergeCommit); + changedFiles, provider, consumer, prNumber, isMergeCommit, + lockLease); } else if (isFirstAnalysis) { log.info( "First analysis for branch {} — skipping direct push analysis (establishing baseline, {} files)", @@ -422,12 +436,15 @@ && matchCache(request, existingBranchOpt, project, consumer)) { log.info("Branch archive: {} files extracted for {} changed files", branchFileSnapshot.contents().size(), changedFiles.size()); + requireConfirmedLease(lockLease); Set existingFiles = branchFileOperationsService.updateBranchFiles( changedFiles, project, request.getTargetBranchName(), branchFileSnapshot); + requireConfirmedLease(lockLease); Branch branch = branchFileOperationsService.createOrUpdateProjectBranch( project, request, existingBranchOpt.orElse(null)); + requireConfirmedLease(lockLease); if (mergedPrNumbers.size() > 1) { branchIssueMappingService.mapCodeAnalysisIssuesToBranch( changedFiles, existingFiles, branch, project, mergedPrNumbers); @@ -435,15 +452,18 @@ && matchCache(request, existingBranchOpt, project, consumer)) { branchIssueMappingService.mapCodeAnalysisIssuesToBranch( changedFiles, existingFiles, branch, project, prNumber); } + requireConfirmedLease(lockLease); branchIssueReconciliationService.reconcileIssueLineNumbers(rawDiff, changedFiles, branch); // Update branch issue counts after mapping + requireConfirmedLease(lockLease); Branch refreshedBranch = refreshAndSaveIssueCounts(branch); log.info("Updated branch issue counts after mapping: total={}, high={}, medium={}, low={}, resolved={}", refreshedBranch.getTotalIssues(), refreshedBranch.getHighSeverityCount(), refreshedBranch.getMediumSeverityCount(), refreshedBranch.getLowSeverityCount(), refreshedBranch.getResolvedCount()); + requireConfirmedLease(lockLease); branchIssueReconciliationService.reanalyzeCandidateIssues( changedFiles, existingFiles, refreshedBranch, project, request, consumer, branchFileSnapshot.contents(), rawDiff, @@ -457,17 +477,22 @@ && matchCache(request, existingBranchOpt, project, consumer)) { EventNotificationEmitter.emitStatus(consumer, "finalizing_branch_state", "Saving changed-file reconciliation results"); + requireConfirmedLease(lockLease); branchFileOperationsService.updateFileSnapshotsForBranch(existingFiles, project, request, branchFileSnapshot); Branch branchForVerify = branchRepository.findByProjectIdAndBranchName( project.getId(), request.getTargetBranchName()).orElse(refreshedBranch); + requireConfirmedLease(lockLease); branchIssueReconciliationService.verifyIssueLineNumbersWithSnippets( changedFiles, project, branchForVerify); // ── Post-analysis housekeeping ──────────────────────────────────── + requireConfirmedLease(lockLease); performIncrementalRagUpdate(request, project, repositoryDiff, consumer, directPushLimited); + requireConfirmedLease(lockLease); branchHealthService.markBranchHealthy(project, request); + requireConfirmedLease(lockLease); branchHealthService.recordCommitsAnalyzed(project, unanalyzedCommits, request.getTargetBranchName()); @@ -477,11 +502,36 @@ && matchCache(request, existingBranchOpt, project, consumer)) { return Map.of("status", "accepted", "cached", false, "branch", request.getTargetBranchName()); + } catch (BranchAnalysisLeaseLostException e) { + log.info("Branch analysis stopped after lock lease ownership was lost: project={}, branch={}", + project.getId(), request.getTargetBranchName()); + throw e; } catch (Exception e) { branchHealthService.handleProcessFailure(project, request, unanalyzedCommits, e); throw e; } finally { - analysisLockService.releaseLock(lockKey.get()); + if (lockLease != null) { + try { + lockLease.close(); + } catch (RuntimeException closeFailure) { + log.info( + "Branch analysis lock lease could not be closed after processing; " + + "continuing lock cleanup: project={}, branch={}, detail={}", + project.getId(), request.getTargetBranchName(), + closeFailure.getMessage()); + } + } + try { + analysisLockService.releaseLock(lockKey.get()); + } catch (RuntimeException releaseFailure) { + // The branch outcome is already decided. Cleanup failure is + // observational; the durable lock lease will expire. + log.info( + "Branch analysis lock could not be released after processing; " + + "leaving it to expiry: project={}, branch={}, detail={}", + project.getId(), request.getTargetBranchName(), + releaseFailure.getMessage()); + } } } @@ -490,12 +540,36 @@ public Map fullReconcile(Long projectId, String branchName, return branchFullReconciliationService.fullReconcile(projectId, branchName, consumer); } + private static void requireActiveLease(AnalysisLockService.LockLease lease) throws IOException { + if (lease == null || lease.isOwnershipLost()) { + throw lostLeaseException(); + } + } + + private static void requireConfirmedLease(AnalysisLockService.LockLease lease) throws IOException { + if (lease == null || !lease.confirmOwnership()) { + throw lostLeaseException(); + } + } + + private static BranchAnalysisLeaseLostException lostLeaseException() { + return new BranchAnalysisLeaseLostException( + "Branch analysis lost its lock lease while the worker was active"); + } + + private static final class BranchAnalysisLeaseLostException extends IOException { + private BranchAnalysisLeaseLostException(String message) { + super(message); + } + } + /** * Check if the incoming commit was already SUCCESSFULLY analyzed. * Uses lastSuccessfulCommitHash so that failed attempts are re-processed. */ private boolean matchCache(BranchProcessRequest request, Optional existingBranchOpt, - Project project, Consumer> consumer) { + Project project, Consumer> consumer, + AnalysisLockService.LockLease lockLease) throws IOException { if (request.getCommitHash() == null || existingBranchOpt.isEmpty()) return false; @@ -517,9 +591,12 @@ private boolean matchCache(BranchProcessRequest request, Optional existi BranchFileOperationsService.BranchFileSnapshot branchFileSnapshot = branchFileOperationsService.downloadBranchFileSnapshot( vcsRepoInfoImpl, request.getCommitHash(), branchFiles); + requireConfirmedLease(lockLease); branchFileOperationsService.updateFileSnapshotsForBranch( branchFiles, project, request, branchFileSnapshot); } + } catch (BranchAnalysisLeaseLostException leaseLost) { + throw leaseLost; } catch (Exception snapEx) { log.warn("Failed to refresh file snapshots on skip path (non-critical): {}", snapEx.getMessage()); @@ -699,7 +776,8 @@ private void performDirectPushAnalysisIfNeeded( EVcsProvider provider, Consumer> consumer, Long mergedPrNumber, - boolean isMergeCommit) { + boolean isMergeCommit, + AnalysisLockService.LockLease lockLease) throws IOException { if (unanalyzedCommits.isEmpty()) { log.debug("No unanalyzed commits — skipping direct push analysis check"); @@ -794,9 +872,12 @@ private void performDirectPushAnalysisIfNeeded( project, request, rawDiff, fileContents, new ArrayList<>(changedFiles)); // Call the inference orchestrator using single request + requireConfirmedLease(lockLease); Map aiResponse = aiAnalysisClient.performAnalysis(aiRequests.get(0), event -> { try { - consumer.accept(event); + if (consumer != null) { + consumer.accept(event); + } } catch (Exception ex) { log.debug("Event consumer failed during direct push analysis: {}", ex.getMessage()); @@ -815,6 +896,7 @@ private void performDirectPushAnalysisIfNeeded( } // Save the analysis with DetectionSource.DIRECT_PUSH_ANALYSIS + requireConfirmedLease(lockLease); CodeAnalysis directPushAnalysis = codeAnalysisService.createDirectPushAnalysisFromAiResponse( project, aiResponse, request.getTargetBranchName(), request.getCommitHash(), fileContents); @@ -842,6 +924,8 @@ private void performDirectPushAnalysisIfNeeded( EventNotificationEmitter.emitStatus(consumer, "direct_push_analysis_complete", "Direct push analysis found " + issuesFound + " issues"); + } catch (BranchAnalysisLeaseLostException leaseLost) { + throw leaseLost; } catch (Exception e) { // Direct push analysis failure is non-fatal — reconciliation will still run log.warn("Direct push analysis failed (non-fatal, reconciliation will still run): {}", @@ -913,7 +997,8 @@ private void performIncrementalRagUpdate(BranchProcessRequest request, Project p boolean ragUpdated = ragOperationsService.triggerIncrementalUpdate( project, targetBranch, request.getCommitHash(), commitDiff, consumer); if (!ragUpdated) { - log.warn("RAG incremental update did not complete for project={}, branch={}, commit={}", + log.info("RAG incremental update did not complete; retaining the last usable index " + + "for project={}, branch={}, commit={}", project.getId(), targetBranch, request.getCommitHash()); return; } @@ -921,7 +1006,8 @@ private void performIncrementalRagUpdate(BranchProcessRequest request, Project p log.info("Non-main branch push - updating branch index for project={}, branch={}", project.getId(), targetBranch); if (!ragOperationsService.updateBranchIndex(project, targetBranch, consumer)) { - log.warn("RAG branch index update did not complete for project={}, branch={}", + log.info("RAG branch index update did not complete; retaining the last usable index " + + "for project={}, branch={}", project.getId(), targetBranch); return; } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java index bb6cfc6b..a8a83947 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java @@ -46,7 +46,6 @@ import java.util.Optional; import java.util.Collections; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.rostilos.codecrow.analysisengine.util.DiffFingerprintUtil; @@ -148,7 +147,7 @@ public Map process( AnalysisLockType.PR_ANALYSIS, request.getCommitHash(), request.getPullRequestId(), - consumer::accept); + event -> emitEvent(consumer, event)); if (acquiredLock.isEmpty()) { String message = String.format( @@ -171,7 +170,12 @@ public Map process( lockKey = acquiredLock.get(); } + AnalysisLockService.LockLease lockLease = null; try { + lockLease = analysisLockService.maintainLockLease( + lockKey, + analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS)); + requireActiveLease(lockLease); EVcsProvider provider = ProjectVcsInfoRetriever.getVcsProvider(project); VcsReportingService reportingService = vcsServiceFactory.getReportingService(provider); // Get all previous analyses for this PR to provide full issue history to AI @@ -190,6 +194,7 @@ public Map process( // Request construction acquires and verifies the provider's immutable // full head object ID. Persist the PR only after that canonicalization, // never from an abbreviated or otherwise unverified webhook value. + requireConfirmedLease(lockLease); PullRequest pullRequest = pullRequestService.createOrUpdatePullRequest( request.getProjectId(), request.getPullRequestId(), @@ -202,7 +207,8 @@ public Map process( String message = "No changed files match the project analysis scope"; log.info("Skipping PR analysis for project={}, PR={}: {}", project.getId(), request.getPullRequestId(), message); - consumer.accept(Map.of("type", "info", "message", message)); + emitEvent(consumer, Map.of("type", "info", "message", message)); + requireConfirmedLease(lockLease); publishAnalysisCompletedEvent(project, request, correlationId, startTime, AnalysisCompletedEvent.CompletionStatus.SUCCESS, 0, 0, null); return Map.of("status", "ignored", "message", message); @@ -216,8 +222,9 @@ public Map process( CacheHitType cacheHit = postAnalysisCacheIfExist( project, pullRequest, request.getCommitHash(), request.getPullRequestId(), reportingService, request.getPlaceholderCommentId(), request.getTargetBranchName(), - request.getSourceBranchName(), diffFingerprint); + request.getSourceBranchName(), diffFingerprint, lockLease); if (cacheHit != CacheHitType.NONE) { + requireConfirmedLease(lockLease); publishAnalysisCompletedEvent(project, request, correlationId, startTime, AnalysisCompletedEvent.CompletionStatus.SUCCESS, 0, 0, null); String cacheStatus = cacheHit == CacheHitType.COMMIT_HASH ? "cached_by_commit" : "cached"; @@ -225,8 +232,10 @@ public Map process( } if (postDiffFingerprintCacheIfExist( - request, diffFingerprint, project, pullRequest, aiRequest, reportingService + request, diffFingerprint, project, pullRequest, aiRequest, reportingService, + lockLease )) { + requireConfirmedLease(lockLease); publishAnalysisCompletedEvent(project, request, correlationId, startTime, AnalysisCompletedEvent.CompletionStatus.SUCCESS, 0, 0, null); return Map.of("status", "cached_by_fingerprint", "cached", true); @@ -240,40 +249,18 @@ public Map process( // Only prepare RAG after the exact snapshot/configuration cache has missed. ensureRagIndexForTargetBranch(project, request.getTargetBranchName(), consumer); - AtomicBoolean lockLeaseLost = new AtomicBoolean(false); Map aiResponse = aiAnalysisClient.performAnalysis(aiRequest, event -> { - if ("processing".equals(String.valueOf(event.get("state")))) { - try { - if (!analysisLockService.renewLock( - lockKey, - analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS))) { - lockLeaseLost.set(true); - log.error("PR analysis lost its lock lease: {}", lockKey); - } - } catch (Exception leaseError) { - lockLeaseLost.set(true); - log.error("Failed to renew PR analysis lock lease: {}", lockKey, leaseError); - } - } - try { - log.debug("Received event from AI client: type={}", event.get("type")); - consumer.accept(event); - log.debug("Event forwarded to consumer successfully"); - } catch (Exception ex) { - log.error("Event consumer failed: {}", ex.getMessage(), ex); - } + log.debug("Received event from AI client: type={}", event.get("type")); + emitEvent(consumer, event); }); - if (lockLeaseLost.get()) { - throw new IOException( - "PR analysis lost its lock lease while the review worker was active"); - } + requireConfirmedLease(lockLease); if (AiAnalysisClient.isPromptDryRunResult(aiResponse)) { Object artifact = aiResponse.get("promptArtifact"); log.warn( "Prompt dry run completed for project={}, PR={}; artifact={}", project.getId(), request.getPullRequestId(), artifact); - consumer.accept(Map.of( + emitEvent(consumer, Map.of( "type", "info", "state", "prompt_dry_run_completed", "message", "Prompt dry run completed without publishing an analysis", @@ -297,6 +284,9 @@ public Map process( request.getCommitHash()); } + // Direct VCS fallback can itself be slow. Establish an atomic lease + // barrier immediately before the first durable analysis write. + requireConfirmedLease(lockLease); CodeAnalysis newAnalysis = codeAnalysisService.createAnalysisFromAiResponse( project, aiResponse, @@ -353,6 +343,10 @@ public Map process( log.warn("PR issue tracking failed (non-critical): {}", trackEx.getMessage()); } + // Ownership may have changed after persistence but before external + // publication. Perform an atomic proof instead of waiting for the + // next scheduled heartbeat to observe it. + requireConfirmedLease(lockLease); try { reportingService.postAnalysisResults( newAnalysis, @@ -362,15 +356,17 @@ public Map process( request.getPlaceholderCommentId()); } catch (IOException e) { log.error("Failed to post analysis results to VCS: {}", e.getMessage(), e); - consumer.accept(Map.of( + emitEvent(consumer, Map.of( "type", "warning", "message", "Analysis completed but failed to post results to VCS: " + e.getMessage())); } // === DAG: Mark PR commits as ANALYZED === + requireConfirmedLease(lockLease); markPrCommitsAnalyzed(project, request, newAnalysis); // Publish successful completion event + requireConfirmedLease(lockLease); publishAnalysisCompletedEvent(project, request, correlationId, startTime, AnalysisCompletedEvent.CompletionStatus.SUCCESS, issuesFound, allChangedFiles != null ? allChangedFiles.size() : 0, null); @@ -378,7 +374,7 @@ public Map process( return aiResponse; } catch (IOException e) { log.error("IOException during PR analysis: {}", e.getMessage(), e); - consumer.accept(Map.of( + emitEvent(consumer, Map.of( "type", "error", "message", "Analysis failed due to I/O error: " + e.getMessage())); @@ -388,12 +384,62 @@ public Map process( return Map.of("status", "error", "message", e.getMessage()); } finally { + if (lockLease != null) { + try { + lockLease.close(); + } catch (RuntimeException closeFailure) { + log.info( + "PR analysis lock lease could not be closed after processing; " + + "continuing lock cleanup: project={}, PR={}, detail={}", + project.getId(), request.getPullRequestId(), + closeFailure.getMessage()); + } + } if (!isPreAcquired) { - analysisLockService.releaseLock(lockKey); + try { + analysisLockService.releaseLock(lockKey); + } catch (RuntimeException releaseFailure) { + // The analysis outcome is already decided. Cleanup failure + // is observational; the durable lock lease will expire. + log.info( + "PR analysis lock could not be released after processing; " + + "leaving it to expiry: project={}, PR={}, detail={}", + project.getId(), request.getPullRequestId(), + releaseFailure.getMessage()); + } } } } + private static void requireActiveLease(AnalysisLockService.LockLease lease) throws IOException { + if (lease == null || lease.isOwnershipLost()) { + throw lostLeaseException(); + } + } + + private static void requireConfirmedLease(AnalysisLockService.LockLease lease) throws IOException { + if (lease == null || !lease.confirmOwnership()) { + throw lostLeaseException(); + } + } + + private static IOException lostLeaseException() { + return new IOException("PR analysis lost its lock lease while the review worker was active"); + } + + /** Progress delivery is observational and cannot change review ownership or outcome. */ + private static void emitEvent(EventConsumer consumer, Map event) { + if (consumer == null) { + return; + } + try { + consumer.accept(event); + } catch (RuntimeException observerFailure) { + log.debug("PR progress observer rejected event type={} state={}: {}", + event.get("type"), event.get("state"), observerFailure.getMessage()); + } + } + private String taskContextValue(AiAnalysisRequest aiRequest, String... keys) { Map taskContext = aiRequest.getTaskContext(); if (taskContext == null || taskContext.isEmpty()) { @@ -494,7 +540,8 @@ private Map fetchFileContentsFromVcs(Project project, List changedFiles) { + String commitHash, List changedFiles, + AnalysisLockService.LockLease lockLease) throws IOException { try { // Strategy 1: Copy PR-level snapshots from the source analysis's original PR if (sourceAnalysis.getPrNumber() != null) { @@ -504,6 +551,7 @@ private void persistPrSnapshotsForCacheHit(PullRequest pullRequest, CodeAnalysis Map sourceContents = fileSnapshotService.getFileContentsMapForPr( sourcePr.get().getId()); if (!sourceContents.isEmpty()) { + requireConfirmedLease(lockLease); fileSnapshotService.persistSnapshotsForPr(pullRequest, cloned, sourceContents, commitHash); log.info("Copied {} PR snapshots from source PR {} to PR {} (cache hit)", sourceContents.size(), sourceAnalysis.getPrNumber(), pullRequest.getPrNumber()); @@ -524,9 +572,12 @@ private void persistPrSnapshotsForCacheHit(PullRequest pullRequest, CodeAnalysis if (!filePaths.isEmpty()) { Map fileContents = fetchFileContentsFromVcs(project, filePaths, commitHash); if (!fileContents.isEmpty()) { + requireConfirmedLease(lockLease); fileSnapshotService.persistSnapshotsForPr(pullRequest, cloned, fileContents, commitHash); } } + } catch (IOException ownershipFailure) { + throw ownershipFailure; } catch (Exception e) { log.warn("Failed to persist PR snapshots for cache hit (non-critical): {}", e.getMessage()); } @@ -538,9 +589,10 @@ protected boolean postDiffFingerprintCacheIfExist( Project project, PullRequest pullRequest, AiAnalysisRequest aiRequest, - VcsReportingService reportingService + VcsReportingService reportingService, + AnalysisLockService.LockLease lockLease - ) { + ) throws IOException { // Get analysis cache by diff fingerprint (any PR ID) - less ideal than commit hash but still a win if(diffFingerprint == null) { return false; @@ -554,14 +606,17 @@ protected boolean postDiffFingerprintCacheIfExist( "Diff fingerprint cache hit for project={}, fingerprint={} (source PR={}). Cloning for PR={}.", project.getId(), diffFingerprint.substring(0, 8) + "...", fingerprintHit.get().getPrNumber(), request.getPullRequestId()); + requireConfirmedLease(lockLease); CodeAnalysis cloned = codeAnalysisService.cloneAnalysisForPr( fingerprintHit.get(), project, request.getPullRequestId(), request.getCommitHash(), request.getTargetBranchName(), request.getSourceBranchName(), diffFingerprint); + requireConfirmedLease(lockLease); copyTaskImplementationEvidence(fingerprintHit.get(), cloned); // Persist PR-level snapshots for the source code viewer persistPrSnapshotsForCacheHit(pullRequest, cloned, fingerprintHit.get(), project, - request.getCommitHash(), aiRequest.getChangedFiles()); + request.getCommitHash(), aiRequest.getChangedFiles(), lockLease); + requireConfirmedLease(lockLease); try { reportingService.postAnalysisResults(cloned, project, request.getPullRequestId(), pullRequest.getId(), @@ -584,8 +639,9 @@ protected CacheHitType postAnalysisCacheIfExist( String placeholderCommentId, String targetBranch, String sourceBranch, - String expectedReviewIdentity - ) { + String expectedReviewIdentity, + AnalysisLockService.LockLease lockLease + ) throws IOException { Optional cachedAnalysis = codeAnalysisService.getCodeAnalysisCache( project.getId(), commitHash, @@ -595,6 +651,7 @@ protected CacheHitType postAnalysisCacheIfExist( if (cachedAnalysis.isPresent() && expectedReviewIdentity != null && expectedReviewIdentity.equals(cachedAnalysis.get().getDiffFingerprint())) { + requireConfirmedLease(lockLease); try { reportingService.postAnalysisResults(cachedAnalysis.get(), project, @@ -617,14 +674,17 @@ protected CacheHitType postAnalysisCacheIfExist( project.getId(), commitHash, commitHashHit.get().getPrNumber(), prId ); + requireConfirmedLease(lockLease); CodeAnalysis cloned = codeAnalysisService.cloneAnalysisForPr( commitHashHit.get(), project, prId, commitHash, targetBranch, sourceBranch, commitHashHit.get().getDiffFingerprint()); + requireConfirmedLease(lockLease); copyTaskImplementationEvidence(commitHashHit.get(), cloned); // Persist PR-level snapshots for the source code viewer persistPrSnapshotsForCacheHit(pullRequest, cloned, commitHashHit.get(), project, - commitHash, null); + commitHash, null, lockLease); + requireConfirmedLease(lockLease); try { reportingService.postAnalysisResults( cloned, @@ -907,7 +967,7 @@ private void ensureRagIndexForTargetBranch(Project project, String targetBranch, boolean ready = ragOperationsService.ensureRagIndexUpToDate( project, targetBranch, - consumer::accept); + event -> emitEvent(consumer, event)); if (ready) { log.info("RAG index ensured up-to-date for PR target branch: project={}, branch={}", project.getId(), targetBranch); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java index c18a613e..3af6d182 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.analysisengine.service; +import jakarta.annotation.PreDestroy; import org.rostilos.codecrow.core.model.analysis.AnalysisLock; import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; import org.rostilos.codecrow.core.model.project.Project; @@ -23,6 +24,14 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @Service @@ -33,6 +42,7 @@ public class AnalysisLockService { private final AnalysisLockRepository lockRepository; private final TransactionTemplate requiresNewTransactionTemplate; private final String instanceId; + private final ScheduledExecutorService leaseHeartbeatExecutor; @Autowired @Lazy @@ -50,13 +60,29 @@ public class AnalysisLockService { @Value("${analysis.lock.wait.retry.interval.seconds:5}") private int lockWaitRetryIntervalSeconds; - public AnalysisLockService(AnalysisLockRepository lockRepository, PlatformTransactionManager transactionManager) { + @Value("${analysis.lock.heartbeat.interval.seconds:60}") + private int lockHeartbeatIntervalSeconds = 60; + + public AnalysisLockService( + AnalysisLockRepository lockRepository, + PlatformTransactionManager transactionManager) { + this(lockRepository, transactionManager, 4); + } + + @Autowired + public AnalysisLockService( + AnalysisLockRepository lockRepository, + PlatformTransactionManager transactionManager, + @Value("${analysis.lock.heartbeat.threads:4}") int heartbeatThreads) { this.lockRepository = lockRepository; this.instanceId = UUID.randomUUID().toString(); // Create a TransactionTemplate with REQUIRES_NEW propagation for fully isolated lock inserts this.requiresNewTransactionTemplate = new TransactionTemplate(transactionManager); this.requiresNewTransactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + this.leaseHeartbeatExecutor = Executors.newScheduledThreadPool( + Math.max(2, heartbeatThreads), + new LeaseHeartbeatThreadFactory()); log.info("AnalysisLockService initialized with instance ID: {}", instanceId); } @@ -162,12 +188,10 @@ public Optional acquireLockWithWait(Project project, String branchName, if (branchName == null || branchName.isBlank()) { log.error("Cannot acquire lock with wait: branchName is required but was null or blank. " + "project={}, lockType={}, prNumber={}", project.getId(), lockType, prNumber); - if (messageConsumer != null) { - messageConsumer.accept(Map.of( + emitLockEvent(messageConsumer, Map.of( "type", "error", "message", "Cannot start analysis: branch information is missing. Please ensure the PR has valid source branch information." )); - } return Optional.empty(); } @@ -183,8 +207,7 @@ public Optional acquireLockWithWait(Project project, String branchName, if (lockKey.isPresent()) { if (attemptCount > 1) { - if (messageConsumer != null) { - Map lockAcquiredMessage = Map.of( + Map lockAcquiredMessage = Map.of( "type", "lock_acquired", "message", String.format("Lock acquired after %d attempts (waited %d seconds)", attemptCount, @@ -192,8 +215,7 @@ public Optional acquireLockWithWait(Project project, String branchName, "lockType", lockType.name(), "branchName", branchName, "attemptCount", attemptCount); - messageConsumer.accept(lockAcquiredMessage); - } + emitLockEvent(messageConsumer, lockAcquiredMessage); log.info("Lock acquired after {} attempts (waited {} seconds)", attemptCount, Duration.between(startTime, OffsetDateTime.now()).getSeconds()); @@ -208,28 +230,18 @@ public Optional acquireLockWithWait(Project project, String branchName, "Lock acquisition attempt {} failed for project={}, branch={}, type={}. Waiting {} seconds before retry...", attemptCount, project.getId(), branchName, lockType, lockWaitRetryIntervalSeconds); - if (messageConsumer != null) { - try { - // Use HashMap instead of Map.of() to allow null values - Map lockWaitMessage = new java.util.HashMap<>(); - lockWaitMessage.put("type", "lock_wait"); - lockWaitMessage.put("message", - String.format("Waiting for lock release... (attempt %d, waited %ds, timeout in %ds)", - attemptCount, waitedSeconds, remainingSeconds)); - lockWaitMessage.put("lockType", lockType.name()); - lockWaitMessage.put("branchName", branchName); - lockWaitMessage.put("attemptCount", attemptCount); - lockWaitMessage.put("waitedSeconds", waitedSeconds); - lockWaitMessage.put("remainingSeconds", remainingSeconds); - log.debug("Sending lock wait message to consumer: {}", lockWaitMessage); - messageConsumer.accept(lockWaitMessage); - log.debug("Lock wait message sent successfully"); - } catch (Exception e) { - log.warn("Failed to send lock wait message: {}", e.getMessage(), e); - } - } else { - log.warn("Message consumer is null, cannot send lock wait message"); - } + // Use HashMap instead of Map.of() to allow null values + Map lockWaitMessage = new java.util.HashMap<>(); + lockWaitMessage.put("type", "lock_wait"); + lockWaitMessage.put("message", + String.format("Waiting for lock release... (attempt %d, waited %ds, timeout in %ds)", + attemptCount, waitedSeconds, remainingSeconds)); + lockWaitMessage.put("lockType", lockType.name()); + lockWaitMessage.put("branchName", branchName); + lockWaitMessage.put("attemptCount", attemptCount); + lockWaitMessage.put("waitedSeconds", waitedSeconds); + lockWaitMessage.put("remainingSeconds", remainingSeconds); + emitLockEvent(messageConsumer, lockWaitMessage); if (OffsetDateTime.now().plusSeconds(lockWaitRetryIntervalSeconds).isAfter(timeout)) { break; @@ -247,18 +259,12 @@ public Optional acquireLockWithWait(Project project, String branchName, log.warn("Failed to acquire lock after {} attempts and {} seconds (timeout exceeded)", attemptCount, Duration.between(startTime, OffsetDateTime.now()).getSeconds()); - if (messageConsumer != null) { - try { - messageConsumer.accept(Map.of( + emitLockEvent(messageConsumer, Map.of( "type", "lock_timeout", "message", "Failed to acquire lock: timeout exceeded", "lockType", lockType.name(), "branchName", branchName, "totalAttempts", attemptCount)); - } catch (Exception e) { - log.warn("Failed to send lock timeout message: {}", e.getMessage()); - } - } return Optional.empty(); } @@ -297,25 +303,170 @@ public boolean extendLock(String lockKey, int additionalMinutes) { return true; } - @Transactional public boolean renewLock(String lockKey, int leaseMinutes) { - Optional lockOpt = lockRepository.findByLockKey(lockKey); - if (lockOpt.isEmpty()) { - log.warn("Cannot renew lock - not found: {}", lockKey); + if (lockKey == null || lockKey.isBlank()) { return false; } - AnalysisLock lock = lockOpt.get(); - if (lock.isExpired()) { - log.warn("Cannot renew expired lock: {}", lockKey); - return false; + int boundedLeaseMinutes = Math.max(1, leaseMinutes); + OffsetDateTime now = OffsetDateTime.now(); + OffsetDateTime expiresAt = now.plusMinutes(boundedLeaseMinutes); + Integer updated = requiresNewTransactionTemplate.execute(status -> + lockRepository.renewActiveLock(lockKey, now, expiresAt)); + boolean renewed = updated != null && updated > 0; + if (renewed) { + log.debug("Renewed lock {} for {} minutes", lockKey, boundedLeaseMinutes); + } else { + // The lease owner records the single contextual ownership-loss + // diagnostic. Keep this primitive reusable without double logging. + log.debug("Cannot renew missing or expired lock: {}", lockKey); } + return renewed; + } + /** + * Maintains a long-running analysis lease independently of worker progress + * events. A quiet but healthy worker must not lose ownership merely because + * it has no streaming message to emit. + */ + public LockLease maintainLockLease(String lockKey, int leaseMinutes) { int boundedLeaseMinutes = Math.max(1, leaseMinutes); - lock.setExpiresAt(OffsetDateTime.now().plusMinutes(boundedLeaseMinutes)); - lockRepository.save(lock); - log.debug("Renewed lock {} for {} minutes", lockKey, boundedLeaseMinutes); - return true; + long leaseSeconds = TimeUnit.MINUTES.toSeconds(boundedLeaseMinutes); + long intervalSeconds = Math.max(1L, Math.min( + Math.max(1, lockHeartbeatIntervalSeconds), + Math.max(1L, leaseSeconds / 3L))); + ActiveLockLease lease = new ActiveLockLease(lockKey, boundedLeaseMinutes); + lease.renew(); + lease.schedule(intervalSeconds); + return lease; + } + + public interface LockLease extends AutoCloseable { + boolean isOwnershipLost(); + + /** Performs a final atomic ownership proof before durable publication. */ + boolean confirmOwnership(); + + @Override + void close(); + } + + private final class ActiveLockLease implements LockLease { + private final String lockKey; + private final int leaseMinutes; + private final AtomicBoolean ownershipLost = new AtomicBoolean(false); + private final AtomicBoolean closed = new AtomicBoolean(false); + private final AtomicBoolean transientFailureReported = new AtomicBoolean(false); + private final AtomicReference knownExpiresAt; + private volatile ScheduledFuture heartbeat; + + private ActiveLockLease(String lockKey, int leaseMinutes) { + this.lockKey = lockKey; + this.leaseMinutes = leaseMinutes; + // A pre-acquired lock may already be old. Do not infer a fresh lease + // window until the first atomic renewal has actually succeeded. + this.knownExpiresAt = new AtomicReference<>(); + } + + private void schedule(long intervalSeconds) { + heartbeat = leaseHeartbeatExecutor.scheduleWithFixedDelay( + this::renew, intervalSeconds, intervalSeconds, TimeUnit.SECONDS); + } + + private boolean renew() { + if (closed.get() || ownershipLost.get()) { + return false; + } + OffsetDateTime renewalStartedAt = OffsetDateTime.now(); + try { + if (!renewLock(lockKey, leaseMinutes)) { + ownershipLost.set(true); + log.error("Analysis lock lease ownership was lost: {}", lockKey); + return false; + } + knownExpiresAt.set( + renewalStartedAt.plusMinutes(leaseMinutes).minusSeconds(1)); + if (transientFailureReported.compareAndSet(true, false)) { + log.info("Analysis lock heartbeat recovered: {}", lockKey); + } + return true; + } catch (RuntimeException renewalFailure) { + // A database outage is not evidence that ownership changed. The + // previous successful lease remains authoritative until its known + // expiry, and the scheduled heartbeat will retry meanwhile. + OffsetDateTime knownExpiry = knownExpiresAt.get(); + boolean previousLeaseStillActive = knownExpiry != null + && OffsetDateTime.now().isBefore(knownExpiry); + if (!previousLeaseStillActive) { + ownershipLost.set(true); + log.error("Analysis lock ownership could not be confirmed before its known expiry: {}", + lockKey, renewalFailure); + return false; + } + if (transientFailureReported.compareAndSet(false, true)) { + log.warn("Analysis lock renewal failed; the prior lease is still active " + + "and renewal will retry: {}", + lockKey, renewalFailure); + } else { + log.debug("Analysis lock heartbeat remains degraded within the active lease: {}", + lockKey); + } + return true; + } + } + + @Override + public boolean isOwnershipLost() { + return ownershipLost.get(); + } + + @Override + public boolean confirmOwnership() { + return renew(); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + ScheduledFuture scheduled = heartbeat; + if (scheduled != null) { + scheduled.cancel(false); + } + } + } + + @PreDestroy + void shutdownLeaseHeartbeatExecutor() { + leaseHeartbeatExecutor.shutdownNow(); + } + + private static final class LeaseHeartbeatThreadFactory implements ThreadFactory { + private final AtomicInteger threadNumber = new AtomicInteger(); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread( + runnable, + "analysis-lock-heartbeat-" + threadNumber.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + } + + private static void emitLockEvent( + Consumer> messageConsumer, + Map event) { + if (messageConsumer == null) { + return; + } + try { + messageConsumer.accept(event); + } catch (RuntimeException observerFailure) { + log.debug("Analysis lock observer rejected event type={}: {}", + event.get("type"), observerFailure.getMessage()); + } } @Transactional(readOnly = true) diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java index 69ed3606..07e64b7f 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java @@ -323,6 +323,8 @@ void shouldIncludeSourceAndTargetBranchNamesInQueuedRequestPayload() throws Exce void shouldBindExactTargetBranchGenerationToQueuedReview() throws Exception { var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag .RagBranchIndexGenerationRepository.class); + var branchRepository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexRepository.class); var generation = mock(org.rostilos.codecrow.core.model.rag .RagBranchIndexGeneration.class); when(generation.getCollectionName()).thenReturn("opaque-master-generation"); @@ -332,6 +334,10 @@ void shouldBindExactTargetBranchGenerationToQueuedReview() throws Exception { .thenReturn(List.of(generation)); org.springframework.test.util.ReflectionTestUtils.setField( client, "branchGenerationRepository", repository); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchIndexRepository", branchRepository); + when(branchRepository.markAccessedIfUnclaimed( + eq(1L), eq("main"), any())).thenReturn(1); AiAnalysisRequest exactRequest = new TestAiAnalysisRequest() { @Override public String getBaseCommitHash() { @@ -357,6 +363,81 @@ public String getBaseCommitHash() { assertThat(requestPayload) .containsEntry("ragCollectionTarget", "opaque-master-generation") .containsEntry("ragBaseGenerationManifestSha256", "master-manifest"); + verify(branchRepository).markAccessedIfUnclaimed( + eq(1L), eq("main"), any()); + } + + @Test + @DisplayName("should not bind a generation claimed by transient cleanup") + void shouldNotBindCleanupClaimedGeneration() throws Exception { + var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexGenerationRepository.class); + var branchRepository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexRepository.class); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchGenerationRepository", repository); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchIndexRepository", branchRepository); + when(branchRepository.markAccessedIfUnclaimed( + eq(1L), eq("main"), any())).thenReturn(0); + AiAnalysisRequest exactRequest = new TestAiAnalysisRequest() { + @Override + public String getBaseCommitHash() { + return "master-base"; + } + }; + when(queueService.rightPop(anyString(), anyLong())) + .thenReturn(objectMapper.writeValueAsString(Map.of( + "type", "final", + "result", Map.of( + "comment", "ok", + "issues", List.of())))); + + client.performAnalysis(exactRequest); + + verifyNoInteractions(repository); + } + + @Test + @DisplayName("should not look up or bind a RAG generation when RAG is disabled") + void shouldNotBindRagGenerationWhenDisabled() throws Exception { + var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexGenerationRepository.class); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchGenerationRepository", repository); + AiAnalysisRequest disabledRequest = new TestAiAnalysisRequest() { + @Override + public boolean getRagEnabled() { + return false; + } + + @Override + public String getBaseCommitHash() { + return "master-base"; + } + }; + Map finalEvent = Map.of( + "type", "final", + "result", Map.of("comment", "ok", "issues", List.of())); + when(queueService.rightPop(anyString(), anyLong())) + .thenReturn(objectMapper.writeValueAsString(finalEvent)); + + client.performAnalysis(disabledRequest); + + var payloadCaptor = org.mockito.ArgumentCaptor.forClass(String.class); + verify(queueService).leftPush(eq("codecrow:analysis:jobs"), payloadCaptor.capture()); + @SuppressWarnings("unchecked") + Map queued = objectMapper.readValue( + payloadCaptor.getValue(), Map.class); + @SuppressWarnings("unchecked") + Map requestPayload = + (Map) queued.get("request"); + assertThat(requestPayload) + .containsEntry("ragEnabled", false) + .doesNotContainKeys( + "ragCollectionTarget", + "ragBaseGenerationManifestSha256"); + verifyNoInteractions(repository); } @Test diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java index a187e6b8..e3923ed6 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java @@ -10,6 +10,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.rostilos.codecrow.queue.RedisQueueService; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; import java.io.IOException; import java.util.HashMap; @@ -36,6 +37,28 @@ class AiCommandClientTest { void setUp() { objectMapper = new ObjectMapper(); client = new AiCommandClient(queueService, objectMapper); + lenient().when(queueService.hasKey(AiCommandClient.CONSUMER_HEARTBEAT_KEY)) + .thenReturn(true); + } + + @Test + @DisplayName("should expose an unambiguous Spring injection constructor") + void shouldConstructThroughSpringContext() { + try (AnnotationConfigApplicationContext context = + new AnnotationConfigApplicationContext()) { + context.registerBean( + RedisQueueService.class, + () -> queueService); + context.registerBean( + ObjectMapper.class, + () -> objectMapper); + context.register(AiCommandClient.class); + + context.refresh(); + + assertThat(context.getBean(AiCommandClient.class)) + .isNotNull(); + } } private AiCommandClient.SummarizeRequest createSummarizeRequest() { @@ -84,6 +107,8 @@ void shouldSuccessfullySummarizePR() throws Exception { assertThat(result.diagramType()).isEqualTo("MERMAID"); verify(queueService).leftPush(eq("codecrow:queue:commands"), anyString()); + verify(queueService).removeFromList( + eq("codecrow:queue:commands"), anyString()); verify(queueService).setExpiry(anyString(), anyLong()); verify(queueService).deleteKey(anyString()); } @@ -126,6 +151,57 @@ void shouldDefaultNullSummarizeResultFields() throws Exception { } } + @Test + @DisplayName("should reject commands before enqueue when no consumer is alive") + void shouldRejectWhenConsumerIsUnavailable() { + when(queueService.hasKey(AiCommandClient.CONSUMER_HEARTBEAT_KEY)) + .thenReturn(false); + + assertThatThrownBy(() -> client.summarize(createSummarizeRequest(), null)) + .isInstanceOf(IOException.class) + .hasMessageContaining("no live command queue consumer"); + + verify(queueService, never()).leftPush(anyString(), anyString()); + } + + @Test + @DisplayName("should remove an unadmitted command when the consumer disappears") + void shouldRemovePendingPayloadWhenConsumerDisappears() { + AiCommandClient supervisedClient = new AiCommandClient( + queueService, objectMapper, 1_000L, 1_000L); + when(queueService.hasKey(AiCommandClient.CONSUMER_HEARTBEAT_KEY)) + .thenReturn(true, true, false); + when(queueService.rightPop(anyString(), anyLong())).thenReturn(null); + + assertThatThrownBy(() -> + supervisedClient.summarize(createSummarizeRequest(), null)) + .isInstanceOf(IOException.class) + .hasMessageContaining("became unavailable before the command job was admitted"); + + verify(queueService).removeFromList( + eq("codecrow:queue:commands"), anyString()); + } + + @Test + @DisplayName("should remove a command that exceeds bounded admission wait") + void shouldRemovePayloadWhenAdmissionTimesOut() { + AiCommandClient supervisedClient = new AiCommandClient( + queueService, objectMapper, 1_000L, 10L); + when(queueService.rightPop(anyString(), anyLong())) + .thenAnswer(invocation -> { + Thread.sleep(12L); + return null; + }); + + assertThatThrownBy(() -> + supervisedClient.summarize(createSummarizeRequest(), null)) + .isInstanceOf(IOException.class) + .hasMessageContaining("was not admitted by Inference Orchestrator"); + + verify(queueService).removeFromList( + eq("codecrow:queue:commands"), anyString()); + } + @Nested @DisplayName("ask()") class AskTests { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java index 5781c71f..8ba074a3 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java @@ -76,6 +76,9 @@ class BranchAnalysisProcessorTest { @Mock private AnalysisLockService analysisLockService; + @Mock + private AnalysisLockService.LockLease lockLease; + @Mock private BranchAnalysisGateService branchAnalysisGateService; @@ -144,6 +147,11 @@ class BranchAnalysisProcessorTest { @BeforeEach void setUp() { when(vcsClientProvider.getClient(vcsConnection)).thenReturn(authorizedClient); + lenient().when(analysisLockService.getLeaseMinutes(any())) + .thenReturn(30); + lenient().when(analysisLockService.maintainLockLease(anyString(), anyInt())) + .thenReturn(lockLease); + lenient().when(lockLease.confirmOwnership()).thenReturn(true); processor = new BranchAnalysisProcessor( projectService, branchRepository, @@ -349,6 +357,76 @@ void shouldSkipWhenCommitAlreadyAnalyzed() throws IOException { assertThat(result).containsEntry("status", "skipped"); assertThat(result).containsEntry("reason", "commit_already_analyzed"); + verify(analysisLockService).maintainLockLease("lock-key", 30); + verify(lockLease).close(); + verify(analysisLockService).releaseLock("lock-key"); + } + + @Test + @DisplayName("cleanup failures cannot override a successful branch outcome") + void cleanupFailuresCannotOverrideSuccessfulBranchOutcome() throws IOException { + BranchProcessRequest request = createRequest(); + when(projectService.getProjectWithConnections(1L)).thenReturn(project); + when(project.getId()).thenReturn(1L); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), any(), any())) + .thenReturn(Optional.of("lock-key")); + Branch existingBranch = new Branch(); + existingBranch.setLastSuccessfulCommitHash("abc123"); + when(branchRepository.findByProjectIdAndBranchName(1L, "main")) + .thenReturn(Optional.of(existingBranch)); + when(branchFileOperationsService.getBranchFilePaths(1L, "main")) + .thenReturn(Collections.emptySet()); + doThrow(new IllegalStateException("lease executor unavailable")) + .when(lockLease).close(); + doThrow(new IllegalStateException("lock database unavailable")) + .when(analysisLockService).releaseLock("lock-key"); + + Map result = processor.process(request, null); + + assertThat(result).containsEntry("status", "skipped"); + assertThat(result).containsEntry("reason", "commit_already_analyzed"); + verify(lockLease).close(); + verify(analysisLockService).releaseLock("lock-key"); + } + + @Test + @DisplayName("should fence cached snapshot publication when branch lease is lost") + void shouldFenceCachedSnapshotPublicationWhenLeaseIsLost() throws IOException { + BranchProcessRequest request = createRequest(); + + when(projectService.getProjectWithConnections(1L)).thenReturn(project); + when(project.getId()).thenReturn(1L); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), any(), any())) + .thenReturn(Optional.of("lock-key")); + when(lockLease.confirmOwnership()).thenReturn(true, false); + + Branch existingBranch = new Branch(); + existingBranch.setLastSuccessfulCommitHash("abc123"); + when(branchRepository.findByProjectIdAndBranchName(1L, "main")) + .thenReturn(Optional.of(existingBranch)); + + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(repoInfo.getRepoWorkspace()).thenReturn("ws"); + when(repoInfo.getRepoSlug()).thenReturn("repo"); + when(branchFileOperationsService.getBranchFilePaths(1L, "main")) + .thenReturn(new HashSet<>(Set.of("src/App.java"))); + when(branchFileOperationsService.downloadBranchFileSnapshot( + any(), eq("abc123"), anySet())) + .thenReturn(archiveSnapshot(Map.of("src/App.java", "content"))); + + assertThatThrownBy(() -> processor.process(request, null)) + .isInstanceOf(IOException.class) + .hasMessageContaining("lost its lock lease"); + + verify(branchFileOperationsService, never()).updateFileSnapshotsForBranch( + anySet(), any(), any(), + isA(BranchFileOperationsService.BranchFileSnapshot.class)); + verify(branchHealthService, never()).handleProcessFailure(any(), any(), anyList(), any()); + verify(lockLease).close(); verify(analysisLockService).releaseLock("lock-key"); } diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java index 13668ba2..8c3af505 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java @@ -68,6 +68,9 @@ class PullRequestAnalysisProcessorTest { @Mock private AnalysisLockService analysisLockService; + @Mock + private AnalysisLockService.LockLease lockLease; + @Mock private AnalyzedCommitService analyzedCommitService; @@ -121,6 +124,12 @@ void setUp() { .thenReturn(TaskImplementationEvidenceService.PersistenceResult.empty()); lenient().when(taskImplementationEvidenceService.copyForAnalysis(any(), any())) .thenReturn(TaskImplementationEvidenceService.PersistenceResult.empty()); + lenient().when(analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS)) + .thenReturn(30); + lenient().when(analysisLockService.maintainLockLease(anyString(), eq(30))) + .thenReturn(lockLease); + lenient().when(lockLease.isOwnershipLost()).thenReturn(false); + lenient().when(lockLease.confirmOwnership()).thenReturn(true); processor = new PullRequestAnalysisProcessor( pullRequestService, codeAnalysisService, @@ -253,6 +262,43 @@ void shouldPersistProviderCanonicalizedFullPrHead() throws Exception { verify(analysisLockService).releaseLock("lock-key-123"); } + @Test + @DisplayName("cleanup failures cannot override a successful PR outcome") + void cleanupFailuresCannotOverrideSuccessfulPrOutcome() throws Exception { + PrProcessRequest request = createRequest(); + PullRequestAnalysisProcessor.EventConsumer consumer = mock( + PullRequestAnalysisProcessor.EventConsumer.class); + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(project.getId()).thenReturn(1L); + when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenReturn(Optional.of("lock-key-123")); + when(vcsServiceFactory.getReportingService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(reportingService); + when(vcsServiceFactory.getAiClientService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(aiClientService); + when(codeAnalysisService.getAllPrAnalyses(anyLong(), anyLong())) + .thenReturn(List.of()); + when(pullRequestService.createOrUpdatePullRequest( + anyLong(), anyLong(), anyString(), anyString(), anyString(), any())) + .thenReturn(pullRequest); + when(aiClientService.buildAiAnalysisRequests(any(), any(), any(), anyList())) + .thenReturn(List.of()); + doThrow(new IllegalStateException("lease executor unavailable")) + .when(lockLease).close(); + doThrow(new IllegalStateException("lock database unavailable")) + .when(analysisLockService).releaseLock("lock-key-123"); + + Map result = processor.process(request, consumer, project); + + assertThat(result).containsEntry("status", "ignored"); + verify(lockLease).close(); + verify(analysisLockService).releaseLock("lock-key-123"); + } + @Test @DisplayName("should successfully process PR analysis") void shouldSuccessfullyProcessPRAnalysis() throws Exception { @@ -401,10 +447,6 @@ void shouldNotPersistOrPublishPromptDryRun() throws Exception { "filename", "capture.json", "containerPath", "/app/logs/prompt-dry-runs/capture.json")); - when(analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS)) - .thenReturn(30); - when(analysisLockService.renewLock("lock-key-123", 30)) - .thenReturn(true); when(aiAnalysisClient.performAnalysis(any(), any())) .thenAnswer(invocation -> { @SuppressWarnings("unchecked") @@ -426,13 +468,15 @@ void shouldNotPersistOrPublishPromptDryRun() throws Exception { verify(codeAnalysisService, never()).createAnalysisFromAiResponse( any(), any(), anyLong(), anyString(), anyString(), anyString(), any(), any(), any(), any(), any(), any()); - verify(analysisLockService).renewLock("lock-key-123", 30); + verify(analysisLockService).maintainLockLease("lock-key-123", 30); + verify(lockLease, times(2)).confirmOwnership(); + verify(lockLease).close(); verify(reportingService, never()).postAnalysisResults( any(), any(), anyLong(), any(), any()); } @Test - @DisplayName("should reject a completed review after lock lease ownership is lost") + @DisplayName("should reject a quiet completed review when the independent lease heartbeat lost ownership") void shouldRejectCompletedReviewAfterLockLeaseIsLost() throws Exception { PrProcessRequest request = createRequest(); PullRequestAnalysisProcessor.EventConsumer consumer = mock( @@ -458,22 +502,10 @@ void shouldRejectCompletedReviewAfterLockLeaseIsLost() throws Exception { .thenReturn(List.of(aiAnalysisRequest)); when(aiAnalysisRequest.getRawDiff()).thenReturn("diff"); when(aiAnalysisRequest.getChangedFiles()).thenReturn(List.of("file.java")); - when(analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS)) - .thenReturn(30); - when(analysisLockService.renewLock("lock-key-123", 30)) - .thenReturn(false); - when(aiAnalysisClient.performAnalysis(any(), any())) - .thenAnswer(invocation -> { - @SuppressWarnings("unchecked") - java.util.function.Consumer> eventHandler = - invocation.getArgument(1); - eventHandler.accept(Map.of( - "type", "status", - "state", "processing")); - return Map.of( - "comment", "must not be published", - "issues", List.of()); - }); + when(lockLease.confirmOwnership()).thenReturn(true, false); + when(aiAnalysisClient.performAnalysis(any(), any())).thenReturn(Map.of( + "comment", "must not be published", + "issues", List.of())); Map result = processor.process(request, consumer, project); @@ -487,9 +519,275 @@ void shouldRejectCompletedReviewAfterLockLeaseIsLost() throws Exception { any(), any(), any(), any(), any(), any()); verify(reportingService, never()).postAnalysisResults( any(), any(), anyLong(), any(), any()); + verify(analysisLockService).maintainLockLease("lock-key-123", 30); + verify(lockLease, times(2)).confirmOwnership(); + verify(lockLease).close(); verify(analysisLockService).releaseLock("lock-key-123"); } + @Test + @DisplayName("should re-confirm ownership after direct VCS fallback before persistence") + void shouldRejectLeaseLostDuringDirectVcsFallbackBeforePersistence() throws Exception { + PrProcessRequest request = createRequest(); + PullRequestAnalysisProcessor.EventConsumer consumer = mock( + PullRequestAnalysisProcessor.EventConsumer.class); + Map aiResponse = Map.of( + "comment", "must not be persisted", + "issues", List.of()); + stubReviewThroughAi(aiResponse); + when(lockLease.confirmOwnership()).thenReturn(true, true, false); + + Map result = processor.process(request, consumer, project); + + assertThat(result).containsEntry("status", "error"); + verify(lockLease, times(3)).confirmOwnership(); + verify(codeAnalysisService, never()).createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any()); + verify(reportingService, never()).postAnalysisResults( + any(), any(), anyLong(), any(), any()); + } + + @Test + @DisplayName("should atomically re-confirm ownership before VCS publication") + void shouldRejectLeaseLostBeforeVcsPublication() throws Exception { + PrProcessRequest request = createRequest(); + PullRequestAnalysisProcessor.EventConsumer consumer = mock( + PullRequestAnalysisProcessor.EventConsumer.class); + Map aiResponse = Map.of( + "comment", "must not be published", + "issues", List.of()); + stubReviewThroughAi(aiResponse); + when(lockLease.confirmOwnership()).thenReturn(true, true, true, false); + when(codeAnalysisService.createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any())) + .thenReturn(codeAnalysis); + + Map result = processor.process(request, consumer, project); + + assertThat(result).containsEntry("status", "error"); + verify(lockLease, times(4)).confirmOwnership(); + verify(codeAnalysisService).createAnalysisFromAiResponse( + any(), eq(aiResponse), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any()); + verify(reportingService, never()).postAnalysisResults( + any(), any(), anyLong(), any(), any()); + } + + private void stubReviewThroughAi(Map aiResponse) throws Exception { + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(project.getId()).thenReturn(1L); + when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenReturn(Optional.of("lock-key-123")); + when(pullRequestService.createOrUpdatePullRequest( + anyLong(), anyLong(), anyString(), anyString(), anyString(), any())) + .thenReturn(pullRequest); + when(vcsServiceFactory.getReportingService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(reportingService); + when(vcsServiceFactory.getAiClientService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(aiClientService); + when(codeAnalysisService.getAllPrAnalyses(anyLong(), anyLong())) + .thenReturn(List.of()); + when(aiClientService.buildAiAnalysisRequests(any(), any(), any(), anyList())) + .thenReturn(List.of(aiAnalysisRequest)); + when(aiAnalysisRequest.getRawDiff()).thenReturn("diff"); + when(aiAnalysisRequest.getChangedFiles()).thenReturn(List.of("file.java")); + when(aiAnalysisClient.performAnalysis(any(), any())).thenReturn(aiResponse); + } + + private String stubCacheLookupPrerequisites() throws Exception { + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(project.getId()).thenReturn(1L); + when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenReturn(Optional.of("lock-key-cache")); + when(vcsServiceFactory.getReportingService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(reportingService); + when(vcsServiceFactory.getAiClientService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(aiClientService); + when(codeAnalysisService.getAllPrAnalyses(anyLong(), anyLong())) + .thenReturn(List.of()); + when(aiClientService.buildAiAnalysisRequests(any(), any(), any(), anyList())) + .thenReturn(List.of(aiAnalysisRequest)); + when(pullRequestService.createOrUpdatePullRequest( + anyLong(), anyLong(), anyString(), anyString(), anyString(), any())) + .thenReturn(pullRequest); + when(aiAnalysisRequest.getRawDiff()).thenReturn( + "diff --git a/file.java b/file.java\n@@ -1 +1 @@\n-old\n+new\n"); + when(aiAnalysisRequest.getChangedFiles()).thenReturn(List.of("file.java")); + return processor.computeReviewIdentity(aiAnalysisRequest); + } + + @Test + @DisplayName("should fence the first durable PR write after request construction") + void shouldRejectLeaseLostBeforePullRequestPersistence() throws Exception { + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(project.getId()).thenReturn(1L); + when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenReturn(Optional.of("lock-key-before-pr")); + when(vcsServiceFactory.getReportingService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(reportingService); + when(vcsServiceFactory.getAiClientService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(aiClientService); + when(codeAnalysisService.getAllPrAnalyses(anyLong(), anyLong())) + .thenReturn(List.of()); + when(aiClientService.buildAiAnalysisRequests(any(), any(), any(), anyList())) + .thenReturn(List.of(aiAnalysisRequest)); + when(lockLease.confirmOwnership()).thenReturn(false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(pullRequestService, never()).createOrUpdatePullRequest( + anyLong(), anyLong(), anyString(), anyString(), anyString(), any()); + } + + @Test + @DisplayName("should fence exact cache publication") + void shouldRejectLeaseLostBeforeExactCachePublication() throws Exception { + String identity = stubCacheLookupPrerequisites(); + when(codeAnalysisService.getCodeAnalysisCache(1L, "abc123", 42L)) + .thenReturn(Optional.of(codeAnalysis)); + when(codeAnalysis.getDiffFingerprint()).thenReturn(identity); + when(lockLease.confirmOwnership()).thenReturn(true, false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(reportingService, never()).postAnalysisResults( + any(), any(), anyLong(), any(), any()); + } + + @Test + @DisplayName("should fence commit cache cloning") + void shouldRejectLeaseLostBeforeCommitCacheClone() throws Exception { + String identity = stubCacheLookupPrerequisites(); + when(codeAnalysisService.getCodeAnalysisCache(1L, "abc123", 42L)) + .thenReturn(Optional.empty()); + CodeAnalysis source = mock(CodeAnalysis.class); + when(source.getDiffFingerprint()).thenReturn(identity); + when(source.getPrNumber()).thenReturn(99L); + when(codeAnalysisService.getAnalysisByCommitHash(1L, "abc123")) + .thenReturn(Optional.of(source)); + when(lockLease.confirmOwnership()).thenReturn(true, false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(codeAnalysisService, never()).cloneAnalysisForPr( + any(), any(), anyLong(), anyString(), anyString(), anyString(), anyString()); + } + + @Test + @DisplayName("should fence fingerprint cache cloning") + void shouldRejectLeaseLostBeforeFingerprintCacheClone() throws Exception { + stubCacheLookupPrerequisites(); + when(codeAnalysisService.getCodeAnalysisCache(1L, "abc123", 42L)) + .thenReturn(Optional.empty()); + when(codeAnalysisService.getAnalysisByCommitHash(1L, "abc123")) + .thenReturn(Optional.empty()); + when(codeAnalysisService.getAnalysisByDiffFingerprint(eq(1L), anyString())) + .thenReturn(Optional.of(mock(CodeAnalysis.class))); + when(lockLease.confirmOwnership()).thenReturn(true, false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(codeAnalysisService, never()).cloneAnalysisForPr( + any(), any(), anyLong(), anyString(), anyString(), anyString(), anyString()); + } + + @Test + @DisplayName("should re-confirm after VCS publication before commit receipts") + void shouldRejectLeaseLostAfterVcsPublicationBeforeCommitReceipts() throws Exception { + Map aiResponse = Map.of("comment", "review", "issues", List.of()); + stubReviewThroughAi(aiResponse); + when(codeAnalysisService.createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any())) + .thenReturn(codeAnalysis); + when(lockLease.confirmOwnership()).thenReturn(true, true, true, true, false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(reportingService).postAnalysisResults( + eq(codeAnalysis), any(), anyLong(), any(), any()); + verifyNoInteractions(analyzedCommitService); + } + + @Test + @DisplayName("observer failure cannot turn a successful review into failure") + void observerFailureDoesNotChangeReviewOutcome() throws Exception { + Map aiResponse = Map.of("comment", "review", "issues", List.of()); + stubReviewThroughAi(aiResponse); + when(codeAnalysisService.createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any())) + .thenReturn(codeAnalysis); + PullRequestAnalysisProcessor.EventConsumer observer = mock( + PullRequestAnalysisProcessor.EventConsumer.class); + doThrow(new IllegalStateException("observer disconnected")) + .when(observer).accept(anyMap()); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + java.util.function.Consumer> progress = + invocation.getArgument(5); + progress.accept(Map.of( + "type", "lock_wait", + "message", "waiting")); + return Optional.of("lock-key-123"); + }); + when(ragOperationsService.ensureRagIndexUpToDate( + any(), anyString(), any())) + .thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + java.util.function.Consumer> progress = + invocation.getArgument(2); + progress.accept(Map.of( + "type", "status", + "state", "rag_update")); + return true; + }); + when(aiAnalysisClient.performAnalysis(any(), any())).thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + java.util.function.Consumer> progress = + invocation.getArgument(1); + progress.accept(Map.of( + "type", "status", + "state", "processing")); + return aiResponse; + }); + + Map result = processor.process(createRequest(), observer, project); + + assertThat(result).isEqualTo(aiResponse); + verify(reportingService).postAnalysisResults( + eq(codeAnalysis), any(), anyLong(), any(), any()); + verify(observer).accept(argThat(event -> "lock_wait".equals(event.get("type")))); + verify(observer).accept(argThat(event -> "rag_update".equals(event.get("state")))); + verify(observer).accept(argThat(event -> "processing".equals(event.get("state")))); + } + @Test @DisplayName("should throw AnalysisLockedException when lock cannot be acquired") void shouldThrowAnalysisLockedExceptionWhenLockCannotBeAcquired() { @@ -884,7 +1182,7 @@ void shouldReturnTrueAndPostWhenCacheExists() throws IOException { PullRequestAnalysisProcessor.CacheHitType result = processor.postAnalysisCacheIfExist( project, pullRequest, "abc123", 42L, reportingService, "placeholder-id", - "main", "feature-branch", "identity"); + "main", "feature-branch", "identity", lockLease); assertThat(result).isEqualTo(PullRequestAnalysisProcessor.CacheHitType.EXACT); verify(reportingService).postAnalysisResults(eq(codeAnalysis), eq(project), eq(42L), eq(100L), @@ -902,7 +1200,7 @@ void shouldReturnFalseWhenNoCacheExists() throws IOException { PullRequestAnalysisProcessor.CacheHitType result = processor.postAnalysisCacheIfExist( project, pullRequest, "abc123", 42L, reportingService, "placeholder-id", - "main", "feature-branch", "identity"); + "main", "feature-branch", "identity", lockLease); assertThat(result).isEqualTo(PullRequestAnalysisProcessor.CacheHitType.NONE); verify(reportingService, never()).postAnalysisResults(any(), any(), anyLong(), any(), any()); @@ -923,7 +1221,7 @@ void shouldRejectCacheEntriesWithDifferentReviewIdentity() throws IOException { PullRequestAnalysisProcessor.CacheHitType result = processor.postAnalysisCacheIfExist( project, pullRequest, "abc123", 42L, reportingService, "placeholder-id", - "main", "feature-branch", "current-identity"); + "main", "feature-branch", "current-identity", lockLease); assertThat(result).isEqualTo(PullRequestAnalysisProcessor.CacheHitType.NONE); verify(reportingService, never()).postAnalysisResults(any(), any(), anyLong(), any(), any()); @@ -944,7 +1242,7 @@ void shouldReturnTrueEvenWhenPostingFails() throws IOException { PullRequestAnalysisProcessor.CacheHitType result = processor.postAnalysisCacheIfExist( project, pullRequest, "abc123", 42L, reportingService, "placeholder-id", - "main", "feature-branch", "identity"); + "main", "feature-branch", "identity", lockLease); // Should still return EXACT (cache existed) assertThat(result).isEqualTo(PullRequestAnalysisProcessor.CacheHitType.EXACT); diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockServiceTest.java index d4439cba..45aec1bd 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockServiceTest.java @@ -1,6 +1,11 @@ package org.rostilos.codecrow.analysisengine.service; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -13,11 +18,14 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.support.TransactionTemplate; +import org.slf4j.LoggerFactory; import java.lang.reflect.Field; import java.time.OffsetDateTime; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -50,6 +58,11 @@ void setUp() throws Exception { setField(lockService, "ragLockTimeoutMinutes", 360); } + @AfterEach + void tearDown() { + lockService.shutdownLeaseHeartbeatExecutor(); + } + private void setId(Object obj, Long id) throws Exception { Field field = obj.getClass().getDeclaredField("id"); field.setAccessible(true); @@ -354,33 +367,151 @@ void testExtendLock_ExpiredLock_ReturnsFalse() { @Test void testRenewLock_UsesLeaseFromCurrentTime() { String lockKey = "lock-1-main-RAG_INDEXING"; - AnalysisLock lock = mock(AnalysisLock.class); - when(lock.isExpired()).thenReturn(false); - when(lockRepository.findByLockKey(lockKey)).thenReturn(Optional.of(lock)); + when(lockRepository.renewActiveLock(eq(lockKey), any(), any())).thenReturn(1); OffsetDateTime before = OffsetDateTime.now().plusMinutes(29); boolean result = lockService.renewLock(lockKey, 30); assertThat(result).isTrue(); + ArgumentCaptor now = ArgumentCaptor.forClass(OffsetDateTime.class); ArgumentCaptor expiry = ArgumentCaptor.forClass(OffsetDateTime.class); - verify(lock).setExpiresAt(expiry.capture()); + verify(lockRepository).renewActiveLock(eq(lockKey), now.capture(), expiry.capture()); assertThat(expiry.getValue()).isAfter(before); assertThat(expiry.getValue()).isBefore(OffsetDateTime.now().plusMinutes(31)); - verify(lockRepository).save(lock); + assertThat(expiry.getValue()).isAfter(now.getValue()); } @Test void testRenewLock_RefusesMissingOrExpiredOwnership() { - AnalysisLock expiredLock = mock(AnalysisLock.class); - when(expiredLock.isExpired()).thenReturn(true); - when(lockRepository.findByLockKey("missing")).thenReturn(Optional.empty()); - when(lockRepository.findByLockKey("expired")).thenReturn(Optional.of(expiredLock)); + when(lockRepository.renewActiveLock(eq("missing"), any(), any())).thenReturn(0); + when(lockRepository.renewActiveLock(eq("expired"), any(), any())).thenReturn(0); assertThat(lockService.renewLock("missing", 30)).isFalse(); assertThat(lockService.renewLock("expired", 30)).isFalse(); verify(lockRepository, never()).save(any()); } + @Test + void lockLeaseHeartbeatRenewsWithoutWorkerProgressAndStopsWhenClosed() throws Exception { + setField(lockService, "lockHeartbeatIntervalSeconds", 1); + AtomicInteger renewals = new AtomicInteger(); + when(lockRepository.renewActiveLock(eq("quiet-review"), any(), any())) + .thenAnswer(invocation -> { + renewals.incrementAndGet(); + return 1; + }); + + AnalysisLockService.LockLease lease = lockService.maintainLockLease("quiet-review", 1); + + verify(lockRepository, timeout(2500).atLeast(2)) + .renewActiveLock(eq("quiet-review"), any(), any()); + assertThat(lease.isOwnershipLost()).isFalse(); + lease.close(); + int afterClose = renewals.get(); + Thread.sleep(1200); + assertThat(renewals).hasValue(afterClose); + } + + @Test + void blockedHeartbeatDoesNotStarveOtherActiveLeases() throws Exception { + setField(lockService, "lockHeartbeatIntervalSeconds", 1); + AtomicInteger blockedRenewals = new AtomicInteger(); + CountDownLatch blockedHeartbeatEntered = new CountDownLatch(1); + CountDownLatch releaseBlockedHeartbeat = new CountDownLatch(1); + when(lockRepository.renewActiveLock(anyString(), any(), any())) + .thenAnswer(invocation -> { + String key = invocation.getArgument(0); + if ("blocked-review".equals(key) + && blockedRenewals.incrementAndGet() > 1) { + blockedHeartbeatEntered.countDown(); + releaseBlockedHeartbeat.await(5, TimeUnit.SECONDS); + } + return 1; + }); + + AnalysisLockService.LockLease blocked = + lockService.maintainLockLease("blocked-review", 1); + AnalysisLockService.LockLease independent = + lockService.maintainLockLease("independent-review", 1); + try { + assertThat(blockedHeartbeatEntered.await(2500, TimeUnit.MILLISECONDS)).isTrue(); + verify(lockRepository, timeout(1500).atLeast(2)) + .renewActiveLock(eq("independent-review"), any(), any()); + } finally { + releaseBlockedHeartbeat.countDown(); + blocked.close(); + independent.close(); + } + } + + @Test + void transientRenewalExceptionDoesNotProveOwnershipLossAfterSuccessfulRenewal() { + when(lockRepository.renewActiveLock(eq("db-blip"), any(), any())) + .thenReturn(1) + .thenThrow(new RuntimeException("temporary database timeout")); + + AnalysisLockService.LockLease lease = lockService.maintainLockLease("db-blip", 30); + + assertThat(lease.isOwnershipLost()).isFalse(); + assertThat(lease.confirmOwnership()).isTrue(); + lease.close(); + } + + @Test + void repeatedTransientRenewalFailuresWarnOnceUntilRecovery() { + when(lockRepository.renewActiveLock(eq("db-outage"), any(), any())) + .thenReturn(1) + .thenThrow(new RuntimeException("database unavailable")) + .thenThrow(new RuntimeException("database unavailable")) + .thenReturn(1); + Logger logger = (Logger) LoggerFactory.getLogger(AnalysisLockService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + try (AnalysisLockService.LockLease lease = + lockService.maintainLockLease("db-outage", 30)) { + assertThat(lease.confirmOwnership()).isTrue(); + assertThat(lease.confirmOwnership()).isTrue(); + assertThat(lease.confirmOwnership()).isTrue(); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage)) + .containsExactly("Analysis lock renewal failed; the prior lease is still active " + + "and renewal will retry: db-outage"); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.INFO) + .map(ILoggingEvent::getFormattedMessage)) + .contains("Analysis lock heartbeat recovered: db-outage"); + } + + @Test + void initialRenewalExceptionCannotAssumeAFullLeaseForPreAcquiredLock() { + when(lockRepository.renewActiveLock(eq("unknown-age"), any(), any())) + .thenThrow(new RuntimeException("database unavailable")); + + AnalysisLockService.LockLease lease = lockService.maintainLockLease("unknown-age", 30); + + assertThat(lease.isOwnershipLost()).isTrue(); + assertThat(lease.confirmOwnership()).isFalse(); + lease.close(); + } + + @Test + void atomicRenewalMissProvesLeaseOwnershipWasLost() { + when(lockRepository.renewActiveLock(eq("replaced"), any(), any())).thenReturn(0); + + AnalysisLockService.LockLease lease = lockService.maintainLockLease("replaced", 30); + + assertThat(lease.isOwnershipLost()).isTrue(); + assertThat(lease.confirmOwnership()).isFalse(); + lease.close(); + } + @Test void testIsLocked_ReturnsTrue_WhenActiveLockExists() { Long projectId = 1L; diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java index 89c5b5c9..f0479472 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java @@ -69,6 +69,12 @@ public class RagBranchIndex { @Column(name = "error_message", columnDefinition = "TEXT") private String errorMessage; + @Column(name = "cleanup_claim_token", length = 64) + private String cleanupClaimToken; + + @Column(name = "cleanup_claimed_at") + private OffsetDateTime cleanupClaimedAt; + /** * Files that were deleted in this branch (for query-time filtering). * These files should be excluded when querying the branch's context. @@ -216,6 +222,22 @@ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } + public String getCleanupClaimToken() { + return cleanupClaimToken; + } + + public void setCleanupClaimToken(String cleanupClaimToken) { + this.cleanupClaimToken = cleanupClaimToken; + } + + public OffsetDateTime getCleanupClaimedAt() { + return cleanupClaimedAt; + } + + public void setCleanupClaimedAt(OffsetDateTime cleanupClaimedAt) { + this.cleanupClaimedAt = cleanupClaimedAt; + } + public Set getDeletedFiles() { return deletedFiles; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java index 8a786830..f3e49ae3 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java @@ -36,6 +36,19 @@ Optional findByProjectIdAndBranchNameAndAnalysisType( @Query("DELETE FROM AnalysisLock l WHERE l.lockKey = :lockKey") int deleteByLockKey(@Param("lockKey") String lockKey); + /** + * Atomically renews an owned, unexpired lease. A read/check/save sequence can + * race the scheduled expired-lock delete at the lease boundary; the guarded + * update makes the winner explicit to the caller. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query("UPDATE AnalysisLock l SET l.expiresAt = :expiresAt " + + "WHERE l.lockKey = :lockKey AND l.expiresAt >= :now") + int renewActiveLock( + @Param("lockKey") String lockKey, + @Param("now") OffsetDateTime now, + @Param("expiresAt") OffsetDateTime expiresAt); + @Query("SELECT CASE WHEN COUNT(l) > 0 THEN true ELSE false END FROM AnalysisLock l " + "WHERE l.project.id = :projectId AND l.branchName = :branchName " + "AND l.analysisType = :analysisType AND l.expiresAt > :now") diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java index 294f88da..3be5fb73 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java @@ -5,6 +5,7 @@ import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @@ -41,5 +42,24 @@ List findByWorkspaceAndStatus(@Param("workspace") String workspa boolean isProjectIndexed(@Param("projectId") Long projectId); + /** + * Restore the last usable checkpoint after a legacy incremental producer + * expires, but only while that exact job still owns the active status. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query("UPDATE RagIndexStatus r SET " + + "r.status = org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.INDEXED, " + + "r.errorMessage = CONCAT('Incremental update failed: ', :errorMessage), " + + "r.failedIncrementalCount = COALESCE(r.failedIncrementalCount, 0) + 1, " + + "r.activeJobId = NULL " + + "WHERE r.project.id = :projectId AND r.activeJobId = :jobId " + + "AND r.status IN (" + + "org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.INDEXING, " + + "org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.UPDATING)") + int recoverAbandonedIncrementalUpdate( + @Param("projectId") Long projectId, + @Param("jobId") Long jobId, + @Param("errorMessage") String errorMessage); + void deleteByProjectId(Long projectId); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java index f96cc670..e51badda 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java @@ -13,6 +13,16 @@ @Repository public interface BranchRepository extends JpaRepository { + interface StaleRetryCandidate { + Long getBranchId(); + Long getProjectId(); + String getBranchName(); + String getCommitHash(); + int getConsecutiveFailures(); + java.time.OffsetDateTime getLastHealthCheckAt(); + boolean getBranchAnalysisEnabled(); + } + Optional findByProjectIdAndBranchName(Long projectId, String branchName); Optional findByProjectIdAndCommitHash(Long projectId, String commitHash); @@ -32,4 +42,22 @@ public interface BranchRepository extends JpaRepository { */ @Query("SELECT b FROM Branch b JOIN FETCH b.project WHERE b.healthStatus = :status") List findByHealthStatusWithProject(@Param("status") BranchHealthStatus status); + + /** + * Materializes only scheduler inputs so the repository's short read + * transaction is closed before a retry performs VCS, RAG, or AI work. + */ + @Query(""" + SELECT b.id AS branchId, + b.project.id AS projectId, + b.branchName AS branchName, + b.commitHash AS commitHash, + b.consecutiveFailures AS consecutiveFailures, + b.lastHealthCheckAt AS lastHealthCheckAt, + b.project.branchAnalysisEnabled AS branchAnalysisEnabled + FROM Branch b + WHERE b.healthStatus = :status + """) + List findStaleRetryCandidates( + @Param("status") BranchHealthStatus status); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobLogRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobLogRepository.java index e17eeb05..8e1a91cb 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobLogRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobLogRepository.java @@ -44,6 +44,13 @@ List findByJobIdAndStep( @Param("step") String step ); + @Query("SELECT CASE WHEN COUNT(l) > 0 THEN true ELSE false END " + + "FROM JobLog l WHERE l.job.id = :jobId AND l.step = :step") + boolean existsByJobIdAndStep( + @Param("jobId") Long jobId, + @Param("step") String step + ); + @Query("SELECT COALESCE(MAX(l.sequenceNumber), 0) + 1 FROM JobLog l WHERE l.job.id = :jobId") Long getNextSequenceNumber(@Param("jobId") Long jobId); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java index 28570720..87f3a6ba 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java @@ -1,11 +1,13 @@ package org.rostilos.codecrow.core.persistence.repository.job; +import jakarta.persistence.LockModeType; import org.rostilos.codecrow.core.model.job.Job; import org.rostilos.codecrow.core.model.job.JobStatus; import org.rostilos.codecrow.core.model.job.JobType; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -16,8 +18,25 @@ public interface JobRepository extends JpaRepository { + /** + * Scalar identity for a legacy incremental RAG job whose producer may + * have disappeared. Keeping recovery coordinates scalar avoids carrying + * detached project proxies out of the repository transaction. + */ + interface LegacyRagJobRecoveryCoordinates { + Long getJobId(); + Long getProjectId(); + String getBranchName(); + String getCommitHash(); + String getErrorMessage(); + } + Optional findByExternalId(String externalId); + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT j FROM Job j WHERE j.id = :jobId") + Optional findByIdForUpdate(@Param("jobId") Long jobId); + @Query("SELECT j FROM Job j WHERE j.project.id = :projectId ORDER BY j.createdAt DESC") Page findByProjectId(@Param("projectId") Long projectId, Pageable pageable); @@ -190,6 +209,8 @@ boolean existsNewerBranchAnalysisJob( @Query("SELECT j FROM Job j WHERE " + "j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED) " + "AND j.updatedAt < :threshold ORDER BY j.createdAt ASC") @@ -199,6 +220,8 @@ List findRecoverableWebhookJobs( @Query("SELECT j FROM Job j WHERE " + "j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING " + "AND j.updatedAt < :threshold ORDER BY j.updatedAt ASC") List findAbandonedRunningWebhookJobs( @@ -208,6 +231,9 @@ List findAbandonedRunningWebhookJobs( @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + "j.updatedAt = :claimedAt WHERE j.id = :jobId " + + "AND j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED) " + "AND j.updatedAt < :threshold") @@ -220,9 +246,129 @@ int claimRecoverableWebhookJob( @Query("UPDATE Job j SET j.updatedAt = :activityAt WHERE j.id = :jobId") int touchJob(@Param("jobId") Long jobId, @Param("activityAt") OffsetDateTime activityAt); + /** + * Atomically renew a live legacy incremental RAG producer. Exact-generation + * jobs are owned by {@code RagIndexOperation} and must never share this + * recovery path. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query(value = """ + UPDATE job j + SET updated_at = :renewedAt + WHERE j.id = :jobId + AND j.job_type = 'RAG_INCREMENTAL_INDEX' + AND j.status = 'RUNNING' + AND j.updated_at >= :validAfter + AND NOT EXISTS ( + SELECT 1 + FROM rag_index_operation o + WHERE o.job_id = j.id + ) + """, nativeQuery = true) + int renewLegacyRagJobLease( + @Param("jobId") Long jobId, + @Param("validAfter") OffsetDateTime validAfter, + @Param("renewedAt") OffsetDateTime renewedAt); + + /** + * Find legacy incremental RAG jobs with no durable producer activity. The + * guarded update below remains the authority; this projection only bounds + * and prioritizes recovery work. + */ + @Query("SELECT j.id AS jobId, j.project.id AS projectId, " + + "j.branchName AS branchName, j.commitHash AS commitHash, " + + "j.errorMessage AS errorMessage " + + "FROM Job j WHERE " + + "j.jobType = org.rostilos.codecrow.core.model.job.JobType.RAG_INCREMENTAL_INDEX " + + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + + "org.rostilos.codecrow.core.model.job.JobStatus.RUNNING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.WAITING) " + + "AND j.updatedAt < :threshold " + + "AND NOT EXISTS (SELECT o.id FROM RagIndexOperation o WHERE o.jobId = j.id) " + + "ORDER BY j.updatedAt ASC") + List findAbandonedLegacyRagJobs( + @Param("threshold") OffsetDateTime threshold, + Pageable pageable); + + /** + * Win recovery only if neither a heartbeat nor a terminal transition moved + * the row after selection. This CAS is the fencing point between a live + * producer and the recovery scheduler. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query(value = """ + UPDATE job j + SET status = 'FAILED', + completed_at = :failedAt, + error_message = :diagnostic, + updated_at = :failedAt + WHERE j.id = :jobId + AND j.job_type = 'RAG_INCREMENTAL_INDEX' + AND j.status IN ('PENDING', 'QUEUED', 'RUNNING', 'WAITING') + AND j.updated_at < :threshold + AND NOT EXISTS ( + SELECT 1 + FROM rag_index_operation o + WHERE o.job_id = j.id + ) + """, nativeQuery = true) + int failAbandonedLegacyRagJob( + @Param("jobId") Long jobId, + @Param("threshold") OffsetDateTime threshold, + @Param("failedAt") OffsetDateTime failedAt, + @Param("diagnostic") String diagnostic); + + /** + * Atomically win the terminal transition for a still-owned legacy RAG + * producer. The surrounding transaction commits this row together with + * project and branch checkpoints, so recovery cannot interleave between + * ownership proof and publication. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query(value = """ + UPDATE job j + SET status = 'COMPLETED', + completed_at = :completedAt, + progress = 100, + updated_at = :completedAt + WHERE j.id = :jobId + AND j.job_type = 'RAG_INCREMENTAL_INDEX' + AND j.status = 'RUNNING' + AND j.updated_at >= :validAfter + AND NOT EXISTS ( + SELECT 1 + FROM rag_index_operation o + WHERE o.job_id = j.id + ) + """, nativeQuery = true) + int completeOwnedLegacyRagJob( + @Param("jobId") Long jobId, + @Param("validAfter") OffsetDateTime validAfter, + @Param("completedAt") OffsetDateTime completedAt); + + /** Retry projection repair if the process stopped after failing the job. */ + @Query("SELECT j.id AS jobId, j.project.id AS projectId, " + + "j.branchName AS branchName, j.commitHash AS commitHash, " + + "j.errorMessage AS errorMessage " + + "FROM Job j WHERE " + + "j.jobType = org.rostilos.codecrow.core.model.job.JobType.RAG_INCREMENTAL_INDEX " + + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.FAILED " + + "AND NOT EXISTS (SELECT o.id FROM RagIndexOperation o WHERE o.jobId = j.id) " + + "AND EXISTS (SELECT s.id FROM RagIndexStatus s WHERE " + + "s.project.id = j.project.id AND s.activeJobId = j.id " + + "AND s.status IN (org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.INDEXING, " + + "org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.UPDATING)) " + + "ORDER BY j.updatedAt ASC") + List findFailedLegacyRagJobsWithActiveStatus( + Pageable pageable); + @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + "j.updatedAt = :claimedAt WHERE j.id = :jobId " + + "AND j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING " + "AND j.updatedAt < :threshold") int claimAbandonedRunningWebhookJob( diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java index a7347032..322d6de5 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java @@ -13,6 +13,14 @@ @Repository public interface RagBranchIndexGenerationRepository extends JpaRepository { + interface CleanupGenerationCandidate { + Long getGenerationId(); + String getCollectionName(); + String getRevision(); + String getManifestDigest(); + RagBranchIndexGenerationStatus getStatus(); + } + Optional findFirstByBranchIndexIdAndRevisionAndStatusInOrderByCreatedAtDesc( Long branchIndexId, String revision, @@ -20,13 +28,34 @@ Optional findFirstByBranchIndexIdAndRevisionAndStatusI List findByBranchIndexIdOrderByCreatedAtDesc(Long branchIndexId); + @Query("SELECT g.id AS generationId, g.collectionName AS collectionName, " + + "g.revision AS revision, g.manifestDigest AS manifestDigest, " + + "g.status AS status " + + "FROM RagBranchIndexGeneration g WHERE g.branchIndex.id = :branchIndexId " + + "ORDER BY g.createdAt DESC") + List findCleanupCandidatesByBranchIndexId( + @Param("branchIndexId") Long branchIndexId); + @Query("SELECT g FROM RagBranchIndexGeneration g " + "JOIN g.branchIndex b WHERE b.project.id = :projectId " + "AND b.branchName = :branchName AND g.revision = :revision " + + "AND b.cleanupClaimToken IS NULL " + "AND g.status IN :statuses ORDER BY g.createdAt DESC") List findAvailableExactGeneration( @Param("projectId") Long projectId, @Param("branchName") String branchName, @Param("revision") String revision, @Param("statuses") List statuses); + + /** + * Physical targets that may still contain a project's PR overlays. Both + * active and superseded published generations are relevant: a generation + * can be superseded between review indexing and the close webhook. + */ + @Query("SELECT DISTINCT g.collectionName FROM RagBranchIndexGeneration g " + + "JOIN g.branchIndex b WHERE b.project.id = :projectId " + + "AND g.status IN :statuses") + List findCollectionNamesByProjectIdAndStatusIn( + @Param("projectId") Long projectId, + @Param("statuses") List statuses); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java index 01c5e701..1757bfbb 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java @@ -3,13 +3,16 @@ import jakarta.persistence.LockModeType; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import java.time.OffsetDateTime; import java.util.List; import java.util.Optional; @@ -20,17 +23,65 @@ public interface RagBranchIndexRepository extends JpaRepository { interface OperatorAliasCandidate { + Long getBranchIndexId(); + Long getGenerationId(); Long getProjectId(); String getWorkspaceName(); String getProjectNamespace(); String getBranchName(); String getRevision(); String getCollectionName(); + String getManifestDigest(); RagBranchIndexKind getIndexKind(); } + /** + * Immutable coordinates required to advance the currently published + * generation. Keeping this boundary scalar prevents callers from carrying + * a lazy generation proxy into long-running VCS or RAG operations after the + * repository transaction has closed. + */ + interface ActiveGenerationCoordinates { + Long getGenerationId(); + String getRevision(); + String getCollectionName(); + String getRepresentationFingerprint(); + Integer getFileCount(); + Integer getChunkCount(); + } + + interface TransientCleanupCandidate { + Long getBranchIndexId(); + Long getProjectId(); + String getWorkspaceName(); + String getProjectNamespace(); + String getBranchName(); + OffsetDateTime getLastAccessedAt(); + OffsetDateTime getUpdatedAt(); + String getCleanupClaimToken(); + OffsetDateTime getCleanupClaimedAt(); + ProjectConfig getProjectConfiguration(); + } + Optional findByProjectIdAndBranchName(Long projectId, String branchName); + @Query(""" + SELECT g.id AS generationId, + g.revision AS revision, + g.collectionName AS collectionName, + g.representationFingerprint AS representationFingerprint, + g.fileCount AS fileCount, + g.chunkCount AS chunkCount + FROM RagBranchIndex b + JOIN b.activeGeneration g + WHERE b.project.id = :projectId + AND b.branchName = :branchName + AND b.cleanupClaimToken IS NULL + """) + Optional findActiveGenerationCoordinates( + @Param("projectId") Long projectId, + @Param("branchName") String branchName); + @Lock(LockModeType.PESSIMISTIC_WRITE) @Query("SELECT b FROM RagBranchIndex b WHERE b.project.id = :projectId AND b.branchName = :branchName") Optional findByProjectIdAndBranchNameForUpdate( @@ -45,6 +96,82 @@ Optional findByProjectIdAndBranchNameForUpdate( List findByIndexKind(RagBranchIndexKind indexKind); + /** Scalar scheduler inputs, materialized before any remote cleanup call. */ + @Query(""" + SELECT b.id AS branchIndexId, + b.project.id AS projectId, + b.project.workspace.name AS workspaceName, + b.project.namespace AS projectNamespace, + b.branchName AS branchName, + b.lastAccessedAt AS lastAccessedAt, + b.updatedAt AS updatedAt, + b.cleanupClaimToken AS cleanupClaimToken, + b.cleanupClaimedAt AS cleanupClaimedAt, + b.project.configuration AS projectConfiguration + FROM RagBranchIndex b + WHERE b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.TRANSIENT + """) + List findTransientCleanupCandidates(); + + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("UPDATE RagBranchIndex b SET b.cleanupClaimToken = :claimToken, " + + "b.cleanupClaimedAt = :claimedAt " + + "WHERE b.id = :branchIndexId " + + "AND b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.TRANSIENT " + + "AND b.lifecycleStatus = " + + "org.rostilos.codecrow.core.model.rag.RagBranchIndexLifecycleStatus.READY " + + "AND (b.cleanupClaimToken IS NULL OR b.cleanupClaimToken = :claimToken " + + "OR b.cleanupClaimedAt < :staleBefore) " + + "AND ((b.lastAccessedAt IS NOT NULL AND b.lastAccessedAt < :cutoff) " + + "OR (b.lastAccessedAt IS NULL AND b.updatedAt < :cutoff))") + int claimExpiredTransientForDeletion( + @Param("branchIndexId") Long branchIndexId, + @Param("cutoff") OffsetDateTime cutoff, + @Param("staleBefore") OffsetDateTime staleBefore, + @Param("claimToken") String claimToken, + @Param("claimedAt") OffsetDateTime claimedAt); + + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("UPDATE RagBranchIndex b SET b.cleanupClaimedAt = :claimedAt " + + "WHERE b.id = :branchIndexId AND b.cleanupClaimToken = :claimToken") + int heartbeatTransientDeletionClaim( + @Param("branchIndexId") Long branchIndexId, + @Param("claimToken") String claimToken, + @Param("claimedAt") OffsetDateTime claimedAt); + + /** + * Atomically records exact-generation use unless cleanup already owns the + * branch. A successful touch makes any expiry claim fail its cutoff check. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("UPDATE RagBranchIndex b SET b.lastAccessedAt = :accessedAt " + + "WHERE b.project.id = :projectId AND b.branchName = :branchName " + + "AND b.cleanupClaimToken IS NULL") + int markAccessedIfUnclaimed( + @Param("projectId") Long projectId, + @Param("branchName") String branchName, + @Param("accessedAt") OffsetDateTime accessedAt); + + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("UPDATE RagBranchIndex b SET b.cleanupClaimToken = NULL, b.cleanupClaimedAt = NULL " + + "WHERE b.id = :branchIndexId AND b.cleanupClaimToken = :claimToken") + int cancelTransientDeletion( + @Param("branchIndexId") Long branchIndexId, + @Param("claimToken") String claimToken); + + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("DELETE FROM RagBranchIndex b WHERE b.id = :branchIndexId " + + "AND b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.TRANSIENT " + + "AND b.cleanupClaimToken = :claimToken") + int deleteClaimedTransientById( + @Param("branchIndexId") Long branchIndexId, + @Param("claimToken") String claimToken); + @Query("SELECT CASE WHEN COUNT(b) > 0 THEN true ELSE false END FROM RagBranchIndex b " + "WHERE b.project.id = :projectId AND b.branchName = :branchName") boolean existsByProjectIdAndBranchName(@Param("projectId") Long projectId, @Param("branchName") String branchName); @@ -65,11 +192,14 @@ Optional findByProjectIdAndBranchNameForUpdate( */ @Query(""" SELECT b.project.id AS projectId, + b.id AS branchIndexId, + b.activeGeneration.id AS generationId, b.project.workspace.name AS workspaceName, b.project.namespace AS projectNamespace, b.branchName AS branchName, b.activeGeneration.revision AS revision, b.activeGeneration.collectionName AS collectionName, + b.activeGeneration.manifestDigest AS manifestDigest, b.indexKind AS indexKind FROM RagBranchIndex b WHERE b.activeGeneration IS NOT NULL @@ -79,4 +209,31 @@ AND b.indexKind IN ( ) """) List findOperatorAliasCandidates(); + + /** + * Re-reads one branch's current publication coordinates without retaining + * an entity or transaction across the remote alias request. + */ + @Query(""" + SELECT b.project.id AS projectId, + b.id AS branchIndexId, + b.activeGeneration.id AS generationId, + b.project.workspace.name AS workspaceName, + b.project.namespace AS projectNamespace, + b.branchName AS branchName, + b.activeGeneration.revision AS revision, + b.activeGeneration.collectionName AS collectionName, + b.activeGeneration.manifestDigest AS manifestDigest, + b.indexKind AS indexKind + FROM RagBranchIndex b + WHERE b.id = :branchIndexId + AND b.activeGeneration IS NOT NULL + AND b.indexKind IN ( + org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.PRIMARY, + org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE + ) + """) + Optional findOperatorAliasCandidateById( + @Param("branchIndexId") Long branchIndexId); + } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java index 65f28ddc..4f2670c6 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java @@ -16,15 +16,43 @@ @Repository public interface RagIndexOperationRepository extends JpaRepository { + /** Detached-safe coordinates for abandoned and failed-operation recovery. */ + interface RecoveryOperationProjection { + Long getOperationId(); + Long getProjectId(); + String getBranchName(); + String getToRevision(); + Long getJobId(); + String getAnalysisLockKey(); + String getErrorMessage(); + } + + interface SucceededOperationProjection { + Long getOperationId(); + Long getProjectId(); + String getBranchName(); + String getToRevision(); + Long getJobId(); + String getAnalysisLockKey(); + Integer getFileCount(); + Integer getChunkCount(); + Boolean getActiveGeneration(); + } + Optional findByProjectIdAndOperationKey(Long projectId, String operationKey); @Lock(LockModeType.PESSIMISTIC_WRITE) @Query("SELECT o FROM RagIndexOperation o WHERE o.id = :operationId") Optional findByIdForUpdate(@Param("operationId") Long operationId); - List findByStatusInAndUpdatedAtBefore( - List statuses, - OffsetDateTime updatedBefore); + @Query("SELECT o.id AS operationId, o.project.id AS projectId, " + + "o.branchName AS branchName, o.toRevision AS toRevision, " + + "o.jobId AS jobId, o.analysisLockKey AS analysisLockKey, " + + "o.errorMessage AS errorMessage FROM RagIndexOperation o " + + "WHERE o.status IN :statuses AND o.updatedAt < :updatedBefore") + List findRecoverableOperationProjections( + @Param("statuses") List statuses, + @Param("updatedBefore") OffsetDateTime updatedBefore); boolean existsByProjectIdAndBranchNameAndStatusIn( Long projectId, @@ -37,7 +65,13 @@ boolean existsByProjectIdAndBranchNameAndStatusIn( * by a partial recovery failure on the next scan. */ @Query(value = """ - SELECT o.* + SELECT o.id AS "operationId", + o.project_id AS "projectId", + o.branch_name AS "branchName", + o.to_revision AS "toRevision", + o.job_id AS "jobId", + o.analysis_lock_key AS "analysisLockKey", + o.error_message AS "errorMessage" FROM rag_index_operation o WHERE o.status = 'FAILED' AND ( @@ -51,17 +85,60 @@ OR EXISTS ( WHERE s.project_id = o.project_id AND s.indexed_branch = o.branch_name AND s.status IN ('INDEXING', 'UPDATING') - AND (s.active_job_id IS NULL OR s.active_job_id = o.job_id) + AND s.active_job_id = o.job_id + ) + OR EXISTS ( + SELECT 1 FROM analysis_lock l + WHERE l.project_id = o.project_id + AND l.branch_name = o.branch_name + AND l.analysis_type = 'RAG_INDEXING' + AND l.lock_key = o.analysis_lock_key + ) + ) + ORDER BY o.completed_at DESC, o.id DESC + """, nativeQuery = true) + List findFailedOperationsWithActiveProjections(); + + /** + * Scalar recovery coordinates for a generation that was published before + * its job/status/lock projections were terminalized. + */ + @Query(value = """ + SELECT o.id AS "operationId", + o.project_id AS "projectId", + o.branch_name AS "branchName", + o.to_revision AS "toRevision", + o.job_id AS "jobId", + o.analysis_lock_key AS "analysisLockKey", + g.file_count AS "fileCount", + g.chunk_count AS "chunkCount", + (b.active_generation_id = g.id) AS "activeGeneration" + FROM rag_index_operation o + JOIN rag_branch_index_generation g ON g.id = o.generation_id + JOIN rag_branch_index b ON b.id = g.branch_index_id + WHERE o.status = 'SUCCEEDED' + AND ( + EXISTS ( + SELECT 1 FROM job j + WHERE j.id = o.job_id + AND j.status IN ('PENDING', 'QUEUED', 'RUNNING', 'WAITING') + ) + OR EXISTS ( + SELECT 1 FROM rag_index_status s + WHERE s.project_id = o.project_id + AND s.indexed_branch = o.branch_name + AND s.status IN ('INDEXING', 'UPDATING') + AND s.active_job_id = o.job_id ) OR EXISTS ( SELECT 1 FROM analysis_lock l WHERE l.project_id = o.project_id AND l.branch_name = o.branch_name AND l.analysis_type = 'RAG_INDEXING' - AND l.commit_hash IS NOT DISTINCT FROM o.to_revision + AND l.lock_key = o.analysis_lock_key ) ) ORDER BY o.completed_at DESC, o.id DESC """, nativeQuery = true) - List findFailedOperationsWithActiveProjections(); + List findSucceededOperationsWithActiveProjections(); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java index 043b4fbe..7606b377 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java @@ -91,6 +91,22 @@ default Job createRagIndexJob( */ void failJob(Job job, String errorMessage); + /** Finish an intentionally skipped job without representing it as a failure. */ + default void skipJob(Job job, String reason) { + completeJob(job, Map.of("status", "skipped", "reason", reason)); + } + + /** + * Announce a success already committed by a fenced repository transition. + * Persistent hosts should override this without updating terminal status. + */ + default void recordExternallyCompletedJob( + Job job, + String state, + String message) { + info(job, state, message); + } + /** * Log an INFO level message to a job. * @param job The job to log to diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java index 5382c051..5602fc7b 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java @@ -358,6 +358,53 @@ public Job completeJob(Job job) { return job; } + /** + * Record and announce a successful terminal transition performed by an + * atomic repository CAS. This never changes job status, so a stale caller + * cannot overwrite a recovery outcome. Repeated calls reuse the durable + * terminal log and still release local streaming subscribers. + */ + @Transactional(propagation = org.springframework.transaction.annotation.Propagation.REQUIRES_NEW) + public void recordExternallyCompletedJob( + Job job, + String step, + String message) { + if (job == null || job.getId() == null) { + return; + } + Job persisted = jobRepository.findByIdForUpdate(job.getId()).orElse(null); + if (persisted == null || persisted.getStatus() != JobStatus.COMPLETED) { + log.info("Ignoring terminal success notification for non-completed job {}", + job.getId()); + return; + } + if (!jobLogRepository.existsByJobIdAndStep(persisted.getId(), step)) { + addLog(persisted, JobLogLevel.INFO, step, message); + } + notifyJobComplete(persisted); + } + + /** Record a failure already committed by an atomic recovery CAS. */ + @Transactional(propagation = org.springframework.transaction.annotation.Propagation.REQUIRES_NEW) + public void recordExternallyFailedJob( + Job job, + String step, + String message) { + if (job == null || job.getId() == null) { + return; + } + Job persisted = jobRepository.findByIdForUpdate(job.getId()).orElse(null); + if (persisted == null || persisted.getStatus() != JobStatus.FAILED) { + log.info("Ignoring terminal failure notification for non-failed job {}", + job.getId()); + return; + } + if (!jobLogRepository.existsByJobIdAndStep(persisted.getId(), step)) { + addLog(persisted, JobLogLevel.ERROR, step, message); + } + notifyJobComplete(persisted); + } + /** * Complete a job and link it to a code analysis. */ @@ -754,6 +801,41 @@ public boolean claimAbandonedRunningWebhookJob(Long jobId, OffsetDateTime thresh jobId, threshold, OffsetDateTime.now()) == 1; } + /** Renew the database-backed lease for one live legacy RAG producer. */ + @Transactional + public boolean renewLegacyRagJobLease( + Long jobId, + OffsetDateTime validAfter, + OffsetDateTime renewedAt) { + return jobRepository.renewLegacyRagJobLease( + jobId, validAfter, renewedAt) == 1; + } + + public List + findAbandonedLegacyRagJobs(OffsetDateTime threshold, int limit) { + return jobRepository.findAbandonedLegacyRagJobs( + threshold, PageRequest.of(0, Math.max(1, limit))); + } + + /** + * Atomically terminalize an abandoned legacy RAG job only while its lease + * is still stale and no exact-generation operation owns it. + */ + @Transactional + public boolean failAbandonedLegacyRagJob( + Long jobId, + OffsetDateTime threshold, + String diagnostic) { + return jobRepository.failAbandonedLegacyRagJob( + jobId, threshold, OffsetDateTime.now(), diagnostic) == 1; + } + + public List + findFailedLegacyRagJobsWithActiveStatus(int limit) { + return jobRepository.findFailedLegacyRagJobsWithActiveStatus( + PageRequest.of(0, Math.max(1, limit))); + } + private void touchJobActivity(Job job) { if (job == null || job.getId() == null) { return; diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.29.0__rag_transient_cleanup_claim.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.29.0__rag_transient_cleanup_claim.sql new file mode 100644 index 00000000..a4f06efa --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.29.0__rag_transient_cleanup_claim.sql @@ -0,0 +1,9 @@ +-- Durable claim for transient-generation cleanup. Readers and rebuilds reject +-- a claimed row while remote physical collections are being removed. +ALTER TABLE rag_branch_index + ADD COLUMN IF NOT EXISTS cleanup_claim_token VARCHAR(64), + ADD COLUMN IF NOT EXISTS cleanup_claimed_at TIMESTAMP WITH TIME ZONE; + +CREATE INDEX IF NOT EXISTS idx_rag_branch_cleanup_claim + ON rag_branch_index(index_kind, cleanup_claimed_at) + WHERE cleanup_claim_token IS NOT NULL; diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java new file mode 100644 index 00000000..ea7006be --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java @@ -0,0 +1,65 @@ +package org.rostilos.codecrow.core.persistence.repository.job; + +import org.junit.jupiter.api.Test; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.Query; + +import java.lang.reflect.Method; +import java.time.OffsetDateTime; + +import static org.assertj.core.api.Assertions.assertThat; + +class JobRepositoryRecoveryQueryTest { + + @Test + void genericWebhookSelectionIsLimitedToPrAndBranchAnalysis() throws Exception { + assertGenericWebhookTypes(query( + "findRecoverableWebhookJobs", + OffsetDateTime.class, + Pageable.class)); + assertGenericWebhookTypes(query( + "findAbandonedRunningWebhookJobs", + OffsetDateTime.class, + Pageable.class)); + } + + @Test + void genericWebhookAtomicClaimsRepeatTheSameTypeBoundary() throws Exception { + assertGenericWebhookTypes(query( + "claimRecoverableWebhookJob", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class)); + assertGenericWebhookTypes(query( + "claimAbandonedRunningWebhookJob", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class)); + } + + private static String query(String methodName, Class... parameters) + throws Exception { + Method method = JobRepository.class.getDeclaredMethod(methodName, parameters); + Query annotation = method.getAnnotation(Query.class); + assertThat(annotation).as("@Query on %s", methodName).isNotNull(); + return annotation.value(); + } + + private static void assertGenericWebhookTypes(String query) { + assertThat(query) + .contains("JobTriggerSource.WEBHOOK") + .contains("JobType.PR_ANALYSIS") + .contains("JobType.BRANCH_ANALYSIS") + .doesNotContain("JobType.BRANCH_RECONCILIATION") + .doesNotContain("JobType.RAG_INITIAL_INDEX") + .doesNotContain("JobType.RAG_INCREMENTAL_INDEX") + .doesNotContain("JobType.MANUAL_ANALYSIS") + .doesNotContain("JobType.REPO_SYNC") + .doesNotContain("JobType.SUMMARIZE_COMMAND") + .doesNotContain("JobType.ASK_COMMAND") + .doesNotContain("JobType.ANALYZE_COMMAND") + .doesNotContain("JobType.REVIEW_COMMAND") + .doesNotContain("JobType.QA_DOC_COMMAND") + .doesNotContain("JobType.IGNORED_COMMENT"); + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/LegacyRagJobRecoveryQueryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/LegacyRagJobRecoveryQueryTest.java new file mode 100644 index 00000000..86cd5e73 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/LegacyRagJobRecoveryQueryTest.java @@ -0,0 +1,110 @@ +package org.rostilos.codecrow.core.persistence.repository.job; + +import org.junit.jupiter.api.Test; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.Query; + +import java.lang.reflect.Method; +import java.time.OffsetDateTime; + +import static org.assertj.core.api.Assertions.assertThat; + +class LegacyRagJobRecoveryQueryTest { + + @Test + void leaseRenewalIsGuardedByTypeStatusAgeAndMissingExactOperation() + throws Exception { + Query query = query( + "renewLegacyRagJobLease", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class); + + assertThat(query.nativeQuery()).isTrue(); + assertThat(query.value()) + .contains("RAG_INCREMENTAL_INDEX") + .contains("RUNNING") + .contains("updated_at >= :validAfter") + .contains("NOT EXISTS") + .contains("rag_index_operation"); + } + + @Test + void abandonedSelectionUsesScalarCoordinatesAndExcludesExactOperations() + throws Exception { + Query query = query( + "findAbandonedLegacyRagJobs", + OffsetDateTime.class, + Pageable.class); + + assertThat(query.value()) + .contains("j.id AS jobId") + .contains("j.project.id AS projectId") + .contains("RAG_INCREMENTAL_INDEX") + .contains("JobStatus.PENDING") + .contains("JobStatus.RUNNING") + .contains("j.updatedAt < :threshold") + .contains("NOT EXISTS") + .contains("RagIndexOperation"); + } + + @Test + void abandonmentClaimRepeatsEveryLeaseFenceAtomically() throws Exception { + Query query = query( + "failAbandonedLegacyRagJob", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class, + String.class); + + assertThat(query.nativeQuery()).isTrue(); + assertThat(query.value()) + .contains("status = 'FAILED'") + .contains("RAG_INCREMENTAL_INDEX") + .contains("'PENDING', 'QUEUED', 'RUNNING', 'WAITING'") + .contains("updated_at < :threshold") + .contains("NOT EXISTS") + .contains("rag_index_operation"); + } + + @Test + void completionRepeatsEveryLeaseFenceAtomically() throws Exception { + Query query = query( + "completeOwnedLegacyRagJob", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class); + + assertThat(query.nativeQuery()).isTrue(); + assertThat(query.value()) + .contains("status = 'COMPLETED'") + .contains("RAG_INCREMENTAL_INDEX") + .contains("status = 'RUNNING'") + .contains("updated_at >= :validAfter") + .contains("NOT EXISTS") + .contains("rag_index_operation"); + } + + @Test + void failedProjectionRepairIsOwnedByTheSameJob() throws Exception { + Query query = query( + "findFailedLegacyRagJobsWithActiveStatus", + Pageable.class); + + assertThat(query.value()) + .contains("JobStatus.FAILED") + .contains("s.activeJobId = j.id") + .contains("RagIndexingStatus.INDEXING") + .contains("RagIndexingStatus.UPDATING") + .contains("NOT EXISTS") + .contains("RagIndexOperation"); + } + + private static Query query(String name, Class... parameterTypes) + throws Exception { + Method method = JobRepository.class.getDeclaredMethod(name, parameterTypes); + Query query = method.getAnnotation(Query.class); + assertThat(query).as("@Query on %s", name).isNotNull(); + return query; + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRecoveryQueryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRecoveryQueryTest.java new file mode 100644 index 00000000..4dca8202 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRecoveryQueryTest.java @@ -0,0 +1,60 @@ +package org.rostilos.codecrow.core.persistence.repository.rag; + +import org.junit.jupiter.api.Test; +import org.springframework.data.jpa.repository.Query; + +import java.lang.reflect.Method; +import java.time.OffsetDateTime; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class RagIndexOperationRecoveryQueryTest { + + @Test + void abandonedSelectionReturnsOnlyDetachedSafeScalarCoordinates() + throws Exception { + Method method = RagIndexOperationRepository.class.getDeclaredMethod( + "findRecoverableOperationProjections", List.class, OffsetDateTime.class); + Query annotation = method.getAnnotation(Query.class); + + assertThat(annotation).isNotNull(); + assertThat(method.getGenericReturnType().getTypeName()) + .contains("RecoveryOperationProjection") + .doesNotContain("RagIndexOperation>"); + assertThat(annotation.value()) + .contains("o.project.id AS projectId") + .doesNotContain("SELECT o FROM"); + } + + @Test + void failedProjectionRecoveryRequiresExactStatusAndLockOwnership() + throws Exception { + String query = query("findFailedOperationsWithActiveProjections"); + + assertThat(query) + .contains("o.project_id AS \"projectId\"") + .contains("s.active_job_id = o.job_id") + .contains("l.lock_key = o.analysis_lock_key") + .doesNotContain("s.active_job_id IS NULL"); + } + + @Test + void succeededProjectionRecoveryRequiresExactStatusAndLockOwnership() + throws Exception { + String query = query("findSucceededOperationsWithActiveProjections"); + + assertThat(query) + .contains("s.active_job_id = o.job_id") + .contains("l.lock_key = o.analysis_lock_key") + .doesNotContain("s.active_job_id IS NULL"); + } + + private static String query(String methodName) throws Exception { + Method method = RagIndexOperationRepository.class.getDeclaredMethod(methodName); + Query annotation = method.getAnnotation(Query.class); + assertThat(annotation).as("@Query on %s", methodName).isNotNull(); + assertThat(annotation.nativeQuery()).isTrue(); + return annotation.value(); + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java index f2b64cb5..b10fdffa 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java @@ -51,6 +51,34 @@ void errorShouldCallLogToJobWithErrorLevel() { verify(service).logToJob(job, JobLogLevel.ERROR, "test-state", "error message"); } + @Test + @DisplayName("skipJob() should complete compatibility implementations without failing") + void skipJobShouldUseNonFailureCompletionResult() { + TestAnalysisJobService service = spy(new TestAnalysisJobService()); + Job job = new Job(); + + service.skipJob(job, "deferred until a later trigger"); + + verify(service).completeJob(job, Map.of( + "status", "skipped", + "reason", "deferred until a later trigger")); + verify(service, never()).failJob(any(), anyString()); + } + + @Test + void externallyCompletedJobDefaultsToNotificationOnly() { + TestAnalysisJobService service = spy(new TestAnalysisJobService()); + Job job = new Job(); + + service.recordExternallyCompletedJob( + job, "rag_complete", "RAG index updated"); + + verify(service).logToJob( + job, JobLogLevel.INFO, "rag_complete", "RAG index updated"); + verify(service, never()).completeJob(any(), any()); + verify(service, never()).failJob(any(), anyString()); + } + } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceLegacyRagRecoveryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceLegacyRagRecoveryTest.java new file mode 100644 index 00000000..70a01e29 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceLegacyRagRecoveryTest.java @@ -0,0 +1,139 @@ +package org.rostilos.codecrow.core.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobLog; +import org.rostilos.codecrow.core.model.job.JobStatus; +import org.rostilos.codecrow.core.persistence.repository.job.JobLogRepository; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.springframework.data.domain.Pageable; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.OffsetDateTime; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class JobServiceLegacyRagRecoveryTest { + @Mock private JobRepository jobs; + @Mock private JobLogRepository logs; + + private JobService service; + + @BeforeEach + void setUp() { + service = new JobService(jobs, logs, new ObjectMapper()); + } + + @Test + void renewsTheExactLeaseWindowProvidedByTheProducer() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + OffsetDateTime renewedAt = OffsetDateTime.now(); + when(jobs.renewLegacyRagJobLease(91L, validAfter, renewedAt)) + .thenReturn(1); + + assertThat(service.renewLegacyRagJobLease( + 91L, validAfter, renewedAt)).isTrue(); + } + + @Test + void exposesOnlyTheBoundedRecoveryBatch() { + JobRepository.LegacyRagJobRecoveryCoordinates coordinates = + org.mockito.Mockito.mock( + JobRepository.LegacyRagJobRecoveryCoordinates.class); + when(jobs.findAbandonedLegacyRagJobs( + any(OffsetDateTime.class), any(Pageable.class))) + .thenReturn(List.of(coordinates)); + + assertThat(service.findAbandonedLegacyRagJobs( + OffsetDateTime.now(), 17)).containsExactly(coordinates); + verify(jobs).findAbandonedLegacyRagJobs( + any(OffsetDateTime.class), argThat(page -> page.getPageSize() == 17)); + } + + @Test + void reportsWhetherRecoveryWonTheAtomicFence() { + when(jobs.failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), any(), eq("abandoned"))) + .thenReturn(1); + + assertThat(service.failAbandonedLegacyRagJob( + 91L, OffsetDateTime.now(), "abandoned")).isTrue(); + } + + @Test + void recordsAnAlreadyCompletedJobWithoutRepeatingItsTerminalTransition() { + Job supplied = new Job(); + ReflectionTestUtils.setField(supplied, "id", 91L); + Job persisted = new Job(); + ReflectionTestUtils.setField(persisted, "id", 91L); + persisted.setExternalId("job-91"); + persisted.setStatus(JobStatus.COMPLETED); + when(jobs.findByIdForUpdate(91L)).thenReturn(java.util.Optional.of(persisted)); + when(logs.existsByJobIdAndStep(91L, "rag_complete")).thenReturn(false); + when(logs.getNextSequenceNumber(91L)).thenReturn(4L); + when(logs.save(any(JobLog.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + service.recordExternallyCompletedJob( + supplied, "rag_complete", "RAG index updated"); + + verify(logs).save(argThat(entry -> + entry.getJob() == persisted + && "rag_complete".equals(entry.getStep()) + && "RAG index updated".equals(entry.getMessage()))); + verify(jobs, never()).save(any()); + } + + @Test + void repeatedTerminalNotificationDoesNotDuplicateTheDurableLog() { + Job supplied = new Job(); + ReflectionTestUtils.setField(supplied, "id", 91L); + Job persisted = new Job(); + ReflectionTestUtils.setField(persisted, "id", 91L); + persisted.setExternalId("job-91"); + persisted.setStatus(JobStatus.COMPLETED); + when(jobs.findByIdForUpdate(91L)).thenReturn(java.util.Optional.of(persisted)); + when(logs.existsByJobIdAndStep(91L, "rag_complete")).thenReturn(true); + + service.recordExternallyCompletedJob( + supplied, "rag_complete", "RAG index updated"); + + verify(logs, never()).save(any()); + verify(jobs, never()).save(any()); + } + + @Test + void recordsAnAlreadyFailedJobWithoutRepeatingItsTerminalTransition() { + Job supplied = new Job(); + ReflectionTestUtils.setField(supplied, "id", 91L); + Job persisted = new Job(); + ReflectionTestUtils.setField(persisted, "id", 91L); + persisted.setExternalId("job-91"); + persisted.setStatus(JobStatus.FAILED); + when(jobs.findByIdForUpdate(91L)) + .thenReturn(java.util.Optional.of(persisted)); + when(logs.existsByJobIdAndStep(91L, "rag_recovery")).thenReturn(false); + when(logs.getNextSequenceNumber(91L)).thenReturn(4L); + when(logs.save(any(JobLog.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + service.recordExternallyFailedJob( + supplied, "rag_recovery", "Job failed: producer expired"); + + verify(logs).save(argThat(entry -> + entry.getJob() == persisted + && entry.getLevel() + == org.rostilos.codecrow.core.model.job.JobLogLevel.ERROR + && "rag_recovery".equals(entry.getStep()))); + verify(jobs, never()).save(any()); + } +} diff --git a/java-ecosystem/libs/events/src/main/java/org/rostilos/codecrow/events/EventNotificationEmitter.java b/java-ecosystem/libs/events/src/main/java/org/rostilos/codecrow/events/EventNotificationEmitter.java index 9474205b..3adcae92 100644 --- a/java-ecosystem/libs/events/src/main/java/org/rostilos/codecrow/events/EventNotificationEmitter.java +++ b/java-ecosystem/libs/events/src/main/java/org/rostilos/codecrow/events/EventNotificationEmitter.java @@ -4,11 +4,25 @@ import java.util.function.Consumer; public class EventNotificationEmitter { + private EventNotificationEmitter() { + } + + /** + * Delivers an observational progress event. A missing or disconnected + * observer must never change the durable analysis outcome. + */ public static void emitStatus(Consumer> consumer, String state, String description) { - consumer.accept( - Map.of( + if (consumer == null) { + return; + } + try { + consumer.accept(Map.of( "type", "status", "state", state, "message", description)); + } catch (RuntimeException ignored) { + // Observers are optional transport adapters. Their lifecycle is not + // part of the durable operation result. + } } } diff --git a/java-ecosystem/libs/events/src/test/java/org/rostilos/codecrow/events/EventNotificationEmitterTest.java b/java-ecosystem/libs/events/src/test/java/org/rostilos/codecrow/events/EventNotificationEmitterTest.java new file mode 100644 index 00000000..a7ec8a1d --- /dev/null +++ b/java-ecosystem/libs/events/src/test/java/org/rostilos/codecrow/events/EventNotificationEmitterTest.java @@ -0,0 +1,28 @@ +package org.rostilos.codecrow.events; + +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThatCode; + +class EventNotificationEmitterTest { + + @Test + void missingObserverIsANoOp() { + assertThatCode(() -> EventNotificationEmitter.emitStatus(null, "started", "Started")) + .doesNotThrowAnyException(); + } + + @Test + void observerFailureCannotChangeOperationOutcome() { + Consumer> disconnected = event -> { + throw new IllegalStateException("stream disconnected"); + }; + + assertThatCode(() -> EventNotificationEmitter.emitStatus( + disconnected, "completed", "Completed")) + .doesNotThrowAnyException(); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java new file mode 100644 index 00000000..d21a3dbf --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java @@ -0,0 +1,173 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.service.AnalysisJobService; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** + * Commits the durable ownership boundary for an exact branch snapshot. + * + *

The RAG operation is registered before its child job is created. The job + * and operation are then linked and moved to RUNNING in this one transaction, + * so recovery can never observe a committed running exact-generation job + * without a corresponding operation.

+ */ +@Service +public class BranchIndexBuildAdmissionService { + + public enum BuildOrigin { + AUTOMATIC("automatic"), + OPERATOR("operator"); + + private final String fingerprintLabel; + + BuildOrigin(String fingerprintLabel) { + this.fingerprintLabel = fingerprintLabel; + } + } + + public enum ProjectStatusAdmission { + NONE, + INDEXING, + UPDATING + } + + public record AdmittedBuild( + Job job, + BranchIndexGenerationBuildService.PreparedBuild preparedBuild, + ProjectStatusAdmission statusAdmission) { + } + + private final RagBranchIndexRegistryService registryService; + private final AnalysisJobService jobService; + private final RagIndexTrackingService trackingService; + + public BranchIndexBuildAdmissionService( + RagBranchIndexRegistryService registryService, + AnalysisJobService jobService, + RagIndexTrackingService trackingService) { + this.registryService = registryService; + this.jobService = jobService; + this.trackingService = trackingService; + } + + @Transactional + public AdmittedBuild admit( + Project project, + String branch, + String revision, + RagBranchIndexKind kind, + JobTriggerSource triggerSource, + String analysisLockKey, + BuildOrigin origin) { + String lockKey = requireText(analysisLockKey, "analysisLockKey"); + BuildOrigin buildOrigin = origin != null ? origin : BuildOrigin.AUTOMATIC; + + var registration = registryService.registerBuild( + project, + branch, + kind, + null, + revision, + operationFingerprint(buildOrigin, lockKey)); + if (registration.existingOperation()) { + // A lock key identifies one acquisition. Seeing it again means a + // previous admission committed and recovery already owns that + // operation; creating a second child job would split ownership. + throw new IllegalStateException( + "Exact RAG build for this lock was already admitted"); + } + + BranchIndexGenerationBuildService.PreparedBuild prepared = + BranchIndexGenerationBuildService.prepare(registration, lockKey); + var activeSource = registration.generation().getParentGeneration(); + ProjectStatusAdmission projectStatus = kind != RagBranchIndexKind.PRIMARY + ? ProjectStatusAdmission.NONE + : (activeSource == null + ? ProjectStatusAdmission.INDEXING + : ProjectStatusAdmission.UPDATING); + + if (projectStatus == ProjectStatusAdmission.UPDATING) { + // The active exact generation is the authoritative completed + // checkpoint. Align a stale/failed legacy status under the branch + // lock before switching it to UPDATING, all in this transaction. + trackingService.preparePublishedGenerationForUpdate( + project, + branch, + activeSource.getRevision(), + activeSource.getFileCount(), + activeSource.getChunkCount()); + } + Job job = jobService.createRagIndexJob( + project, + projectStatus == ProjectStatusAdmission.INDEXING, + triggerSource, + branch, + revision); + if (job == null || job.getId() == null) { + throw new IllegalStateException( + "A durable branch-bound RAG job could not be created"); + } + + // Attach the operation before starting the job. The surrounding + // transaction commits both state transitions together. + registryService.startBuild(prepared.operationId(), job.getId(), lockKey); + jobService.startJob(job); + if (projectStatus == ProjectStatusAdmission.INDEXING) { + trackingService.markIndexingStarted( + project, branch, revision, job.getId()); + } else if (projectStatus == ProjectStatusAdmission.UPDATING) { + trackingService.markUpdatingStarted( + project, branch, revision, job.getId()); + } + return new AdmittedBuild(job, prepared, projectStatus); + } + + /** + * Immediately terminalizes an admitted build that cannot reach execute. + * This is only for failures in the narrow local hand-off after admission; + * failures inside execute are owned by the build service itself. + */ + public void abortOperation(AdmittedBuild admission, String diagnostic) { + if (admission == null) { + return; + } + String failure = diagnostic == null || diagnostic.isBlank() + ? "Exact RAG build failed before execution" + : diagnostic; + registryService.fail(admission.preparedBuild().operationId(), failure); + } + + static String operationFingerprint(BuildOrigin origin, String analysisLockKey) { + return "exact-full-snapshot:" + origin.fingerprintLabel + ":" + + digest(requireText(analysisLockKey, "analysisLockKey")); + } + + private static String requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " is required"); + } + return value.trim(); + } + + private static String digest(String value) { + try { + return HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException failure) { + throw new IllegalStateException("SHA-256 is unavailable", failure); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java index cbc3a45d..589d1f25 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.ragengine.branch; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; @@ -10,6 +11,8 @@ import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.IOException; @@ -18,11 +21,8 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; /** @@ -34,21 +34,57 @@ public class BranchIndexGenerationBuildService { private static final Logger log = LoggerFactory.getLogger( BranchIndexGenerationBuildService.class); - private static final long HEARTBEAT_INTERVAL_SECONDS = 15; - private final BranchArchiveService archiveService; private final RagPipelineClient pipelineClient; private final RagBranchIndexRegistryService registryService; - private final ScheduledExecutorService heartbeatExecutor; + private final RagIndexOperationHeartbeatService heartbeatService; + private final AnalysisLockService analysisLockService; + private final int ragLockLeaseMinutes; + + /** + * The durable coordinates needed after build admission commits. Keeping + * this boundary scalar prevents callers from carrying detached registry + * entities (and their lazy associations) into archive or RAG I/O. + */ + public record PreparedBuild( + long operationId, + String collectionTarget, + boolean alreadySucceeded, + String manifestDigest, + String analysisLockKey) { + + public PreparedBuild { + if (operationId <= 0) { + throw new IllegalArgumentException("operationId must be positive"); + } + if (collectionTarget == null || collectionTarget.isBlank()) { + throw new IllegalArgumentException("collectionTarget is required"); + } + } + } public BranchIndexGenerationBuildService( BranchArchiveService archiveService, RagPipelineClient pipelineClient, - RagBranchIndexRegistryService registryService) { + RagBranchIndexRegistryService registryService, + RagIndexOperationHeartbeatService heartbeatService) { + this(archiveService, pipelineClient, registryService, heartbeatService, null, 360); + } + + @Autowired + public BranchIndexGenerationBuildService( + BranchArchiveService archiveService, + RagPipelineClient pipelineClient, + RagBranchIndexRegistryService registryService, + RagIndexOperationHeartbeatService heartbeatService, + AnalysisLockService analysisLockService, + @Value("${analysis.lock.rag.timeout.minutes:360}") int ragLockLeaseMinutes) { this.archiveService = archiveService; this.pipelineClient = pipelineClient; this.registryService = registryService; - this.heartbeatExecutor = Executors.newSingleThreadScheduledExecutor(new HeartbeatThreadFactory()); + this.heartbeatService = heartbeatService; + this.analysisLockService = analysisLockService; + this.ragLockLeaseMinutes = Math.max(1, ragLockLeaseMinutes); } public Map build( @@ -169,25 +205,57 @@ private Map buildInternal( boolean forceRebuild) throws IOException { var registration = registryService.registerBuild( project, branch, kind, null, revision, - forceRebuild ? "operator-refresh:" + requireJobId(jobId) : null); - if (registration.existingOperation() - && registration.operation().getStatus() - == RagIndexOperationStatus.SUCCEEDED) { + forceRebuild + ? "full-snapshot:job:" + requireJobId(jobId) + ":" + UUID.randomUUID() + : null); + PreparedBuild prepared = prepare(registration, analysisLockKey); + if (prepared.alreadySucceeded()) { return Map.of( "status", "reused", - "collection_target", registration.generation().getCollectionName(), - "generation_manifest_sha256", registration.generation().getManifestDigest()); + "collection_target", prepared.collectionTarget(), + "generation_manifest_sha256", prepared.manifestDigest()); } registryService.startBuild( - registration.operation().getId(), jobId, analysisLockKey); - ScheduledFuture heartbeat = heartbeatExecutor.scheduleAtFixedRate( - () -> heartbeat(registration.operation().getId()), - HEARTBEAT_INTERVAL_SECONDS, - HEARTBEAT_INTERVAL_SECONDS, - TimeUnit.SECONDS); + prepared.operationId(), jobId, analysisLockKey); + return execute(project, connection, vcsWorkspace, repoSlug, branch, revision, + kind, includePatterns, excludePatterns, prepared, progressEvents); + } + + /** + * Executes an already-admitted build. The operation must have been linked + * to its durable job and transitioned to RUNNING in the admission + * transaction before this method is called. + */ + public Map execute( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + PreparedBuild prepared, + Consumer> progressEvents) throws IOException { + if (prepared == null) { + throw new IllegalArgumentException("prepared build is required"); + } + if (prepared.alreadySucceeded()) { + return Map.of( + "status", "reused", + "collection_target", prepared.collectionTarget(), + "generation_manifest_sha256", prepared.manifestDigest()); + } + + RagIndexOperationHeartbeatService.HeartbeatScope heartbeat = null; + AnalysisLockService.LockLease lockLease = null; Path snapshot = null; + AtomicBoolean snapshotOwnershipTransferred = new AtomicBoolean(false); try { + lockLease = startAnalysisLockLease(prepared); + heartbeat = heartbeatService.start(prepared.operationId()); snapshot = Files.createTempDirectory("codecrow-rag-branch-generation-"); archiveService.downloadAndExtractSnapshotToDirectory( connection, @@ -203,30 +271,35 @@ private Map buildInternal( ? pipelineClient.indexRepository( snapshot.toString(), project.getWorkspace().getName(), project.getNamespace(), branch, revision, includePatterns, - excludePatterns, registration.generation().getCollectionName(), + excludePatterns, prepared.collectionTarget(), false, false) : pipelineClient.indexRepository( snapshot.toString(), project.getWorkspace().getName(), project.getNamespace(), branch, revision, includePatterns, - excludePatterns, registration.generation().getCollectionName(), - false, false, + excludePatterns, prepared.collectionTarget(), + false, false, true, + () -> snapshotOwnershipTransferred.set(true), progressEvents); Object manifest = result.get("generation_manifest_sha256"); if (!(manifest instanceof String digest) || digest.isBlank()) { throw new IOException("RAG full branch generation has no manifest digest"); } + if (lockLease != null && !lockLease.confirmOwnership()) { + throw new IOException( + "RAG indexing lock ownership was lost before generation publication"); + } var published = registryService.publish( - registration.operation().getId(), + prepared.operationId(), digest, number(result.get("document_count")), number(result.get("chunk_count"))); publishReadableAliasesIfActive( - project, branch, revision, registration.generation().getCollectionName(), + project, branch, revision, prepared.collectionTarget(), published, publishBranchAlias, publishLegacyProjectAlias); return result; } catch (Throwable failure) { registryService.fail( - registration.operation().getId(), + prepared.operationId(), failure.getMessage() != null ? failure.getMessage() : failure.getClass().getSimpleName()); @@ -238,13 +311,58 @@ private Map buildInternal( } throw new IOException("Failed to build exact branch generation", failure); } finally { - heartbeat.cancel(false); - if (snapshot != null) { + if (lockLease != null) { + lockLease.close(); + } + if (heartbeat != null) { + heartbeat.close(); + } + if (snapshot != null && !snapshotOwnershipTransferred.get()) { deleteTree(snapshot); } } } + /** Maps a managed registration to the scalar post-transaction boundary. */ + public static PreparedBuild prepare( + RagBranchIndexRegistryService.BuildRegistration registration) { + return prepare(registration, null); + } + + public static PreparedBuild prepare( + RagBranchIndexRegistryService.BuildRegistration registration, + String analysisLockKey) { + if (registration == null || registration.operation() == null + || registration.generation() == null) { + throw new IllegalArgumentException("A complete build registration is required"); + } + boolean succeeded = registration.existingOperation() + && registration.operation().getStatus() == RagIndexOperationStatus.SUCCEEDED; + return new PreparedBuild( + registration.operation().getId(), + registration.generation().getCollectionName(), + succeeded, + registration.generation().getManifestDigest(), + analysisLockKey); + } + + private AnalysisLockService.LockLease startAnalysisLockLease(PreparedBuild prepared) + throws IOException { + if (prepared.analysisLockKey() == null || prepared.analysisLockKey().isBlank()) { + return null; + } + if (analysisLockService == null) { + throw new IOException("Analysis-lock lease service is unavailable for exact RAG build"); + } + AnalysisLockService.LockLease lease = analysisLockService.maintainLockLease( + prepared.analysisLockKey(), ragLockLeaseMinutes); + if (lease.isOwnershipLost()) { + lease.close(); + throw new IOException("RAG indexing lock ownership was lost before snapshot build"); + } + return lease; + } + private void publishReadableAliasesIfActive( Project project, String branch, @@ -262,11 +380,13 @@ private void publishReadableAliasesIfActive( pipelineClient.publishGenerationAliases( project.getWorkspace().getName(), project.getNamespace(), branch, revision, collectionTarget, + published.getManifestDigest(), true, publishLegacyProjectAlias); - } catch (IOException aliasFailure) { + } catch (IOException | RuntimeException aliasFailure) { // Readable aliases are operator convenience. Exact retrieval uses // the registry target, and reconciliation repairs this alias later. - log.warn("Could not publish readable aliases for RAG generation {}: {}", + log.info("Readable aliases were not published for active RAG generation {}; " + + "the reconciliation scheduler will retry: {}", published.getId(), aliasFailure.getMessage()); } } @@ -282,24 +402,6 @@ private static String requireJobId(Long jobId) { return jobId.toString(); } - private void heartbeat(long operationId) { - try { - registryService.heartbeatBuild(operationId); - } catch (Exception ignored) { - // A later heartbeat may still succeed. If the producer actually - // stops, recovery turns the durable operation into a failure. - } - } - - private static final class HeartbeatThreadFactory implements ThreadFactory { - @Override - public Thread newThread(Runnable runnable) { - Thread thread = new Thread(runnable, "rag-generation-heartbeat"); - thread.setDaemon(true); - return thread; - } - } - private static void deleteTree(Path root) { try { if (!Files.exists(root)) { diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java index 2b5accdd..f6665d54 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java @@ -13,6 +13,8 @@ import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -39,9 +41,13 @@ */ @Service public class BranchIndexMaintenanceService { + private static final Logger log = LoggerFactory.getLogger( + BranchIndexMaintenanceService.class); + private final RagOperationsService ragOperationsService; private final VcsClientProvider vcsClientProvider; private final BranchIndexGenerationBuildService generationBuildService; + private final BranchIndexBuildAdmissionService buildAdmissionService; private final RagIndexTrackingService trackingService; private final AnalysisLockService lockService; private final AnalysisJobService jobService; @@ -52,6 +58,7 @@ public BranchIndexMaintenanceService( RagOperationsService ragOperationsService, VcsClientProvider vcsClientProvider, BranchIndexGenerationBuildService generationBuildService, + BranchIndexBuildAdmissionService buildAdmissionService, RagIndexTrackingService trackingService, AnalysisLockService lockService, AnalysisJobService jobService, @@ -60,6 +67,7 @@ public BranchIndexMaintenanceService( this.ragOperationsService = ragOperationsService; this.vcsClientProvider = vcsClientProvider; this.generationBuildService = generationBuildService; + this.buildAdmissionService = buildAdmissionService; this.trackingService = trackingService; this.lockService = lockService; this.jobService = jobService; @@ -86,7 +94,7 @@ public Map rebuild(Project project, String requestedBranch, bool String message = failure.getMessage() != null ? failure.getMessage() : failure.getClass().getSimpleName(); failures.put(branch, message); - events.accept(Map.of("type", "progress", "stage", "branch_failed", + emitEvent(events, Map.of("type", "progress", "stage", "branch_failed", "branch", branch, "message", "RAG snapshot failed for branch '" + branch + "': " + message)); } @@ -101,7 +109,7 @@ public Map rebuild(Project project, String requestedBranch, bool for (BranchBuildPlan plan : plans.subList(from, to)) { String branch = plan.branch(); wave.put(branch, CompletableFuture.runAsync(() -> { - events.accept(Map.of("type", "progress", "stage", "branch", + emitEvent(events, Map.of("type", "progress", "stage", "branch", "branch", branch, "message", "Building exact RAG snapshot for branch '" + branch + "'")); rebuildOne(project, plan, events); @@ -116,7 +124,7 @@ public Map rebuild(Project project, String requestedBranch, bool String message = cause.getMessage() != null ? cause.getMessage() : cause.getClass().getSimpleName(); failures.put(build.getKey(), message); - events.accept(Map.of("type", "progress", "stage", "branch_failed", + emitEvent(events, Map.of("type", "progress", "stage", "branch_failed", "branch", build.getKey(), "message", "RAG snapshot failed for branch '" + build.getKey() + "': " + message)); @@ -172,36 +180,45 @@ private void rebuildOne(Project project, BranchBuildPlan plan, Consumer result = generationBuildService.rebuild( + if (config.includePatterns() == null || config.excludePatterns() == null) { + throw new IllegalStateException("RAG include/exclude patterns are unavailable"); + } + admittedBuild = buildAdmissionService.admit( + project, + branch, + revision, + kind, + JobTriggerSource.UI, + lock.get(), + BranchIndexBuildAdmissionService.BuildOrigin.OPERATOR); + job = admittedBuild.job(); + Job admittedJob = job; + jobService.logToJob( + admittedJob, + JobLogLevel.INFO, + "branch_snapshot", + "Building exact RAG snapshot for branch: " + branch, + Map.of("branch", branch, "commit", revision)); + executionStarted = true; + Map result = generationBuildService.execute( project, connection, workspace, repository, branch, revision, - primary ? RagBranchIndexKind.PRIMARY : RagBranchIndexKind.DURABLE, + kind, config.includePatterns(), config.excludePatterns(), - job.getId(), lock.get(), event -> { + admittedBuild.preparedBuild(), event -> { Map forwarded = new LinkedHashMap<>(event); forwarded.put("type", "progress"); forwarded.put("branch", branch); @@ -209,44 +226,88 @@ private void rebuildOne(Project project, BranchBuildPlan plan, Consumer> events, + Map event) { + if (events == null) { + return; + } + try { + events.accept(event); + } catch (RuntimeException observerFailure) { + log.debug("RAG maintenance observer rejected event stage={}: {}", + event.get("stage"), observerFailure.getMessage()); + } + } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseService.java new file mode 100644 index 00000000..182d02c1 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseService.java @@ -0,0 +1,207 @@ +package org.rostilos.codecrow.ragengine.branch; + +import jakarta.annotation.PreDestroy; +import org.rostilos.codecrow.core.service.JobService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Maintains a database-backed producer lease for legacy shared-collection RAG + * updates. Exact-generation jobs use {@code RagIndexOperation} instead. + */ +@Service +public class LegacyRagJobLeaseService { + private static final Logger log = LoggerFactory.getLogger( + LegacyRagJobLeaseService.class); + + private final JobService jobService; + private final ScheduledExecutorService executor; + private final long leaseSeconds; + private final long heartbeatIntervalSeconds; + + @Autowired + public LegacyRagJobLeaseService( + JobService jobService, + @Value("${codecrow.rag.legacy-job.lease-seconds:120}") long leaseSeconds, + @Value("${codecrow.rag.legacy-job.heartbeat-interval-seconds:15}") + long heartbeatIntervalSeconds, + @Value("${codecrow.rag.legacy-job.heartbeat-threads:4}") + int heartbeatThreads) { + this( + jobService, + Executors.newScheduledThreadPool( + Math.max(2, heartbeatThreads), + new HeartbeatThreadFactory()), + leaseSeconds, + heartbeatIntervalSeconds); + } + + LegacyRagJobLeaseService( + JobService jobService, + ScheduledExecutorService executor, + long leaseSeconds, + long heartbeatIntervalSeconds) { + this.jobService = jobService; + this.executor = executor; + this.leaseSeconds = Math.max(30, leaseSeconds); + this.heartbeatIntervalSeconds = Math.max( + 1, + Math.min(heartbeatIntervalSeconds, this.leaseSeconds / 3)); + } + + /** + * Start supervising a RUNNING legacy RAG job. The first renewal is + * synchronous, so callers can refuse remote mutation without proven + * ownership. + */ + public JobLease start(long jobId) { + ActiveJobLease lease = new ActiveJobLease(jobId); + lease.renew(); + lease.schedule(); + return lease; + } + + public interface JobLease extends AutoCloseable { + boolean isOwnershipLost(); + + /** Earliest activity timestamp that is still owned by this lease. */ + OffsetDateTime validAfter(); + + /** Atomically prove ownership immediately before durable publication. */ + boolean confirmOwnership(); + + @Override + void close(); + } + + private final class ActiveJobLease implements JobLease { + private final long jobId; + private final AtomicBoolean ownershipLost = new AtomicBoolean(false); + private final AtomicBoolean closed = new AtomicBoolean(false); + private final AtomicBoolean transientFailureReported = new AtomicBoolean(false); + private final AtomicReference knownExpiresAt = + new AtomicReference<>(); + private volatile ScheduledFuture heartbeat; + + private ActiveJobLease(long jobId) { + this.jobId = jobId; + } + + private void schedule() { + if (ownershipLost.get()) { + return; + } + heartbeat = executor.scheduleWithFixedDelay( + this::renew, + heartbeatIntervalSeconds, + heartbeatIntervalSeconds, + TimeUnit.SECONDS); + } + + private boolean renew() { + if (closed.get() || ownershipLost.get()) { + return false; + } + OffsetDateTime renewalStartedAt = OffsetDateTime.now(); + try { + boolean renewed = jobService.renewLegacyRagJobLease( + jobId, + renewalStartedAt.minusSeconds(leaseSeconds), + renewalStartedAt); + if (!renewed) { + ownershipLost.set(true); + log.info("Legacy RAG job lease ownership was lost: job={}", jobId); + return false; + } + knownExpiresAt.set( + renewalStartedAt.plusSeconds(leaseSeconds).minusSeconds(1)); + if (transientFailureReported.compareAndSet(true, false)) { + log.info("Legacy RAG job heartbeat recovered: job={}", jobId); + } + return true; + } catch (RuntimeException renewalFailure) { + OffsetDateTime knownExpiry = knownExpiresAt.get(); + boolean previousLeaseStillActive = knownExpiry != null + && OffsetDateTime.now().isBefore(knownExpiry); + if (!previousLeaseStillActive) { + ownershipLost.set(true); + log.info( + "Legacy RAG job ownership could not be confirmed before its known expiry: " + + "job={}, detail={}", + jobId, + renewalFailure.getMessage()); + return false; + } + if (transientFailureReported.compareAndSet(false, true)) { + log.warn( + "Legacy RAG job heartbeat failed within the active lease; retrying: " + + "job={}, detail={}", + jobId, + renewalFailure.getMessage()); + } else { + log.debug( + "Legacy RAG job heartbeat remains degraded within the active lease: job={}", + jobId); + } + return true; + } + } + + @Override + public boolean isOwnershipLost() { + return ownershipLost.get(); + } + + @Override + public OffsetDateTime validAfter() { + return OffsetDateTime.now().minusSeconds(leaseSeconds); + } + + @Override + public boolean confirmOwnership() { + return renew(); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + ScheduledFuture scheduled = heartbeat; + if (scheduled != null) { + scheduled.cancel(false); + } + } + } + + @PreDestroy + void close() { + executor.shutdownNow(); + } + + private static final class HeartbeatThreadFactory implements ThreadFactory { + private final AtomicInteger sequence = new AtomicInteger(); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread( + runnable, + "legacy-rag-job-heartbeat-" + sequence.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryService.java new file mode 100644 index 00000000..19dc22d0 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryService.java @@ -0,0 +1,148 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.service.JobService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Terminalizes legacy incremental RAG jobs whose database lease expired. The + * RAG lock is deliberately not released here: the legacy remote mutation has + * no fencing token, so its maintained lease must expire before a retry starts. + */ +@Service +public class LegacyRagJobRecoveryService { + private static final Logger log = LoggerFactory.getLogger( + LegacyRagJobRecoveryService.class); + + private final JobService jobService; + private final RagIndexTrackingService trackingService; + private final long staleAfterSeconds; + private final int batchSize; + private final AtomicBoolean recoveryDegraded = new AtomicBoolean(false); + + public LegacyRagJobRecoveryService( + JobService jobService, + RagIndexTrackingService trackingService, + @Value("${codecrow.rag.legacy-job.lease-seconds:120}") + long staleAfterSeconds, + @Value("${codecrow.rag.legacy-job.recovery-batch-size:100}") + int batchSize) { + this.jobService = jobService; + this.trackingService = trackingService; + this.staleAfterSeconds = Math.max(30, staleAfterSeconds); + this.batchSize = Math.max(1, batchSize); + } + + @Scheduled( + fixedDelayString = "${codecrow.rag.legacy-job.recovery-interval-ms:30000}", + initialDelayString = "${codecrow.rag.legacy-job.recovery-initial-delay-ms:30000}") + public void failAbandonedJobs() { + OffsetDateTime cutoff = OffsetDateTime.now().minusSeconds(staleAfterSeconds); + String diagnostic = "Legacy RAG update producer stopped heartbeating for " + + staleAfterSeconds + + " seconds; the last completed checkpoint was preserved"; + boolean passDegraded = false; + + try { + for (var coordinates : jobService.findAbandonedLegacyRagJobs( + cutoff, batchSize)) { + try { + if (!jobService.failAbandonedLegacyRagJob( + coordinates.getJobId(), cutoff, diagnostic)) { + continue; + } + log.warn( + "Failed abandoned legacy RAG job {} for project={}, branch={}, commit={}", + coordinates.getJobId(), + coordinates.getProjectId(), + coordinates.getBranchName(), + coordinates.getCommitHash()); + recordDurableFailure(coordinates.getJobId(), diagnostic); + if (!repairProjectStatus(coordinates, diagnostic)) { + passDegraded = true; + } + } catch (Exception failure) { + passDegraded = true; + reportDegraded( + "Could not terminalize abandoned legacy RAG job " + + coordinates.getJobId(), + failure); + } + } + + // If a process stopped after the job CAS but before repairing its + // project-level status, retry that projection independently. + for (var coordinates : jobService.findFailedLegacyRagJobsWithActiveStatus( + batchSize)) { + String priorDiagnostic = coordinates.getErrorMessage(); + if (!repairProjectStatus( + coordinates, + priorDiagnostic != null && !priorDiagnostic.isBlank() + ? priorDiagnostic + : diagnostic)) { + passDegraded = true; + } + } + } catch (Exception selectionFailure) { + passDegraded = true; + reportDegraded( + "Could not scan legacy RAG job recovery state", + selectionFailure); + } + + if (!passDegraded && recoveryDegraded.compareAndSet(true, false)) { + log.info("Legacy RAG job recovery scan recovered"); + } + } + + private void recordDurableFailure(long jobId, String diagnostic) { + try { + jobService.findById(jobId).ifPresent(job -> + jobService.recordExternallyFailedJob( + job, "rag_recovery", "Job failed: " + diagnostic)); + } catch (Exception failure) { + // The guarded CAS already made the job terminal. Projection repair + // is independent of an optional UI log/notification. + log.debug("Could not append optional recovery diagnostics to legacy RAG job {}: {}", + jobId, failure.getMessage()); + } + } + + private boolean repairProjectStatus( + JobRepository.LegacyRagJobRecoveryCoordinates coordinates, + String diagnostic) { + try { + // This is an incremental job, so retain the last usable checkpoint + // even if an older producer happened to label the live state INDEXING. + trackingService.recoverAbandonedIncrementalUpdate( + coordinates.getProjectId(), coordinates.getJobId(), diagnostic); + return true; + } catch (Exception failure) { + reportDegraded( + "Could not repair RAG status after legacy producer abandonment: project=" + + coordinates.getProjectId() + + ", job=" + coordinates.getJobId(), + failure); + return false; + } + } + + private void reportDegraded(String operation, Exception failure) { + String detail = failure.getMessage() != null + ? failure.getMessage() + : failure.getClass().getSimpleName(); + if (recoveryDegraded.compareAndSet(false, true)) { + log.warn("Legacy RAG job recovery degraded: {}: {}", operation, detail); + } else { + log.debug("Legacy RAG job recovery remains degraded: {}: {}", operation, detail); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionService.java new file mode 100644 index 00000000..ce5028fa --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionService.java @@ -0,0 +1,115 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.persistence.repository.analysis.RagIndexStatusRepository; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.interceptor.TransactionAspectSupport; + +import java.time.OffsetDateTime; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** Commits one legacy RAG job and all of its Java checkpoints atomically. */ +@Service +public class LegacyRagUpdateCompletionService { + private final JobRepository jobRepository; + private final RagIndexTrackingService trackingService; + private final RagIndexStatusRepository indexStatusRepository; + private final RagBranchIndexRepository branchIndexRepository; + + public LegacyRagUpdateCompletionService( + JobRepository jobRepository, + RagIndexTrackingService trackingService, + RagIndexStatusRepository indexStatusRepository, + RagBranchIndexRepository branchIndexRepository) { + this.jobRepository = jobRepository; + this.trackingService = trackingService; + this.indexStatusRepository = indexStatusRepository; + this.branchIndexRepository = branchIndexRepository; + } + + /** + * @return {@code false} when recovery or lease expiry won the job row first + */ + @Transactional + public boolean complete( + Project project, + String branchName, + String commitHash, + long jobId, + OffsetDateTime validAfter, + boolean tracksProjectStatus, + int addedFiles, + int deletedFiles, + Integer chunkCount, + Set deletedPaths) { + OffsetDateTime completedAt = OffsetDateTime.now(); + if (jobRepository.completeOwnedLegacyRagJob( + jobId, validAfter, completedAt) != 1) { + return false; + } + + if (tracksProjectStatus) { + var status = indexStatusRepository + .findByProjectIdForUpdate(project.getId()) + .orElseThrow(() -> new LegacyRagCompletionConflictException( + "RAG index status disappeared before legacy completion")); + if (status.getStatus() != RagIndexingStatus.UPDATING + || !Objects.equals(status.getActiveJobId(), jobId)) { + throw new LegacyRagCompletionConflictException( + "Legacy RAG job no longer owns the project index status"); + } + trackingService.markUpdatingCompleted( + project, + branchName, + commitHash, + addedFiles, + deletedFiles, + chunkCount, + jobId); + } + try { + updateBranchCheckpoint(project, branchName, commitHash, deletedPaths); + } catch (RuntimeException checkpointFailure) { + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + throw checkpointFailure; + } + return true; + } + + private void updateBranchCheckpoint( + Project project, + String branchName, + String commitHash, + Set deletedPaths) { + RagBranchIndex branchIndex = branchIndexRepository + .findByProjectIdAndBranchNameForUpdate( + project.getId(), branchName) + .orElseGet(() -> new RagBranchIndex(project, branchName)); + branchIndex.setCommitHash(commitHash); + branchIndex.setUpdatedAt(OffsetDateTime.now()); + if (deletedPaths != null && !deletedPaths.isEmpty()) { + Set accumulated = branchIndex.getDeletedFiles() != null + ? new HashSet<>(branchIndex.getDeletedFiles()) + : new HashSet<>(); + accumulated.addAll(deletedPaths); + branchIndex.setDeletedFiles(accumulated); + } + branchIndexRepository.save(branchIndex); + } + + /** Signals a fenced projection conflict; the producer must not fail newer state. */ + public static final class LegacyRagCompletionConflictException + extends RuntimeException { + public LegacyRagCompletionConflictException(String message) { + super(message); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java index 9544ffe4..3eda8437 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java @@ -1,5 +1,10 @@ package org.rostilos.codecrow.ragengine.branch; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; @@ -19,11 +24,13 @@ */ @Service public class RagBranchOperatorAliasReconciliationService { + private static final int MAX_GENERATION_REFRESHES = 3; private static final Logger log = LoggerFactory.getLogger( RagBranchOperatorAliasReconciliationService.class); private final RagBranchIndexRepository branchIndexRepository; private final RagPipelineClient pipelineClient; + private ReconciliationState reconciliationState = ReconciliationState.HEALTHY; public RagBranchOperatorAliasReconciliationService( RagBranchIndexRepository branchIndexRepository, @@ -35,24 +42,141 @@ public RagBranchOperatorAliasReconciliationService( @Scheduled( fixedDelayString = "${codecrow.rag.operator-alias.reconcile-interval-ms:300000}", initialDelayString = "${codecrow.rag.operator-alias.reconcile-initial-delay-ms:15000}") - public void reconcileActiveGenerationAliases() { - for (var candidate : branchIndexRepository.findOperatorAliasCandidates()) { + public synchronized void reconcileActiveGenerationAliases() { + List candidates; + try { + candidates = branchIndexRepository.findOperatorAliasCandidates(); + } catch (RuntimeException repositoryFailure) { + recordDegradedRun("Could not read active RAG aliases for reconciliation; " + + "retrying next run: " + repositoryFailure.getMessage()); + return; + } + + int candidateRejections = 0; + for (var candidate : candidates) { try { - pipelineClient.publishGenerationAliases( - candidate.getWorkspaceName(), - candidate.getProjectNamespace(), + publishCurrentGeneration(candidate); + } catch (RagPipelineClient.RagApiException apiFailure) { + if (apiFailure.isServiceFailure()) { + recordDegradedRun("RAG alias reconciliation stopped after a service failure; " + + "remaining candidates will retry next run: " + apiFailure.getMessage()); + return; + } + candidateRejections++; + log.debug( + "RAG alias candidate was rejected for project={} branch={}; continuing: {}", + candidate.getProjectId(), candidate.getBranchName(), - candidate.getRevision(), - candidate.getCollectionName(), - true, - candidate.getIndexKind() == RagBranchIndexKind.PRIMARY); - } catch (Exception failure) { - log.warn( - "Could not reconcile readable RAG alias for project={} branch={}: {}", + apiFailure.getMessage()); + } catch (IOException transportFailure) { + // One unavailable RAG service would make every remaining call + // fail and generate the same warning. Stop this bounded run; + // the scheduler's fixed delay is the retry backoff. + recordDegradedRun("RAG alias reconciliation stopped after a transport failure; " + + "remaining candidates will retry next run: " + transportFailure.getMessage()); + return; + } catch (CandidateRegistryReadException repositoryFailure) { + recordDegradedRun("RAG alias reconciliation stopped after a registry read failure; " + + "remaining candidates will retry next run: " + repositoryFailure.getMessage()); + return; + } catch (RuntimeException candidateFailure) { + candidateRejections++; + log.debug( + "RAG alias candidate failed for project={} branch={}; continuing: {}", candidate.getProjectId(), candidate.getBranchName(), - failure.getMessage()); + candidateFailure.getMessage()); + } + } + if (candidateRejections > 0) { + recordDegradedRun("RAG alias reconciliation completed with " + candidateRejections + + " rejected candidate(s); they will retry next run"); + } else { + recordHealthyRun(); + } + } + + private void publishCurrentGeneration( + RagBranchIndexRepository.OperatorAliasCandidate scheduledCandidate) throws IOException { + RagBranchIndexRepository.OperatorAliasCandidate current = refreshCandidate(scheduledCandidate) + .orElse(null); + if (current == null) { + return; + } + + for (int refresh = 0; refresh < MAX_GENERATION_REFRESHES; refresh++) { + publish(current); + Optional afterPublication = + refreshCandidate(current); + if (afterPublication.isEmpty()) { + return; + } + if (sameGeneration(current, afterPublication.get())) { + return; } + log.info("RAG active generation changed during alias reconciliation for " + + "project={} branch={}; publishing the current generation instead", + current.getProjectId(), current.getBranchName()); + current = afterPublication.get(); + } + + throw new IllegalStateException( + "RAG active generation kept changing during alias reconciliation for project=" + + current.getProjectId() + " branch=" + current.getBranchName()); + } + + private Optional refreshCandidate( + RagBranchIndexRepository.OperatorAliasCandidate candidate) { + try { + return branchIndexRepository.findOperatorAliasCandidateById(candidate.getBranchIndexId()); + } catch (RuntimeException repositoryFailure) { + throw new CandidateRegistryReadException(repositoryFailure); + } + } + + private void publish(RagBranchIndexRepository.OperatorAliasCandidate candidate) throws IOException { + pipelineClient.publishGenerationAliases( + candidate.getWorkspaceName(), + candidate.getProjectNamespace(), + candidate.getBranchName(), + candidate.getRevision(), + candidate.getCollectionName(), + candidate.getManifestDigest(), + true, + candidate.getIndexKind() == RagBranchIndexKind.PRIMARY); + } + + private static boolean sameGeneration( + RagBranchIndexRepository.OperatorAliasCandidate first, + RagBranchIndexRepository.OperatorAliasCandidate second) { + if (first.getGenerationId() != null || second.getGenerationId() != null) { + return Objects.equals(first.getGenerationId(), second.getGenerationId()); + } + return Objects.equals(first.getCollectionName(), second.getCollectionName()) + && Objects.equals(first.getManifestDigest(), second.getManifestDigest()); + } + + private void recordDegradedRun(String message) { + if (reconciliationState == ReconciliationState.HEALTHY) { + reconciliationState = ReconciliationState.DEGRADED; + log.warn(message); + } else { + log.info(message); + } + } + + private void recordHealthyRun() { + if (reconciliationState == ReconciliationState.DEGRADED) { + reconciliationState = ReconciliationState.HEALTHY; + log.info("RAG alias reconciliation recovered"); + } + } + + private enum ReconciliationState { HEALTHY, DEGRADED } + + private static final class CandidateRegistryReadException extends RuntimeException { + private CandidateRegistryReadException(RuntimeException cause) { + super(cause.getMessage(), cause); } } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java new file mode 100644 index 00000000..220e8552 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java @@ -0,0 +1,84 @@ +package org.rostilos.codecrow.ragengine.branch; + +import jakarta.annotation.PreDestroy; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** Keeps a live exact-generation operation owned while remote work is running. */ +@Service +public class RagIndexOperationHeartbeatService { + private static final long HEARTBEAT_INTERVAL_SECONDS = 15; + private static final int HEARTBEAT_THREADS = 4; + + private final RagBranchIndexRegistryService registryService; + private final ScheduledExecutorService executor; + private final long heartbeatIntervalSeconds; + + @Autowired + public RagIndexOperationHeartbeatService( + RagBranchIndexRegistryService registryService) { + this(registryService, + Executors.newScheduledThreadPool( + HEARTBEAT_THREADS, new HeartbeatThreadFactory()), + HEARTBEAT_INTERVAL_SECONDS); + } + + RagIndexOperationHeartbeatService( + RagBranchIndexRegistryService registryService, + ScheduledExecutorService executor, + long heartbeatIntervalSeconds) { + this.registryService = registryService; + this.executor = executor; + this.heartbeatIntervalSeconds = heartbeatIntervalSeconds; + } + + public HeartbeatScope start(long operationId) { + ScheduledFuture heartbeat = executor.scheduleAtFixedRate( + () -> heartbeat(operationId), + heartbeatIntervalSeconds, + heartbeatIntervalSeconds, + TimeUnit.SECONDS); + return () -> heartbeat.cancel(false); + } + + private void heartbeat(long operationId) { + try { + registryService.heartbeatBuild(operationId); + } catch (Exception ignored) { + // A later heartbeat may still succeed. If the producer stops, + // durable operation recovery owns the terminal transition. + } + } + + @PreDestroy + void close() { + executor.shutdownNow(); + } + + @FunctionalInterface + public interface HeartbeatScope extends AutoCloseable { + @Override + void close(); + } + + private static final class HeartbeatThreadFactory implements ThreadFactory { + private final AtomicInteger sequence = new AtomicInteger(); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread( + runnable, + "rag-generation-heartbeat-" + sequence.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java index 6d307278..4087421e 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java @@ -4,8 +4,8 @@ import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; import org.rostilos.codecrow.core.model.job.Job; -import org.rostilos.codecrow.core.model.rag.RagIndexOperation; import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagIndexOperationRepository; import org.rostilos.codecrow.core.service.JobService; import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; @@ -16,6 +16,7 @@ import org.springframework.stereotype.Service; import java.time.OffsetDateTime; +import java.util.concurrent.atomic.AtomicBoolean; /** * Terminates registry operations whose producer disappeared before publishing @@ -35,6 +36,7 @@ public class RagIndexOperationRecoveryService { private final RagOperationsService ragOperationsService; private final AnalysisLockService lockService; private final long staleAfterMinutes; + private final AtomicBoolean recoveryDegraded = new AtomicBoolean(false); public RagIndexOperationRecoveryService( RagBranchIndexRegistryService registryService, @@ -59,112 +61,233 @@ public RagIndexOperationRecoveryService( initialDelayString = "${codecrow.rag.generation.recovery-initial-delay-ms:60000}") public void failAbandonedOperations() { OffsetDateTime cutoff = OffsetDateTime.now().minusMinutes(staleAfterMinutes); - for (var operation : registryService.findRecoverableOperations(cutoff)) { - String diagnostic = "Exact RAG generation producer stopped heartbeating for " - + staleAfterMinutes + " minutes; the previous active generation was preserved"; - try { - if (!registryService.failIfAbandoned(operation.getId(), cutoff, diagnostic)) { + boolean passDegraded = false; + try { + for (var operation : registryService.findRecoverableOperations(cutoff)) { + String diagnostic = "Exact RAG generation producer stopped heartbeating for " + + staleAfterMinutes + " minutes; the previous active generation was preserved"; + try { + if (!registryService.failIfAbandoned( + operation.getOperationId(), cutoff, diagnostic)) { + continue; + } + log.warn("Failed abandoned RAG generation operation {} for branch {}", + operation.getOperationId(), operation.getBranchName()); + } catch (Exception failure) { + passDegraded = true; + reportDegraded( + "Could not terminalize abandoned RAG generation operation " + + operation.getOperationId(), + failure); continue; } - log.warn("Failed abandoned RAG generation operation {} for branch {}", - operation.getId(), operation.getBranchName()); - } catch (Exception failure) { - log.error("Could not terminalize abandoned RAG generation operation {}", - operation.getId(), failure); - continue; + + if (!recoverProjections(operation, diagnostic)) { + passDegraded = true; + } } - recoverProjections(operation, diagnostic); + for (var operation : registryService.findFailedOperationsWithActiveProjections()) { + String diagnostic = operation.getErrorMessage() != null + && !operation.getErrorMessage().isBlank() + ? operation.getErrorMessage() + : "RAG generation failed before its durable projections were terminalized"; + if (!recoverProjections(operation, diagnostic)) { + passDegraded = true; + } + } + + for (var operation : registryService.findSucceededOperationsWithActiveProjections()) { + if (!recoverPublishedProjections(operation)) { + passDegraded = true; + } + } + } catch (Exception selectionFailure) { + passDegraded = true; + reportDegraded( + "Could not scan exact RAG operation recovery state", + selectionFailure); } - for (var operation : registryService.findFailedOperationsWithActiveProjections()) { - String diagnostic = operation.getErrorMessage() != null - && !operation.getErrorMessage().isBlank() - ? operation.getErrorMessage() - : "RAG generation failed before its durable projections were terminalized"; - recoverProjections(operation, diagnostic); + if (!passDegraded && recoveryDegraded.compareAndSet(true, false)) { + log.info("Exact RAG operation recovery scan recovered"); } } - private void recoverProjections( - RagIndexOperation operation, + private boolean recoverPublishedProjections( + RagIndexOperationRepository.SucceededOperationProjection operation) { + boolean recovered = true; + Long projectId = operation.getProjectId(); + String branchName = operation.getBranchName(); + Long jobId = operation.getJobId(); + try { + var project = projectRepository.findByIdWithFullDetails(projectId) + .orElse(null); + if (Boolean.TRUE.equals(operation.getActiveGeneration()) + && project != null + && branchName.equals(ragOperationsService.getBaseBranch(project))) { + trackingService.reconcilePublishedGeneration( + project, + branchName, + operation.getToRevision(), + operation.getFileCount(), + operation.getChunkCount(), + jobId); + } + } catch (Exception failure) { + recovered = false; + reportDegraded( + "Could not reconcile published RAG project status: project=" + + projectId + ", branch=" + branchName, + failure); + } + + if (jobId != null) { + try { + Job job = jobService.findById(jobId).orElse(null); + if (job != null && !job.isTerminal()) { + jobService.completeJob(job); + } + } catch (Exception failure) { + recovered = false; + reportDegraded( + "Could not complete durable job " + jobId + + " for published RAG generation", + failure); + } + } + + return releasePublishedLock(operation) && recovered; + } + + private boolean releasePublishedLock( + RagIndexOperationRepository.SucceededOperationProjection operation) { + String lockKey = operation.getAnalysisLockKey(); + if (lockKey == null || lockKey.isBlank()) { + return true; + } + try { + lockService.releaseLock(lockKey); + log.info( + "Released completed RAG indexing lock during projection recovery: " + + "project={}, branch={}, commit={}", + operation.getProjectId(), operation.getBranchName(), + operation.getToRevision()); + return true; + } catch (Exception failure) { + reportDegraded( + "Could not release completed RAG indexing lock: project=" + + operation.getProjectId() + ", branch=" + + operation.getBranchName(), + failure); + return false; + } + } + + private boolean recoverProjections( + RagIndexOperationRepository.RecoveryOperationProjection operation, String diagnostic) { - failDurableJob(operation.getJobId(), diagnostic); - terminalizePrimaryStatus(operation, diagnostic); - releaseAbandonedLock(operation); + boolean jobRecovered = failDurableJob(operation.getJobId(), diagnostic); + boolean statusRecovered = terminalizePrimaryStatus(operation, diagnostic); + boolean lockRecovered = releaseAbandonedLock(operation); + return jobRecovered && statusRecovered && lockRecovered; } - private void failDurableJob(Long jobId, String diagnostic) { + private boolean failDurableJob(Long jobId, String diagnostic) { if (jobId == null) { - return; + return true; } try { Job job = jobService.findById(jobId).orElse(null); if (job != null && !job.isTerminal()) { jobService.failJob(job, diagnostic); } + return true; } catch (Exception failure) { - log.error("Could not fail durable RAG job {} after producer abandonment", - jobId, failure); + reportDegraded( + "Could not fail durable RAG job " + jobId + + " after producer abandonment", + failure); + return false; } } - private void terminalizePrimaryStatus(RagIndexOperation operation, String diagnostic) { - Long projectId = operation.getProject().getId(); + private boolean terminalizePrimaryStatus( + RagIndexOperationRepository.RecoveryOperationProjection operation, + String diagnostic) { + Long projectId = operation.getProjectId(); String branchName = operation.getBranchName(); + Long jobId = operation.getJobId(); try { var project = projectRepository.findByIdWithFullDetails(projectId) .orElse(null); if (project == null || !branchName.equals(ragOperationsService.getBaseBranch(project))) { - return; + return true; } var status = trackingService.getIndexStatus(project).orElse(null); if (status == null) { - return; + return true; } - if (status.getActiveJobId() != null - && !status.getActiveJobId().equals(operation.getJobId())) { + if (status.getActiveJobId() == null + || !status.getActiveJobId().equals(jobId)) { log.info( - "Preserving RAG status owned by newer job {} while recovering abandoned job {}: " + "Preserving RAG status owned by job {} while recovering abandoned job {}: " + "project={}, branch={}", - status.getActiveJobId(), operation.getJobId(), projectId, branchName); - return; + status.getActiveJobId(), jobId, projectId, branchName); + return true; } if (status.getStatus() == RagIndexingStatus.INDEXING) { - trackingService.markIndexingFailed(project, diagnostic, operation.getJobId()); + trackingService.markIndexingFailed(project, diagnostic, jobId); } else if (status.getStatus() == RagIndexingStatus.UPDATING) { trackingService.markIncrementalUpdateFailed( - project, diagnostic, operation.getJobId()); + project, diagnostic, jobId); } + return true; } catch (Exception failure) { - log.error( + reportDegraded( "Could not terminalize RAG project status after producer abandonment: " - + "project={}, branch={}", - projectId, branchName, failure); + + "project=" + projectId + ", branch=" + branchName, + failure); + return false; } } - private void releaseAbandonedLock(RagIndexOperation operation) { - Long projectId = operation.getProject().getId(); + private boolean releaseAbandonedLock( + RagIndexOperationRepository.RecoveryOperationProjection operation) { + Long projectId = operation.getProjectId(); String branchName = operation.getBranchName(); String lockKey = operation.getAnalysisLockKey(); if (lockKey == null || lockKey.isBlank()) { - log.warn( + log.info( "Cannot release abandoned RAG lock without its exact owner key; " + "leaving it to expire: project={}, branch={}, commit={}", projectId, branchName, operation.getToRevision()); - return; + return true; } try { lockService.releaseLock(lockKey); - log.warn( + log.info( "Released abandoned RAG indexing lock for project={}, branch={}, commit={}", projectId, branchName, operation.getToRevision()); + return true; } catch (Exception failure) { - log.error( - "Could not release abandoned RAG indexing lock: project={}, branch={}", - projectId, branchName, failure); + reportDegraded( + "Could not release abandoned RAG indexing lock: project=" + + projectId + ", branch=" + branchName, + failure); + return false; + } + } + + private void reportDegraded(String operation, Exception failure) { + String detail = failure.getMessage() != null + ? failure.getMessage() + : failure.getClass().getSimpleName(); + if (recoveryDegraded.compareAndSet(false, true)) { + log.warn("Exact RAG operation recovery degraded: {}: {}", operation, detail); + } else { + log.debug("Exact RAG operation recovery remains degraded: {}: {}", operation, detail); } } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java index 2aa28b55..fe060f86 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java @@ -1,7 +1,6 @@ package org.rostilos.codecrow.ragengine.branch; -import org.rostilos.codecrow.core.model.rag.RagBranchIndex; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; @@ -9,9 +8,11 @@ import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.time.OffsetDateTime; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; /** Removes only expired PR-target generations explicitly classified transient. */ @Service @@ -22,6 +23,8 @@ public class RagTransientBranchIndexCleanupService { private final RagBranchIndexRepository branchRepository; private final RagBranchIndexGenerationRepository generationRepository; private final RagPipelineClient pipelineClient; + private final String cleanupOwner = UUID.randomUUID().toString(); + private CleanupState cleanupState = CleanupState.HEALTHY; public RagTransientBranchIndexCleanupService( RagBranchIndexRepository branchRepository, @@ -35,14 +38,21 @@ public RagTransientBranchIndexCleanupService( @Scheduled( fixedDelayString = "${codecrow.rag.transient.cleanup-interval-ms:3600000}", initialDelayString = "${codecrow.rag.transient.cleanup-initial-delay-ms:300000}") - @Transactional - public void cleanupExpired() { + public synchronized void cleanupExpired() { OffsetDateTime now = OffsetDateTime.now(); - for (RagBranchIndex index : branchRepository.findByIndexKind( - RagBranchIndexKind.TRANSIENT)) { - var project = index.getProject(); - var config = project.getConfiguration() != null - ? project.getConfiguration().ragConfig() + List candidates; + try { + candidates = branchRepository.findTransientCleanupCandidates(); + } catch (RuntimeException repositoryFailure) { + recordDegradedRun("Could not read transient RAG cleanup candidates; retrying next run: " + + repositoryFailure.getMessage()); + return; + } + + int rejectedCandidates = 0; + for (var index : candidates) { + var config = index.getProjectConfiguration() != null + ? index.getProjectConfiguration().ragConfig() : null; int retentionDays = config != null ? config.getEffectiveBranchRetentionDays() @@ -54,24 +64,178 @@ public void cleanupExpired() { continue; } + OffsetDateTime cutoff = now.minusDays(retentionDays); + int claimed; + try { + claimed = branchRepository.claimExpiredTransientForDeletion( + index.getBranchIndexId(), cutoff, now.minusMinutes(10), + cleanupOwner, now); + } catch (RuntimeException repositoryFailure) { + recordDegradedRun("Transient RAG cleanup could not claim project=" + + index.getProjectId() + " branch=" + index.getBranchName() + + "; retrying next run: " + repositoryFailure.getMessage()); + return; + } + if (claimed == 0) { + // Access or a concurrent build won the atomic registry race. + continue; + } + boolean removed = true; - for (var generation : generationRepository - .findByBranchIndexIdOrderByCreatedAtDesc(index.getId())) { + boolean physicalDeletionStarted = false; + boolean deletionOutcomeUncertain = false; + List generations; + try { + generations = generationRepository.findCleanupCandidatesByBranchIndexId( + index.getBranchIndexId()); + } catch (RuntimeException repositoryFailure) { + cancelClaim(index.getBranchIndexId(), cleanupOwner); + recordDegradedRun("Transient RAG cleanup stopped after a registry read failure; " + + "remaining candidates will retry next run: " + repositoryFailure.getMessage()); + return; + } + // A readable active generation is deleted last. If an earlier + // target-specific rejection occurs, releasing the claim is safe as + // long as no physical target was removed; after a partial cleanup + // the durable claim stays in place until the next idempotent retry. + generations = generations.stream() + .sorted(Comparator.comparing(generation -> + generation.getStatus() == RagBranchIndexGenerationStatus.ACTIVE)) + .toList(); + for (var generation : generations) { + if (generation.getStatus() == RagBranchIndexGenerationStatus.ACTIVE + && !removed) { + // Keep the currently readable target intact if an older + // target could not be reconciled. If an older target was + // already removed, the claim below remains durable and + // prevents reads of that partial generation set. + break; + } + RagPipelineClient.BranchDeletionOutcome outcome; try { - removed &= pipelineClient.deleteBranch( - project.getWorkspace().getName(), project.getNamespace(), - index.getBranchName(), generation.getCollectionName()); - } catch (Exception failure) { + if (branchRepository.heartbeatTransientDeletionClaim( + index.getBranchIndexId(), cleanupOwner, OffsetDateTime.now()) == 0) { + recordDegradedRun("Transient RAG cleanup lost its durable claim for project=" + + index.getProjectId() + " branch=" + index.getBranchName() + + "; remaining targets will retry next run"); + return; + } + outcome = pipelineClient.deleteBranchWithOutcome( + index.getWorkspaceName(), index.getProjectNamespace(), + index.getBranchName(), generation.getCollectionName(), + generation.getRevision(), generation.getManifestDigest()); + } catch (RuntimeException unexpectedFailure) { + // The request may have reached RAG before the client threw. + // Keep the claim until an idempotent retry can reconcile it. + recordDegradedRun("Transient RAG cleanup stopped at target=" + + generation.getCollectionName() + + " after an unexpected client failure; remaining generations " + + "will retry next run: " + unexpectedFailure.getMessage()); + return; + } + if (outcome.successful()) { + physicalDeletionStarted = true; + } + if (!outcome.successful()) { + if (isRagDisabled(outcome)) { + // Deployment-level disablement is intentional. Retain + // the registry and avoid an hourly degraded signal. + cancelClaim(index.getBranchIndexId(), cleanupOwner); + recordHealthyRun(); + return; + } removed = false; - log.warn("Failed to clean transient RAG generation {}: {}", - generation.getId(), failure.getMessage()); + deletionOutcomeUncertain |= outcome.failure() + == RagPipelineClient.BranchDeletionFailure.TRANSPORT; + log.debug("Transient RAG generation cleanup rejected generation={} target={}: " + + "status={} detail={}", + generation.getGenerationId(), outcome.targetLabel(), + outcome.statusCode() != null ? outcome.statusCode() : outcome.failure(), + outcome.detail()); + if (outcome.shouldStopRemainingTargets()) { + if (!physicalDeletionStarted && !deletionOutcomeUncertain) { + cancelClaim(index.getBranchIndexId(), cleanupOwner); + } + recordDegradedRun("Transient RAG cleanup stopped at target=" + + outcome.targetLabel() + " after " + outcome.failure() + + " failure; remaining generations will retry next run: " + + outcome.detail()); + return; + } } } if (removed) { - branchRepository.delete(index); + int deleted; + try { + deleted = branchRepository.deleteClaimedTransientById( + index.getBranchIndexId(), cleanupOwner); + } catch (RuntimeException repositoryFailure) { + recordDegradedRun("Transient RAG cleanup could not finalize registry deletion " + + "for project=" + index.getProjectId() + " branch=" + + index.getBranchName() + "; retrying next run: " + + repositoryFailure.getMessage()); + return; + } + if (deleted == 0) { + // A claimed row cannot be made readable or rebuilt. A zero + // delete means registry finalization failed independently; + // delete leaves the durable token in place so destroyed data + // is never exposed. + recordDegradedRun("Transient RAG cleanup deleted physical targets but could not " + + "finalize its registry claim for project=" + index.getProjectId() + + " branch=" + index.getBranchName() + "; retrying next run"); + return; + } log.info("Removed expired transient RAG branch index project={}, branch={}", - project.getId(), index.getBranchName()); + index.getProjectId(), index.getBranchName()); + } else { + if (!physicalDeletionStarted && !deletionOutcomeUncertain) { + cancelClaim(index.getBranchIndexId(), cleanupOwner); + } + rejectedCandidates++; } } + if (rejectedCandidates > 0) { + recordDegradedRun("Transient RAG cleanup completed with " + rejectedCandidates + + " rejected candidate(s); they will retry next run"); + } else { + recordHealthyRun(); + } + } + + private void cancelClaim(long branchIndexId, String claimToken) { + try { + branchRepository.cancelTransientDeletion(branchIndexId, claimToken); + } catch (RuntimeException cancellationFailure) { + // Keeping the token is fail-safe: no reader will receive a target + // whose cleanup outcome cannot be reconciled. + log.debug("Could not release transient cleanup claim branchIndex={}: {}", + branchIndexId, cancellationFailure.getMessage()); + } } + + private static boolean isRagDisabled( + RagPipelineClient.BranchDeletionOutcome outcome) { + return outcome.statusCode() == null + && outcome.failure() == RagPipelineClient.BranchDeletionFailure.TARGET + && "RAG disabled".equals(outcome.detail()); + } + + private void recordDegradedRun(String message) { + if (cleanupState == CleanupState.HEALTHY) { + cleanupState = CleanupState.DEGRADED; + log.warn(message); + } else { + log.info(message); + } + } + + private void recordHealthyRun() { + if (cleanupState == CleanupState.DEGRADED) { + cleanupState = CleanupState.HEALTHY; + log.info("Transient RAG cleanup recovered"); + } + } + + private enum CleanupState { HEALTHY, DEGRADED } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java index c6c0473d..76b8c3b1 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java @@ -28,6 +28,28 @@ public class RagPipelineClient { private final boolean ragEnabled; private final String serviceSecret; + /** HTTP response failure with enough structure for bounded retry decisions. */ + public static final class RagApiException extends IOException { + private final int statusCode; + + public RagApiException(int statusCode, String detail) { + super("RAG API error: " + statusCode + " — " + detail); + this.statusCode = statusCode; + } + + public int getStatusCode() { + return statusCode; + } + + public boolean isServiceFailure() { + return statusCode == 401 + || statusCode == 403 + || statusCode == 408 + || statusCode == 429 + || statusCode >= 500; + } + } + public RagPipelineClient( @Value("${codecrow.rag.api.url:http://rag-pipeline:8001}") String ragApiUrl, @Value("${codecrow.rag.api.enabled:true}") boolean ragEnabled, @@ -182,6 +204,34 @@ public Map indexRepository( boolean publishBranchAlias, boolean publishLegacyProjectAlias, Consumer> progressConsumer + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, collectionTarget, + publishBranchAlias, publishLegacyProjectAlias, false, null, + progressConsumer); + } + + /** + * Streaming exact-generation indexing with an explicit shared-snapshot + * ownership handoff. The RAG service atomically moves the snapshot before + * it emits admission, so a lost stream cannot expose its active worker to + * caller-side deletion of the original path. + */ + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias, + boolean transferRepositoryOwnership, + Runnable ownershipAdmissionConsumer, + Consumer> progressConsumer ) throws IOException { if (!ragEnabled) { log.debug("RAG indexing disabled, skipping repository indexing"); @@ -203,6 +253,9 @@ public Map indexRepository( if (publishLegacyProjectAlias) { payload.put("publish_legacy_project_alias", true); } + if (transferRepositoryOwnership) { + payload.put("transfer_repo_ownership", true); + } payload.put("source_tree_sha256", RepositorySourceTreeIdentity.sha256(Path.of(repoPath))); if (includePatterns != null && !includePatterns.isEmpty()) { payload.put("include_patterns", includePatterns); @@ -211,7 +264,8 @@ public Map indexRepository( payload.put("exclude_patterns", excludePatterns); } return postLongRunningSse( - ragApiUrl + "/index/repository/stream", payload, progressConsumer); + ragApiUrl + "/index/repository/stream", payload, + ownershipAdmissionConsumer, progressConsumer); } public Map updateFiles( @@ -374,6 +428,7 @@ public void publishGenerationAliases( String branch, String commit, String collectionTarget, + String generationManifestSha256, boolean publishBranchAlias, boolean publishLegacyProjectAlias) throws IOException { if (!ragEnabled || !publishBranchAlias) { @@ -385,6 +440,7 @@ public void publishGenerationAliases( payload.put("branch", branch); payload.put("commit", commit); payload.put("collection_target", collectionTarget); + payload.put("generation_manifest_sha256", generationManifestSha256); payload.put("publish_branch_alias", true); if (publishLegacyProjectAlias) { payload.put("publish_legacy_project_alias", true); @@ -515,31 +571,122 @@ public void deleteIndex(String workspace, String project, String branch) throws * @return true if points were deleted or already absent, false on error */ public boolean deletePrFiles(String workspace, String project, int prNumber) { + return deletePrFiles(workspace, project, prNumber, null); + } + + /** + * Deletes PR overlay points from one immutable physical generation. + * Passing no target retains the legacy project-alias behavior for projects + * that predate the generation registry. + */ + public boolean deletePrFiles( + String workspace, + String project, + int prNumber, + String collectionTarget) { + PrFilesDeletionOutcome outcome = deletePrFilesWithOutcome( + workspace, project, prNumber, collectionTarget); + if (!outcome.successful()) { + log.warn("Failed to delete PR #{} files from {}/{} target={}: status={} detail={}", + prNumber, workspace, project, outcome.targetLabel(), + outcome.statusCode() != null ? outcome.statusCode() : "transport", + outcome.detail()); + } + return outcome.successful(); + } + + /** + * Performs one PR-overlay deletion without logging. Multi-generation callers + * use the structured result to emit one contextual diagnostic and stop after + * a service-wide failure instead of repeating the same timeout per target. + */ + public PrFilesDeletionOutcome deletePrFilesWithOutcome( + String workspace, + String project, + int prNumber, + String collectionTarget) { + String targetLabel = collectionTarget != null && !collectionTarget.isBlank() + ? collectionTarget + : "legacy-alias"; if (!ragEnabled) { log.debug("RAG disabled, skipping PR files deletion"); - return true; + return PrFilesDeletionOutcome.success(targetLabel); } - String url = String.format("%s/index/pr-files/%s/%s/%d", ragApiUrl, workspace, project, prNumber); + HttpUrl.Builder urlBuilder = HttpUrl.get(String.format( + "%s/index/pr-files/%s/%s/%d", ragApiUrl, workspace, project, prNumber)).newBuilder(); + if (collectionTarget != null && !collectionTarget.isBlank()) { + urlBuilder.addQueryParameter("collection_target", collectionTarget); + } Request.Builder builder = new Request.Builder() - .url(url) + .url(urlBuilder.build()) .delete(); addAuthHeader(builder); Request request = builder.build(); try (Response response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { - log.info("Deleted PR #{} indexed data from {}/{}", prNumber, workspace, project); - return true; + log.info("Deleted PR #{} indexed data from {}/{} target={}", + prNumber, workspace, project, targetLabel); + return PrFilesDeletionOutcome.success(targetLabel); } else { - log.warn("Failed to delete PR #{} files: {} - {}", prNumber, response.code(), - response.body() != null ? response.body().string() : "no body"); - return false; + int statusCode = response.code(); + String detail = response.body() != null ? response.body().string() : "no body"; + boolean serviceFailure = statusCode == 401 + || statusCode == 403 + || statusCode == 408 + || statusCode == 409 + || statusCode == 429 + || statusCode >= 500; + return PrFilesDeletionOutcome.failure( + targetLabel, + serviceFailure + ? PrFilesDeletionFailure.SERVICE + : PrFilesDeletionFailure.TARGET, + statusCode, + truncateDetail(detail)); } } catch (IOException e) { - log.warn("Error deleting PR #{} files from {}/{}: {}", prNumber, workspace, project, e.getMessage()); - return false; + return PrFilesDeletionOutcome.failure( + targetLabel, + PrFilesDeletionFailure.TRANSPORT, + null, + e.getMessage()); + } + } + + public enum PrFilesDeletionFailure { + NONE, + TARGET, + SERVICE, + TRANSPORT + } + + public record PrFilesDeletionOutcome( + String targetLabel, + boolean successful, + PrFilesDeletionFailure failure, + Integer statusCode, + String detail) { + + public static PrFilesDeletionOutcome success(String targetLabel) { + return new PrFilesDeletionOutcome( + targetLabel, true, PrFilesDeletionFailure.NONE, null, null); + } + + public static PrFilesDeletionOutcome failure( + String targetLabel, + PrFilesDeletionFailure failure, + Integer statusCode, + String detail) { + return new PrFilesDeletionOutcome( + targetLabel, false, failure, statusCode, detail); + } + + public boolean shouldStopRemainingTargets() { + return failure == PrFilesDeletionFailure.SERVICE + || failure == PrFilesDeletionFailure.TRANSPORT; } } @@ -563,8 +710,62 @@ public boolean deleteBranch( String branch, String collectionTarget ) throws IOException { + return deleteBranch( + workspace, project, branch, collectionTarget, null, null); + } + + public boolean deleteBranch( + String workspace, + String project, + String branch, + String collectionTarget, + String generationRevision, + String generationManifestSha256 + ) throws IOException { + BranchDeletionOutcome outcome = deleteBranchWithOutcome( + workspace, project, branch, collectionTarget, + generationRevision, generationManifestSha256); + if (outcome.failure() == BranchDeletionFailure.TRANSPORT) { + throw new IOException(outcome.detail()); + } + if (!outcome.successful() + && !(outcome.statusCode() == null && "RAG disabled".equals(outcome.detail()))) { + log.warn("Failed to delete branch data target={}: status={} detail={}", + outcome.targetLabel(), outcome.statusCode(), outcome.detail()); + } + return outcome.successful(); + } + + /** Structured, non-logging branch deletion for multi-generation cleanup. */ + public BranchDeletionOutcome deleteBranchWithOutcome( + String workspace, + String project, + String branch, + String collectionTarget + ) { + return deleteBranchWithOutcome( + workspace, project, branch, collectionTarget, null, null); + } + + /** + * Deletes one exact generation using its registry-owned revision and + * manifest digest as an O(1) ownership proof. The RAG service retrieves the + * deterministic manifest point; it does not scan every collection member. + */ + public BranchDeletionOutcome deleteBranchWithOutcome( + String workspace, + String project, + String branch, + String collectionTarget, + String generationRevision, + String generationManifestSha256 + ) { + String targetLabel = collectionTarget != null && !collectionTarget.isBlank() + ? collectionTarget + : "legacy-alias"; if (!ragEnabled) { - return false; + return BranchDeletionOutcome.failure( + targetLabel, BranchDeletionFailure.TARGET, null, "RAG disabled"); } // URL-encode branch name to handle slashes (e.g., feature/xyz -> feature%2Fxyz) @@ -573,6 +774,14 @@ public boolean deleteBranch( "%s/index/%s/%s/branch/%s", ragApiUrl, workspace, project, encodedBranch)).newBuilder(); if (collectionTarget != null && !collectionTarget.isBlank()) { urlBuilder.addQueryParameter("collection_target", collectionTarget); + if (generationRevision != null && !generationRevision.isBlank()) { + urlBuilder.addQueryParameter("generation_revision", generationRevision); + } + if (generationManifestSha256 != null + && !generationManifestSha256.isBlank()) { + urlBuilder.addQueryParameter( + "generation_manifest_sha256", generationManifestSha256); + } } Request.Builder builder = new Request.Builder() @@ -583,13 +792,68 @@ public boolean deleteBranch( try (Response response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { - log.info("Deleted branch data for {}/{}/{}", workspace, project, branch); - return true; + log.info("Deleted branch data for {}/{}/{} target={}", + workspace, project, branch, targetLabel); + return BranchDeletionOutcome.success(targetLabel); } else { - log.warn("Failed to delete branch data: {} - {}", response.code(), - response.body() != null ? response.body().string() : "no body"); - return false; + int statusCode = response.code(); + String detail = response.body() != null + ? response.body().string() + : "no body"; + boolean serviceFailure = statusCode == 401 + || statusCode == 403 + || statusCode == 408 + || statusCode == 409 + || statusCode == 429 + || statusCode >= 500; + return BranchDeletionOutcome.failure( + targetLabel, + serviceFailure + ? BranchDeletionFailure.SERVICE + : BranchDeletionFailure.TARGET, + statusCode, + truncateDetail(detail)); } + } catch (IOException transportFailure) { + return BranchDeletionOutcome.failure( + targetLabel, + BranchDeletionFailure.TRANSPORT, + null, + transportFailure.getMessage()); + } + } + + public enum BranchDeletionFailure { + NONE, + TARGET, + SERVICE, + TRANSPORT + } + + public record BranchDeletionOutcome( + String targetLabel, + boolean successful, + BranchDeletionFailure failure, + Integer statusCode, + String detail) { + + public static BranchDeletionOutcome success(String targetLabel) { + return new BranchDeletionOutcome( + targetLabel, true, BranchDeletionFailure.NONE, null, null); + } + + public static BranchDeletionOutcome failure( + String targetLabel, + BranchDeletionFailure failure, + Integer statusCode, + String detail) { + return new BranchDeletionOutcome( + targetLabel, false, failure, statusCode, detail); + } + + public boolean shouldStopRemainingTargets() { + return failure == BranchDeletionFailure.SERVICE + || failure == BranchDeletionFailure.TRANSPORT; } } @@ -761,6 +1025,7 @@ private Map postLongRunning(String url, Map payl private Map postLongRunningSse( String url, Map payload, + Runnable ownershipAdmissionConsumer, Consumer> progressConsumer ) throws IOException { RequestBody body = RequestBody.create(objectMapper.writeValueAsString(payload), JSON); @@ -773,7 +1038,7 @@ private Map postLongRunningSse( try (Response response = longRunningHttpClient.newCall(builder.build()).execute()) { if (!response.isSuccessful()) { String detail = response.body() != null ? response.body().string() : "{}"; - throw new IOException("RAG API error: " + response.code() + " — " + detail); + throw new RagApiException(response.code(), detail); } if (response.body() == null) { throw new IOException("RAG progress stream returned no body"); @@ -789,6 +1054,15 @@ private Map postLongRunningSse( } Map event = objectMapper.readValue(json, Map.class); String type = String.valueOf(event.get("type")); + if ("admitted".equals(type)) { + if (Boolean.TRUE.equals( + event.get("repositoryOwnershipTransferred")) + && ownershipAdmissionConsumer != null) { + ownershipAdmissionConsumer.run(); + ownershipAdmissionConsumer = null; + } + continue; + } if ("progress".equals(type)) { if (progressConsumer != null) { progressConsumer.accept(new LinkedHashMap<>(event)); @@ -810,6 +1084,13 @@ private Map postLongRunningSse( throw new IOException("RAG progress stream ended without a terminal result"); } + private static String truncateDetail(String detail) { + if (detail == null) { + return "no detail"; + } + return detail.length() > 500 ? detail.substring(0, 500) + "..." : detail; + } + /** * Adds the x-service-secret header to the request if a secret is configured. */ @@ -834,12 +1115,14 @@ private Map doRequest(String url, Map payload, O String responseBody = response.body() != null ? response.body().string() : "{}"; if (!response.isSuccessful()) { - log.error("RAG API request failed: {} - {}", response.code(), responseBody); + // Callers own contextual, rate-bounded diagnostics. Keep the + // complete detail on the exception without logging it twice. + log.debug("RAG API request failed: {} - {}", response.code(), responseBody); // Include truncated response body in exception so callers can see the actual error String detail = responseBody.length() > 500 ? responseBody.substring(0, 500) + "..." : responseBody; - throw new IOException("RAG API error: " + response.code() + " — " + detail); + throw new RagApiException(response.code(), detail); } return objectMapper.readValue(responseBody, Map.class); diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java index e12ae8ff..8e2dda75 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java @@ -14,6 +14,7 @@ import java.time.OffsetDateTime; import java.util.HexFormat; import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -63,8 +64,17 @@ public BuildRegistration registerBuild( .findByProjectIdAndOperationKey(project.getId(), operationKey); if (existing.isPresent()) { RagIndexOperation operation = existing.get(); + Long branchIndexId = operation.getGeneration().getBranchIndex().getId(); + RagBranchIndex branchIndex = branchIndexRepository + .findByIdForPublication(branchIndexId) + .orElseThrow(() -> new IllegalStateException( + "RAG branch index not found: " + branchIndexId)); + rejectCleanupClaim(branchIndex); + branchIndex.markAccessed(); + branchIndexRepository.save(branchIndex); + operation.getGeneration().setBranchIndex(branchIndex); return new BuildRegistration( - operation.getGeneration().getBranchIndex(), + branchIndex, operation.getGeneration(), operation, true); @@ -73,6 +83,7 @@ public BuildRegistration registerBuild( RagBranchIndex branchIndex = branchIndexRepository .findByProjectIdAndBranchNameForUpdate(project.getId(), branch) .orElseGet(() -> new RagBranchIndex(project, branch, kind)); + rejectCleanupClaim(branchIndex); if (branchIndex.getIndexKind() == RagBranchIndexKind.LEGACY || kind == RagBranchIndexKind.PRIMARY || (branchIndex.getIndexKind() == RagBranchIndexKind.TRANSIENT @@ -112,12 +123,20 @@ public void startBuild(long operationId, Long jobId, String analysisLockKey) { if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { return; } + String normalizedLockKey = normalizeOptional(analysisLockKey); + if (operation.getStatus() == RagIndexOperationStatus.RUNNING + && Objects.equals(operation.getJobId(), jobId) + && Objects.equals(operation.getAnalysisLockKey(), normalizedLockKey)) { + operation.heartbeat(); + operationRepository.save(operation); + return; + } if (operation.getStatus() == RagIndexOperationStatus.FAILED) { operation.getGeneration().retry(); generationRepository.save(operation.getGeneration()); } operation.setJobId(jobId); - operation.setAnalysisLockKey(normalizeOptional(analysisLockKey)); + operation.setAnalysisLockKey(normalizedLockKey); operation.start(); operationRepository.save(operation); } @@ -143,6 +162,7 @@ public RagBranchIndexGeneration publish( .orElseThrow(() -> new IllegalStateException( "RAG branch index not found: " + branchIndexId)); generation.setBranchIndex(branchIndex); + rejectCleanupClaim(branchIndex); String digest = requireText(manifestDigest, "manifestDigest"); if (!generation.getRevision().equals(branchIndex.getDesiredCommitHash())) { @@ -196,7 +216,8 @@ public boolean failIfAbandoned( } private boolean failOperation(RagIndexOperation operation, String errorMessage) { - if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { + if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED + || operation.getStatus() == RagIndexOperationStatus.FAILED) { return false; } String failure = requireText(errorMessage, "errorMessage"); @@ -224,11 +245,17 @@ public Optional findAvailableGeneration( String branchName, String revision) { Optional branchIndex = branchIndexRepository - .findByProjectIdAndBranchName(projectId, requireText(branchName, "branchName")); + .findByProjectIdAndBranchNameForUpdate( + projectId, requireText(branchName, "branchName")); if (branchIndex.isEmpty()) { return Optional.empty(); } RagBranchIndex index = branchIndex.get(); + if (index.getCleanupClaimToken() != null) { + // Cleanup has a durable claim. Returning a generation here would + // expose a physical target that may already have been deleted. + return Optional.empty(); + } index.markAccessed(); branchIndexRepository.save(index); if (index.getActiveGeneration() != null @@ -249,16 +276,23 @@ public void heartbeatBuild(long operationId) { operationRepository.save(operation); } - public List findRecoverableOperations(OffsetDateTime updatedBefore) { - return operationRepository.findByStatusInAndUpdatedAtBefore( + public List + findRecoverableOperations(OffsetDateTime updatedBefore) { + return operationRepository.findRecoverableOperationProjections( List.of(RagIndexOperationStatus.PENDING, RagIndexOperationStatus.RUNNING), updatedBefore); } - public List findFailedOperationsWithActiveProjections() { + public List + findFailedOperationsWithActiveProjections() { return operationRepository.findFailedOperationsWithActiveProjections(); } + public List + findSucceededOperationsWithActiveProjections() { + return operationRepository.findSucceededOperationsWithActiveProjections(); + } + public boolean hasLiveOperation(long projectId, String branchName) { return operationRepository.existsByProjectIdAndBranchNameAndStatusIn( projectId, @@ -316,6 +350,13 @@ private static String normalizeOptional(String value) { return value == null || value.isBlank() ? null : value.trim(); } + private static void rejectCleanupClaim(RagBranchIndex branchIndex) { + if (branchIndex.getCleanupClaimToken() != null) { + throw new IllegalStateException( + "RAG branch index cleanup owns this transient branch; retry later"); + } + } + private static String nullToEmpty(String value) { return value == null ? "" : value; } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java index 1cc28523..49b7e03a 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java @@ -207,6 +207,122 @@ public RagIndexStatus markUpdatingStarted( return status; } + /** + * Atomically aligns the project checkpoint with an already-published exact + * generation. This is used by same-revision no-ops and post-publication + * recovery, neither of which needs a synthetic RUNNING status transition. + * A newer job owner is never overwritten. + */ + @Transactional + public boolean reconcilePublishedGeneration( + Project project, + String branchName, + String commitHash, + Integer fileCount, + Integer chunkCount) { + return reconcilePublishedGeneration( + project, branchName, commitHash, fileCount, chunkCount, null); + } + + @Transactional + public boolean reconcilePublishedGeneration( + Project project, + String branchName, + String commitHash, + Integer fileCount, + Integer chunkCount, + Long expectedActiveJobId) { + Optional existing = + ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()); + RagIndexStatus status; + if (existing.isPresent()) { + status = existing.get(); + if (expectedActiveJobId != null + && !Objects.equals(status.getActiveJobId(), expectedActiveJobId)) { + log.info( + "Preserving RAG status owned by job {} while job {} reconciles " + + "a published generation for project {}", + status.getActiveJobId(), expectedActiveJobId, project.getId()); + return false; + } + if (expectedActiveJobId == null && status.getActiveJobId() != null) { + log.info( + "Preserving RAG status owned by live job {} while reconciling " + + "published generation for project {}", + status.getActiveJobId(), project.getId()); + return false; + } + } else { + if (expectedActiveJobId != null) { + log.info( + "Published generation for job {} has no owned RAG status to reconcile " + + "for project {}", + expectedActiveJobId, project.getId()); + return false; + } + status = new RagIndexStatus(); + status.setProject(project); + status.setWorkspaceName(project.getWorkspace().getName()); + status.setProjectName(project.getName()); + status.setCollectionName(generateCollectionName(project)); + } + + status.setStatus(RagIndexingStatus.INDEXED); + status.setIndexedBranch(branchName); + status.setIndexedCommitHash(commitHash); + if (fileCount != null) { + status.setTotalFilesIndexed(fileCount); + } + if (chunkCount != null) { + status.setChunkCount(chunkCount); + } + status.setLastIndexedAt(OffsetDateTime.now()); + status.setErrorMessage(null); + status.setActiveJobId(null); + status.resetFailedIncrementalCount(); + ragIndexStatusRepository.save(status); + return true; + } + + /** + * Restores the active exact generation as the completed checkpoint while + * the caller owns the branch RAG lock. The caller immediately admits the + * replacement job in the same transaction, so an older status owner + * cannot later publish through the job-id ownership checks. + */ + @Transactional + public void preparePublishedGenerationForUpdate( + Project project, + String branchName, + String commitHash, + Integer fileCount, + Integer chunkCount) { + RagIndexStatus status = ragIndexStatusRepository + .findByProjectIdForUpdate(project.getId()) + .orElseGet(() -> { + RagIndexStatus created = new RagIndexStatus(); + created.setProject(project); + created.setWorkspaceName(project.getWorkspace().getName()); + created.setProjectName(project.getName()); + created.setCollectionName(generateCollectionName(project)); + return created; + }); + status.setStatus(RagIndexingStatus.INDEXED); + status.setIndexedBranch(branchName); + status.setIndexedCommitHash(commitHash); + if (fileCount != null) { + status.setTotalFilesIndexed(fileCount); + } + if (chunkCount != null) { + status.setChunkCount(chunkCount); + } + status.setLastIndexedAt(OffsetDateTime.now()); + status.setErrorMessage(null); + status.setActiveJobId(null); + status.resetFailedIncrementalCount(); + ragIndexStatusRepository.save(status); + } + /** * Marks an incremental update as completed. * Updates totalFilesIndexed by adding addedFiles count and subtracting @@ -330,6 +446,20 @@ public RagIndexStatus markIncrementalUpdateFailed( return status; } + /** + * Quiet, owner-guarded repair used by the periodic legacy-job recovery + * scan. The scheduler owns degraded/recovered log transitions, preventing + * the same database outage from producing one warning per scan and job. + */ + @Transactional + public boolean recoverAbandonedIncrementalUpdate( + Long projectId, + Long expectedActiveJobId, + String errorMessage) { + return ragIndexStatusRepository.recoverAbandonedIncrementalUpdate( + projectId, expectedActiveJobId, errorMessage) == 1; + } + @Transactional(readOnly = true) public boolean canStartIndexing(Project project) { Optional statusOpt = ragIndexStatusRepository.findByProjectId(project.getId()); diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java index 88c0da8f..6cad6d3f 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java @@ -18,6 +18,9 @@ import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; +import org.rostilos.codecrow.ragengine.branch.BranchIndexBuildAdmissionService; +import org.rostilos.codecrow.ragengine.branch.LegacyRagJobLeaseService; +import org.rostilos.codecrow.ragengine.branch.LegacyRagUpdateCompletionService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.slf4j.Logger; @@ -30,6 +33,7 @@ import java.io.IOException; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -58,6 +62,9 @@ public class RagOperationsServiceImpl implements RagOperationsService { private final RagPipelineClient ragPipelineClient; private final RagBranchIndexRegistryService branchIndexRegistryService; private final BranchIndexGenerationBuildService branchGenerationBuildService; + private final BranchIndexBuildAdmissionService branchIndexBuildAdmissionService; + private final LegacyRagJobLeaseService legacyRagJobLeaseService; + private final LegacyRagUpdateCompletionService legacyRagUpdateCompletionService; @Autowired(required = false) private RagBranchIndexGenerationRepository branchGenerationRepository; @@ -65,6 +72,9 @@ public class RagOperationsServiceImpl implements RagOperationsService { @Value("${codecrow.rag.api.enabled:true}") private boolean ragApiEnabled; + @Value("${analysis.lock.rag.timeout.minutes:360}") + private int legacyRagLockLeaseMinutes = 360; + public RagOperationsServiceImpl( RagIndexTrackingService ragIndexTrackingService, IncrementalRagUpdateService incrementalRagUpdateService, @@ -76,7 +86,7 @@ public RagOperationsServiceImpl( this(ragIndexTrackingService, incrementalRagUpdateService, analysisLockService, analysisJobService, ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, null, null); + ragPipelineClient, null, null, null, null, null); } public RagOperationsServiceImpl( @@ -91,10 +101,9 @@ public RagOperationsServiceImpl( this(ragIndexTrackingService, incrementalRagUpdateService, analysisLockService, analysisJobService, ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, branchIndexRegistryService, null); + ragPipelineClient, branchIndexRegistryService, null, null, null, null); } - @Autowired public RagOperationsServiceImpl( RagIndexTrackingService ragIndexTrackingService, IncrementalRagUpdateService incrementalRagUpdateService, @@ -105,6 +114,46 @@ public RagOperationsServiceImpl( RagPipelineClient ragPipelineClient, RagBranchIndexRegistryService branchIndexRegistryService, BranchIndexGenerationBuildService branchGenerationBuildService) { + this(ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, branchIndexRegistryService, + branchGenerationBuildService, null, null, null); + } + + public RagOperationsServiceImpl( + RagIndexTrackingService ragIndexTrackingService, + IncrementalRagUpdateService incrementalRagUpdateService, + AnalysisLockService analysisLockService, + AnalysisJobService analysisJobService, + RagBranchIndexRepository ragBranchIndexRepository, + VcsClientProvider vcsClientProvider, + RagPipelineClient ragPipelineClient, + RagBranchIndexRegistryService branchIndexRegistryService, + BranchIndexGenerationBuildService branchGenerationBuildService, + BranchIndexBuildAdmissionService branchIndexBuildAdmissionService) { + this(ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, branchIndexRegistryService, + branchGenerationBuildService, branchIndexBuildAdmissionService, + null, null); + } + + @Autowired + public RagOperationsServiceImpl( + RagIndexTrackingService ragIndexTrackingService, + IncrementalRagUpdateService incrementalRagUpdateService, + AnalysisLockService analysisLockService, + AnalysisJobService analysisJobService, + RagBranchIndexRepository ragBranchIndexRepository, + VcsClientProvider vcsClientProvider, + RagPipelineClient ragPipelineClient, + RagBranchIndexRegistryService branchIndexRegistryService, + BranchIndexGenerationBuildService branchGenerationBuildService, + BranchIndexBuildAdmissionService branchIndexBuildAdmissionService, + LegacyRagJobLeaseService legacyRagJobLeaseService, + LegacyRagUpdateCompletionService legacyRagUpdateCompletionService) { this.ragIndexTrackingService = ragIndexTrackingService; this.incrementalRagUpdateService = incrementalRagUpdateService; this.analysisLockService = analysisLockService; @@ -114,6 +163,9 @@ public RagOperationsServiceImpl( this.ragPipelineClient = ragPipelineClient; this.branchIndexRegistryService = branchIndexRegistryService; this.branchGenerationBuildService = branchGenerationBuildService; + this.branchIndexBuildAdmissionService = branchIndexBuildAdmissionService; + this.legacyRagJobLeaseService = legacyRagJobLeaseService; + this.legacyRagUpdateCompletionService = legacyRagUpdateCompletionService; } @Override @@ -136,6 +188,19 @@ public boolean isRagIndexReady(Project project) { if (!isRagEnabled(project)) { return false; } + if (usesExactGenerations(project)) { + String primaryBranch = getBaseBranch(project); + boolean primaryRegistered = ragBranchIndexRepository + .existsByProjectIdAndBranchName(project.getId(), primaryBranch); + if (primaryRegistered) { + if (ragBranchIndexRepository.markAccessedIfUnclaimed( + project.getId(), primaryBranch, OffsetDateTime.now()) == 0) { + return false; + } + return ragBranchIndexRepository.findActiveGenerationCoordinates( + project.getId(), primaryBranch).isPresent(); + } + } return ragIndexTrackingService.isProjectIndexed(project); } @@ -149,13 +214,81 @@ public boolean deletePrFiles(Project project, int prNumber) { try { String workspace = project.getWorkspace().getName(); String namespace = project.getNamespace(); - return ragPipelineClient.deletePrFiles(workspace, namespace, prNumber); + Set collectionTargets = new LinkedHashSet<>(); + if (branchGenerationRepository != null) { + List registeredTargets = branchGenerationRepository + .findCollectionNamesByProjectIdAndStatusIn( + project.getId(), + List.of( + RagBranchIndexGenerationStatus.ACTIVE, + RagBranchIndexGenerationStatus.SUPERSEDED)); + if (registeredTargets != null) { + registeredTargets.stream() + .filter(target -> target != null && !target.isBlank()) + .map(String::trim) + .forEach(collectionTargets::add); + } + } + + if (collectionTargets.isEmpty()) { + // Legacy projects have no physical-generation registry. Their + // project alias remains the only cleanup target. + RagPipelineClient.PrFilesDeletionOutcome outcome; + try { + outcome = ragPipelineClient.deletePrFilesWithOutcome( + workspace, namespace, prNumber, null); + } catch (RuntimeException unexpectedFailure) { + log.warn("Failed to delete PR #{} files for project={} target=legacy-alias: " + + "status=unexpected detail={}", + prNumber, project.getId(), unexpectedFailure.getMessage()); + return false; + } + if (!outcome.successful()) { + logPrCleanupFailure(project, prNumber, outcome); + } + return outcome.successful(); + } + + boolean allTargetsCleaned = true; + for (String collectionTarget : collectionTargets) { + RagPipelineClient.PrFilesDeletionOutcome outcome; + try { + outcome = ragPipelineClient.deletePrFilesWithOutcome( + workspace, namespace, prNumber, collectionTarget); + } catch (RuntimeException unexpectedFailure) { + log.warn("Failed to delete PR #{} files for project={} target={}: " + + "status=unexpected detail={}", + prNumber, project.getId(), collectionTarget, + unexpectedFailure.getMessage()); + return false; + } + if (!outcome.successful()) { + allTargetsCleaned = false; + logPrCleanupFailure(project, prNumber, outcome); + if (outcome.shouldStopRemainingTargets()) { + break; + } + } + } + return allTargetsCleaned; } catch (Exception e) { log.warn("Failed to delete PR #{} files for project {}: {}", prNumber, project.getId(), e.getMessage()); return false; } } + private static void logPrCleanupFailure( + Project project, + int prNumber, + RagPipelineClient.PrFilesDeletionOutcome outcome) { + log.warn("Failed to delete PR #{} files for project={} target={}: status={} detail={}", + prNumber, + project.getId(), + outcome.targetLabel(), + outcome.statusCode() != null ? outcome.statusCode() : outcome.failure(), + outcome.detail()); + } + @Override public boolean triggerIncrementalUpdate( Project project, @@ -167,64 +300,64 @@ public boolean triggerIncrementalUpdate( log.info("triggerIncrementalUpdate called for project={}, branch={}, commit={}, diffLength={}", project.getId(), branchName, commitHash, rawDiff != null ? rawDiff.length() : 0); try { - if (!incrementalRagUpdateService.shouldPerformIncrementalUpdate(project)) { + boolean exactGenerationMode = usesExactGenerations(project); + boolean normalIncrementalReady = + incrementalRagUpdateService.shouldPerformIncrementalUpdate(project); + boolean exactRecoveryCandidate = exactGenerationMode + && isRagEnabled(project) + && ragBranchIndexRepository.existsByProjectIdAndBranchName( + project.getId(), branchName); + if (!normalIncrementalReady && !exactRecoveryCandidate) { log.info( "Skipping RAG incremental update for project={}, branch={} - RAG not enabled or main branch not yet indexed", project.getId(), branchName); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "message", "Skipping RAG update - main branch must be indexed first")); return false; } - boolean exactGenerationMode = usesExactGenerations(project); boolean tracksProjectStatus = branchName.equals(getBaseBranch(project)); RagBranchIndexKind exactGenerationKind = exactGenerationMode ? indexKind(project, branchName) : null; - boolean publishBranchAlias = exactGenerationKind == RagBranchIndexKind.PRIMARY - || exactGenerationKind == RagBranchIndexKind.DURABLE; - boolean publishLegacyProjectAlias = exactGenerationKind - == RagBranchIndexKind.PRIMARY; - RagBranchIndexGeneration initialSourceGeneration = exactGenerationMode - ? ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), branchName) - .map(RagBranchIndex::getActiveGeneration) - .orElse(null) - : null; - boolean fullExactSnapshotRequired = exactGenerationMode - && initialSourceGeneration == null; - - String effectiveRawDiff = fullExactSnapshotRequired - ? "" - : resolveDiffFromCompletedCheckpoint( - project, branchName, commitHash, rawDiff); - log.info("RAG checkpoint reconciliation complete; parsing effective diff..."); - - // Parse the diff to find changed files - IncrementalRagUpdateService.DiffResult diffResult = - incrementalRagUpdateService.parseDiffForRag(effectiveRawDiff); - Set addedFiles = diffResult.added(); - Set modifiedFiles = diffResult.modified(); - Set deletedFiles = diffResult.deleted(); - - int addedOrModifiedSize = addedFiles.size() + modifiedFiles.size(); - - log.info("Diff parsed: added={}, modified={}, deleted={}", addedFiles, modifiedFiles, deletedFiles); - - if (addedOrModifiedSize == 0 && deletedFiles.isEmpty() - && !exactGenerationMode) { - log.info("Skipping RAG incremental update - no files changed in diff"); - return true; + Set addedFiles = Set.of(); + Set modifiedFiles = Set.of(); + Set deletedFiles = Set.of(); + int addedOrModifiedSize = 0; + + // Preserve the legacy checkpoint flow, including its no-op return + // before a job or lock is created. Exact generations are resolved + // below only after this update owns the branch RAG lock. + if (!exactGenerationMode) { + String effectiveRawDiff = resolveDiffFromCompletedCheckpoint( + project, branchName, commitHash, rawDiff); + log.info("RAG checkpoint reconciliation complete; parsing effective diff..."); + IncrementalRagUpdateService.DiffResult diffResult = + incrementalRagUpdateService.parseDiffForRag(effectiveRawDiff); + addedFiles = diffResult.added(); + modifiedFiles = diffResult.modified(); + deletedFiles = diffResult.deleted(); + addedOrModifiedSize = addedFiles.size() + modifiedFiles.size(); + + log.info("Diff parsed: added={}, modified={}, deleted={}", + addedFiles, modifiedFiles, deletedFiles); + if (addedOrModifiedSize == 0 && deletedFiles.isEmpty()) { + log.info("Skipping RAG incremental update - no files changed in diff"); + return true; + } + log.info("RAG incremental update: {} files to add/update, {} files to delete", + addedOrModifiedSize, deletedFiles.size()); } - log.info("RAG incremental update: {} files to add/update, {} files to delete", - addedOrModifiedSize, deletedFiles.size()); - - job = analysisJobService.createRagIndexJob(project, false, JobTriggerSource.WEBHOOK); - analysisJobService.info(job, "rag_init", - String.format( - "Starting incremental RAG update for branch '%s' (commit: %s) - %d files to update, %d to delete", - branchName, commitHash, addedOrModifiedSize, deletedFiles.size())); + if (!exactGenerationMode) { + job = analysisJobService.createRagIndexJob( + project, false, JobTriggerSource.WEBHOOK, branchName, commitHash); + analysisJobService.startJob(job); + analysisJobService.info(job, "rag_init", + String.format( + "Starting incremental RAG update for branch '%s' (commit: %s) - %d files to update, %d to delete", + branchName, commitHash, addedOrModifiedSize, deletedFiles.size())); + } Optional ragLockKey = analysisLockService.acquireLock( project, @@ -234,21 +367,134 @@ public boolean triggerIncrementalUpdate( null); if (ragLockKey.isEmpty()) { - log.warn("RAG update already in progress for project={}, branch={}", + log.info("RAG update already in progress for project={}, branch={}; " + + "deferring this revision to a later trigger", project.getId(), branchName); - analysisJobService.warn(job, "rag_skip", "RAG update already in progress - skipping"); - analysisJobService.failJob(job, "RAG update already in progress"); + String reason = "RAG update already in progress; this revision was skipped and " + + "the previous checkpoint is retained for the next trigger"; + if (job != null) { + analysisJobService.info(job, "rag_skip", reason); + analysisJobService.skipJob(job, reason); + } + emitEvent(eventConsumer, Map.of( + "type", "info", + "state", "rag_skip", + "message", reason)); return false; } + BranchIndexBuildAdmissionService.AdmittedBuild admittedBuild = null; + boolean exactExecutionStarted = false; + boolean exactPublicationCompleted = false; + LegacyRagJobLeaseService.JobLease legacyJobLease = null; + AnalysisLockService.LockLease legacyLockLease = null; try { - eventConsumer.accept(Map.of( - "type", "status", - "state", "rag_update", - "message", - "Updating RAG index with " + (addedOrModifiedSize + deletedFiles.size()) + " changed files")); + if (!exactGenerationMode) { + if (legacyRagJobLeaseService == null + || job == null + || job.getId() == null) { + throw new IllegalStateException( + "Legacy RAG job lease service is unavailable"); + } + legacyJobLease = legacyRagJobLeaseService.start(job.getId()); + legacyLockLease = analysisLockService.maintainLockLease( + ragLockKey.get(), Math.max(1, legacyRagLockLeaseMinutes)); + requireLegacyOwnership(legacyJobLease, legacyLockLease); + } + RagBranchIndexRepository.ActiveGenerationCoordinates sourceGeneration = null; + if (exactGenerationMode) { + boolean existingExactIndex = ragBranchIndexRepository + .existsByProjectIdAndBranchName(project.getId(), branchName); + if (existingExactIndex) { + int accessed = ragBranchIndexRepository.markAccessedIfUnclaimed( + project.getId(), branchName, OffsetDateTime.now()); + if (accessed == 0) { + log.info("Skipping exact RAG update while transient cleanup owns " + + "project={}, branch={}", + project.getId(), branchName); + String reason = "Temporary branch index cleanup is in progress; " + + "the update will retry later"; + emitEvent(eventConsumer, Map.of( + "type", "info", + "state", "rag_skipped", + "message", reason)); + return false; + } + sourceGeneration = ragBranchIndexRepository + .findActiveGenerationCoordinates(project.getId(), branchName) + .orElse(null); + } - if (tracksProjectStatus) { + if (sourceGeneration != null + && sourceGeneration.getRevision().equals(commitHash)) { + String message = String.format( + "RAG generation for branch '%s' already represents commit %s", + branchName, commitHash); + if (tracksProjectStatus) { + // Publication is authoritative. Repair a project + // checkpoint left stale by a crash in one atomic + // terminal transition, without creating an + // ownerless UPDATING window. + ragIndexTrackingService.preparePublishedGenerationForUpdate( + project, branchName, commitHash, + sourceGeneration.getFileCount(), + sourceGeneration.getChunkCount()); + } + emitEvent(eventConsumer, Map.of( + "type", "info", + "state", "rag_complete", + "message", message)); + return true; + } + + if (branchIndexBuildAdmissionService == null) { + throw new IllegalStateException( + "Exact RAG build admission service is unavailable"); + } + VcsRepoBinding exactBinding = project.getVcsRepoBinding(); + if (exactBinding == null || exactBinding.getVcsConnection() == null + || exactBinding.getExternalNamespace() == null + || exactBinding.getExternalNamespace().isBlank() + || exactBinding.getExternalRepoSlug() == null + || exactBinding.getExternalRepoSlug().isBlank()) { + throw new IllegalStateException( + "Project has no complete VcsRepoBinding configured"); + } + if (project.getConfiguration() == null + || project.getConfiguration().ragConfig() == null) { + throw new IllegalStateException( + "Project has no RAG configuration"); + } + admittedBuild = branchIndexBuildAdmissionService.admit( + project, + branchName, + commitHash, + exactGenerationKind, + JobTriggerSource.WEBHOOK, + ragLockKey.get(), + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + job = admittedBuild.job(); + analysisJobService.info(job, "rag_init", + String.format( + "Starting exact RAG generation rebuild for branch '%s' (commit: %s)", + branchName, commitHash)); + } + + emitEvent(eventConsumer, exactGenerationMode + ? Map.of( + "type", "status", + "state", "rag_update", + "message", String.format( + "Building exact RAG snapshot for branch '%s' at commit %s", + branchName, commitHash)) + : Map.of( + "type", "status", + "state", "rag_update", + "message", "Updating RAG index with " + + (addedOrModifiedSize + deletedFiles.size()) + + " changed files")); + + if (tracksProjectStatus && !exactGenerationMode) { ragIndexTrackingService.markUpdatingStarted( project, branchName, commitHash, job != null ? job.getId() : null); } @@ -266,102 +512,55 @@ public boolean triggerIncrementalUpdate( String workspaceSlug = vcsRepoBinding.getExternalNamespace(); String repoSlug = vcsRepoBinding.getExternalRepoSlug(); - RagBranchIndexGeneration sourceGeneration = null; - RagBranchIndexRegistryService.BuildRegistration branchBuild = null; + Map result; if (exactGenerationMode) { - sourceGeneration = initialSourceGeneration; - if (sourceGeneration != null) { - branchBuild = branchIndexRegistryService.registerBuild( - project, - branchName, - exactGenerationKind, - sourceGeneration.getRevision(), - commitHash, - sourceGeneration.getRepresentationFingerprint()); - branchIndexRegistryService.startBuild( - branchBuild.operation().getId(), - job != null ? job.getId() : null, - ragLockKey.get()); + if (branchGenerationBuildService == null) { + throw new IllegalStateException( + "Exact RAG generation build service is unavailable"); } - } - - Map result; - try { - if (fullExactSnapshotRequired) { - var ragConfig = project.getConfiguration().ragConfig(); - result = branchGenerationBuildService.build( + var ragConfig = project.getConfiguration().ragConfig(); + // A same-revision active generation returned above. Any + // remaining mismatch must create and activate a fresh + // snapshot, including A -> B -> A where an older A + // generation already succeeded but is no longer active. + exactExecutionStarted = true; + result = branchGenerationBuildService.execute( + project, + vcsConnection, + workspaceSlug, + repoSlug, + branchName, + commitHash, + exactGenerationKind, + ragConfig.includePatterns(), + ragConfig.excludePatterns(), + admittedBuild.preparedBuild(), + null); + exactPublicationCompleted = true; + } else { + result = incrementalRagUpdateService.performIncrementalUpdate( project, vcsConnection, workspaceSlug, repoSlug, branchName, commitHash, - exactGenerationKind, - ragConfig.includePatterns(), - ragConfig.excludePatterns(), - job != null ? job.getId() : null, - ragLockKey.get(), - null); - } else if (exactGenerationMode) { - result = incrementalRagUpdateService.performIncrementalUpdate( - project, - vcsConnection, - workspaceSlug, - repoSlug, - branchName, - commitHash, - addedFiles, - modifiedFiles, - deletedFiles, - sourceGeneration.getRevision(), - sourceGeneration.getCollectionName(), - branchBuild.generation().getCollectionName(), - false, - false); - } else { - result = incrementalRagUpdateService.performIncrementalUpdate( - project, - vcsConnection, - workspaceSlug, - repoSlug, - branchName, - commitHash, - addedFiles, - modifiedFiles, - deletedFiles); - } - if (exactGenerationMode && !fullExactSnapshotRequired) { - Object digest = result.get("generation_manifest_sha256"); - if (!(digest instanceof String manifestDigest) - || manifestDigest.isBlank()) { - throw new IllegalStateException( - "Advanced RAG generation has no manifest digest"); - } - RagBranchIndexGeneration published = branchIndexRegistryService.publish( - branchBuild.operation().getId(), - manifestDigest, - ((Number) result.getOrDefault("document_count", 0)).intValue(), - ((Number) result.getOrDefault("chunk_count", 0)).intValue()); - publishReadableAliasesIfActive( - project, branchName, commitHash, - branchBuild.generation().getCollectionName(), - published, publishBranchAlias, - publishLegacyProjectAlias); - } - } catch (Exception generationFailure) { - if (branchBuild != null) { - branchIndexRegistryService.fail( - branchBuild.operation().getId(), - generationFailure.getMessage() != null - ? generationFailure.getMessage() - : generationFailure.getClass().getSimpleName()); - } - throw generationFailure; + addedFiles, + modifiedFiles, + deletedFiles); + } + + if (!exactGenerationMode) { + confirmLegacyOwnership(legacyJobLease, legacyLockLease); } - int filesUpdated = (Integer) result.getOrDefault("updatedFiles", 0); - int filesDeleted = (Integer) result.getOrDefault("deletedFiles", 0); - int filesSkipped = (Integer) result.getOrDefault("skippedFiles", 0); + int documentCount = result.get("document_count") instanceof Number number + ? number.intValue() : 0; + int filesUpdated = exactGenerationMode + ? documentCount + : ((Number) result.getOrDefault("updatedFiles", 0)).intValue(); + int filesDeleted = ((Number) result.getOrDefault("deletedFiles", 0)).intValue(); + int filesSkipped = ((Number) result.getOrDefault("skippedFiles", 0)).intValue(); Integer newlyAddedFilesCount = (Integer) result.get("addedFilesCount"); Integer chunkCount = null; @@ -369,68 +568,160 @@ public boolean triggerIncrementalUpdate( chunkCount = ((Number) result.get("chunk_count")).intValue(); } - if (tracksProjectStatus) { - ragIndexTrackingService.markUpdatingCompleted( + if (tracksProjectStatus && exactGenerationMode) { + ragIndexTrackingService.reconcilePublishedGeneration( project, branchName, commitHash, - newlyAddedFilesCount != null ? newlyAddedFilesCount : 0, - filesDeleted, + documentCount, chunkCount, job != null ? job.getId() : null); } - // Track branch index for deleted files - trackBranchIndex(project, branchName, commitHash, deletedFiles); + if (!exactGenerationMode) { + if (legacyRagUpdateCompletionService == null + || job == null + || job.getId() == null + || !legacyRagUpdateCompletionService.complete( + project, + branchName, + commitHash, + job.getId(), + legacyJobLease.validAfter(), + tracksProjectStatus, + newlyAddedFilesCount != null + ? newlyAddedFilesCount : 0, + filesDeleted, + chunkCount, + deletedFiles)) { + throw new LegacyRagOwnershipLostException( + "Legacy RAG update lost durable ownership before publication"); + } + } - eventConsumer.accept(Map.of( + String completionMessage = exactGenerationMode + ? String.format( + "Exact RAG snapshot activated: %d documents, %d chunks", + documentCount, chunkCount != null ? chunkCount : 0) + : String.format( + "RAG index updated: %d files updated, %d deleted, %d non-text files skipped", + filesUpdated, filesDeleted, filesSkipped); + emitEvent(eventConsumer, Map.of( "type", "status", "state", "rag_complete", - "message", - String.format( - "RAG index updated: %d files updated, %d deleted, %d non-text files skipped", - filesUpdated, filesDeleted, filesSkipped))); + "message", completionMessage)); log.info("RAG incremental update completed for project={}: {} files updated, {} deleted, " + "{} non-text files skipped", project.getId(), filesUpdated, filesDeleted, filesSkipped); - analysisJobService.info(job, "rag_complete", - String.format( - "RAG incremental update completed: %d files updated, %d deleted, " - + "%d non-text files skipped", - filesUpdated, filesDeleted, filesSkipped)); - analysisJobService.completeJob(job, null); + if (exactGenerationMode) { + analysisJobService.info(job, "rag_complete", completionMessage); + analysisJobService.completeJob(job, null); + } else { + try { + analysisJobService.recordExternallyCompletedJob( + job, "rag_complete", completionMessage); + } catch (Exception notificationFailure) { + // The job and checkpoints already committed atomically. + // A local observer failure cannot reverse that outcome. + log.warn( + "Could not announce completed legacy RAG job {}: {}", + job != null ? job.getId() : null, + notificationFailure.getMessage()); + } + } return true; } catch (Exception e) { - // Use markIncrementalUpdateFailed (keeps status INDEXED, increments failure counter) - // NOT markIndexingFailed which would set status to FAILED and permanently block - // all future incremental updates even though the base index is still valid. - if (tracksProjectStatus) { - ragIndexTrackingService.markIncrementalUpdateFailed( - project, e.getMessage(), job != null ? job.getId() : null); + // execute owns operation failure after it starts. Fail the + // admitted operation here only when a local hand-off step + // between admission and execute raised first. + if (admittedBuild != null && !exactExecutionStarted) { + try { + branchIndexBuildAdmissionService.abortOperation( + admittedBuild, + e.getMessage() != null + ? e.getMessage() + : e.getClass().getSimpleName()); + } catch (Exception abortFailure) { + log.error("Could not terminalize admitted exact RAG build {}", + admittedBuild.preparedBuild().operationId(), abortFailure); + } } - log.error("RAG incremental update failed", e); - if (job != null) { - analysisJobService.error(job, "rag_error", "RAG incremental update failed: " + e.getMessage()); - analysisJobService.failJob(job, "RAG incremental update failed: " + e.getMessage()); + boolean legacyOwnershipLost = + e instanceof LegacyRagOwnershipLostException + || e instanceof LegacyRagUpdateCompletionService + .LegacyRagCompletionConflictException; + if (exactPublicationCompleted) { + // execute returns only after the registry operation is + // SUCCEEDED and the generation is active. A later status, + // job-log, or job-completion failure is projection drift, + // not a failed build. Preserve its RUNNING ownership for + // RagIndexOperationRecoveryService to finish idempotently. + log.error( + "Exact RAG generation was published but its projections could not be finalized: " + + "project={}, branch={}, job={}", + project.getId(), branchName, + job != null ? job.getId() : null, e); + } else if (legacyOwnershipLost) { + // Recovery may already own the job/status terminal + // transition. Never overwrite that durable outcome from a + // producer that can no longer prove ownership. + log.info("Legacy RAG update stopped after ownership loss: {}", + e.getMessage()); + } else { + if (tracksProjectStatus) { + if (exactGenerationMode && admittedBuild != null + && admittedBuild.statusAdmission() + == BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING) { + ragIndexTrackingService.markIndexingFailed( + project, e.getMessage(), + job != null ? job.getId() : null); + } else { + // Incremental failure preserves the preceding + // completed checkpoint and usable index. + ragIndexTrackingService.markIncrementalUpdateFailed( + project, e.getMessage(), + job != null ? job.getId() : null); + } + } + log.error("RAG incremental update failed", e); + if (job != null) { + analysisJobService.failJob( + job, "RAG incremental update failed: " + e.getMessage()); + } } - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "state", "rag_error", "message", "RAG incremental update failed: " + e.getMessage())); return false; } finally { - analysisLockService.releaseLock(ragLockKey.get()); + if (legacyJobLease != null) { + legacyJobLease.close(); + } + if (legacyLockLease != null) { + legacyLockLease.close(); + } + try { + analysisLockService.releaseLock(ragLockKey.get()); + } catch (RuntimeException releaseFailure) { + // Publication/job terminalization is authoritative. A lock + // cleanup outage must not reverse a completed operation; + // exact recovery or the lock TTL will remove the row. + log.info( + "RAG indexing lock could not be released after processing; " + + "leaving it to recovery/expiry: project={}, branch={}, detail={}", + project.getId(), branchName, releaseFailure.getMessage()); + } } } catch (Exception e) { log.warn("RAG incremental update failed (non-critical): {}", e.getMessage()); if (job != null) { - analysisJobService.error(job, "rag_error", - "RAG incremental update failed (non-critical): " + e.getMessage()); - analysisJobService.failJob(job, e.getMessage()); + analysisJobService.failJob( + job, "RAG incremental update failed: " + e.getMessage()); } - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "state", "rag_error", "message", "RAG incremental update failed: " + e.getMessage())); @@ -438,27 +729,28 @@ public boolean triggerIncrementalUpdate( } } - private void publishReadableAliasesIfActive( - Project project, - String branch, - String revision, - String collectionTarget, - RagBranchIndexGeneration published, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias) { - if (published == null - || published.getStatus() != RagBranchIndexGenerationStatus.ACTIVE - || !publishBranchAlias) { - return; + private static void requireLegacyOwnership( + LegacyRagJobLeaseService.JobLease jobLease, + AnalysisLockService.LockLease lockLease) { + if (jobLease.isOwnershipLost() || lockLease.isOwnershipLost()) { + throw new LegacyRagOwnershipLostException( + "Legacy RAG update lost durable ownership before remote mutation"); } - try { - ragPipelineClient.publishGenerationAliases( - project.getWorkspace().getName(), project.getNamespace(), - branch, revision, collectionTarget, - true, publishLegacyProjectAlias); - } catch (IOException aliasFailure) { - log.warn("Readable alias publication failed for active RAG generation {}: {}", - published.getId(), aliasFailure.getMessage()); + } + + private static void confirmLegacyOwnership( + LegacyRagJobLeaseService.JobLease jobLease, + AnalysisLockService.LockLease lockLease) { + if (!jobLease.confirmOwnership() || !lockLease.confirmOwnership()) { + throw new LegacyRagOwnershipLostException( + "Legacy RAG update lost durable ownership before publication"); + } + } + + private static final class LegacyRagOwnershipLostException + extends IllegalStateException { + private LegacyRagOwnershipLostException(String message) { + super(message); } } @@ -571,7 +863,6 @@ public boolean isBranchIndexReady(Project project, String branchName) { } @Override - @Transactional public void createOrUpdateBranchIndex( Project project, String branchName, @@ -594,7 +885,7 @@ public boolean updateBranchIndex( return false; } - if (!isRagIndexReady(project)) { + if (!isRagIndexReady(project) && !usesExactGenerations(project)) { log.warn("Cannot update branch index - base RAG index not ready for project={}", project.getId()); return false; } @@ -603,7 +894,7 @@ public boolean updateBranchIndex( && !shouldHaveBranchIndex(project, targetBranch)) { log.info("Skipping branch index update for non-retained branch: project={}, branch={}", project.getId(), targetBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "state", "rag_skipped", "message", "Branch is not configured as a retained RAG branch")); @@ -632,6 +923,20 @@ public boolean updateBranchIndex( VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); String targetCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); + if (usesExactGenerations(project)) { + if (isExactGenerationCurrent( + project, targetBranch, targetCommit, false)) { + log.info("Exact branch generation already represents project={}, branch={}, commit={}", + project.getId(), targetBranch, targetCommit); + return true; + } + log.info("Binding exact branch update after acquiring its RAG lock: " + + "project={}, branch={}, target={}", + project.getId(), targetBranch, targetCommit); + return triggerIncrementalUpdate( + project, targetBranch, targetCommit, "", eventConsumer); + } + Optional completedBranchIndex = ragBranchIndexRepository .findByProjectIdAndBranchName(project.getId(), targetBranch) .filter(index -> index.getCommitHash() != null && !index.getCommitHash().isBlank()); @@ -655,7 +960,7 @@ public boolean updateBranchIndex( log.info("Seeding legacy branch index for project={}, branch={} (diff vs {})", project.getId(), targetBranch, baseBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "branch_index", "message", String.format("Calculating diff between '%s' and '%s'", baseBranch, targetBranch))); @@ -666,7 +971,7 @@ public boolean updateBranchIndex( if (rawDiff == null || rawDiff.isEmpty()) { log.info("No diff between {} and {} - branch has same content as base", baseBranch, targetBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "message", String.format("Branch '%s' has same content as '%s'", targetBranch, baseBranch))); if (usesExactGenerations(project)) { @@ -689,7 +994,7 @@ public boolean updateBranchIndex( } catch (Exception e) { log.error("Failed to update branch index for project={}, branch={}", project.getId(), targetBranch, e); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "error", "message", "Failed to update branch index: " + e.getMessage())); return false; @@ -714,7 +1019,7 @@ public boolean ensureBranchIndexForPrTarget( // Check if base index is ready if (!isRagIndexReady(project)) { log.warn("Cannot ensure branch index - base RAG index not ready for project={}", project.getId()); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "message", "Base RAG index not ready")); return false; @@ -771,7 +1076,7 @@ public boolean ensureBranchIndexForPrTarget( log.info("Fetching diff between base branch '{}' and target branch '{}' for project={}", baseBranch, targetBranch, project.getId()); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "branch_index", "message", String.format("Indexing branch '%s' (diff vs '%s')", targetBranch, baseBranch))); @@ -786,7 +1091,7 @@ public boolean ensureBranchIndexForPrTarget( if (rawDiff == null || rawDiff.isEmpty()) { log.info("No diff between '{}' and '{}' - branch has same content as base, using main index", baseBranch, targetBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "message", String.format("No changes between %s and %s - using main branch index", baseBranch, targetBranch))); @@ -806,7 +1111,7 @@ public boolean ensureBranchIndexForPrTarget( } catch (Exception e) { log.error("Failed to index branch data for project={}, branch={}: {}", project.getId(), targetBranch, e.getMessage(), e); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "state", "branch_error", "message", "Failed to index branch: " + e.getMessage())); @@ -819,24 +1124,31 @@ public boolean deleteBranchIndex( Project project, String branchName, Consumer> eventConsumer) { + return deleteBranchIndexWithOutcome(project, branchName, eventConsumer).successful(); + } + + private BranchIndexDeletionResult deleteBranchIndexWithOutcome( + Project project, + String branchName, + Consumer> eventConsumer) { if (!isRagEnabled(project)) { log.debug("RAG not enabled for project={}", project.getId()); - return false; + return new BranchIndexDeletionResult(false, false); } String baseBranch = getBaseBranch(project); if (branchName.equals(baseBranch)) { log.warn("Cannot delete main branch index for project={}", project.getId()); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "message", "Cannot delete main branch index")); - return false; + return new BranchIndexDeletionResult(false, false); } VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); if (vcsRepoBinding == null) { log.error("Project has no VcsRepoBinding configured"); - return false; + return new BranchIndexDeletionResult(false, false); } String workspaceSlug = vcsRepoBinding.getExternalNamespace(); @@ -845,12 +1157,13 @@ public boolean deleteBranchIndex( try { log.info("Deleting branch index for project={}, branch={}", project.getId(), branchName); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "branch_delete", "message", String.format("Deleting RAG index for branch '%s'", branchName))); boolean success; + boolean stopRemainingBranches = false; Optional trackedIndex = ragBranchIndexRepository .findByProjectIdAndBranchName(project.getId(), branchName); List generations = trackedIndex.isPresent() @@ -859,16 +1172,49 @@ public boolean deleteBranchIndex( trackedIndex.get().getId()) : List.of(); if (!generations.isEmpty()) { + generations = generations.stream() + .sorted(Comparator.comparing(generation -> + generation.getStatus() + == RagBranchIndexGenerationStatus.ACTIVE)) + .toList(); success = true; for (RagBranchIndexGeneration generation : generations) { - success &= ragPipelineClient.deleteBranch( - project.getWorkspace().getName(), project.getNamespace(), - branchName, generation.getCollectionName()); + if (generation.getStatus() == RagBranchIndexGenerationStatus.ACTIVE + && !success) { + // A failed older-target deletion must not leave the + // registry pointing at a destroyed active target. + break; + } + RagPipelineClient.BranchDeletionOutcome outcome = + ragPipelineClient.deleteBranchWithOutcome( + project.getWorkspace().getName(), project.getNamespace(), + branchName, generation.getCollectionName(), + generation.getRevision(), generation.getManifestDigest()); + if (!outcome.successful()) { + success = false; + if (isRagDisabled(outcome)) { + return new BranchIndexDeletionResult(false, true); + } + logBranchCleanupFailure(project, branchName, outcome); + if (outcome.shouldStopRemainingTargets()) { + stopRemainingBranches = true; + break; + } + } } } else { // Backward-compatible cleanup for the legacy shared collection. - success = ragPipelineClient.deleteBranch( - workspaceSlug, projectSlug, branchName); + RagPipelineClient.BranchDeletionOutcome outcome = + ragPipelineClient.deleteBranchWithOutcome( + workspaceSlug, projectSlug, branchName, null); + success = outcome.successful(); + if (!success) { + if (isRagDisabled(outcome)) { + return new BranchIndexDeletionResult(false, true); + } + logBranchCleanupFailure(project, branchName, outcome); + stopRemainingBranches = outcome.shouldStopRemainingTargets(); + } } if (success) { @@ -876,26 +1222,47 @@ public boolean deleteBranchIndex( ragBranchIndexRepository.deleteByProjectIdAndBranchName(project.getId(), branchName); log.info("Successfully deleted branch index for project={}, branch={}", project.getId(), branchName); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "success", "message", String.format("Deleted RAG index for branch '%s'", branchName))); - return true; + return new BranchIndexDeletionResult(true, false); } else { - log.warn("Failed to delete branch index from RAG pipeline for project={}, branch={}", - project.getId(), branchName); - return false; + return new BranchIndexDeletionResult(false, stopRemainingBranches); } } catch (Exception e) { log.error("Failed to delete branch index for project={}, branch={}", project.getId(), branchName, e); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "error", "message", "Failed to delete branch index: " + e.getMessage())); - return false; + return new BranchIndexDeletionResult(false, true); } } + private record BranchIndexDeletionResult( + boolean successful, + boolean shouldStopRemainingBranches) { + } + + private static void logBranchCleanupFailure( + Project project, + String branchName, + RagPipelineClient.BranchDeletionOutcome outcome) { + log.warn("Failed to delete branch RAG generation for project={}, branch={}, target={}: " + + "status={} detail={}", + project.getId(), branchName, outcome.targetLabel(), + outcome.statusCode() != null ? outcome.statusCode() : outcome.failure(), + outcome.detail()); + } + + private static boolean isRagDisabled( + RagPipelineClient.BranchDeletionOutcome outcome) { + return outcome.statusCode() == null + && outcome.failure() == RagPipelineClient.BranchDeletionFailure.TARGET + && "RAG disabled".equals(outcome.detail()); + } + @Override public Map cleanupStaleBranches( Project project, @@ -943,7 +1310,7 @@ public Map cleanupStaleBranches( log.info("Cleaning up {} stale branches for project={}: {}", staleBranches.size(), project.getId(), staleBranches); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "cleanup", "message", String.format("Cleaning up %d stale branches", staleBranches.size()))); @@ -953,11 +1320,20 @@ public Map cleanupStaleBranches( for (String branch : staleBranches) { try { - boolean success = deleteBranchIndex(project, branch, eventConsumer); - if (success) { + BranchIndexDeletionResult deletion = + deleteBranchIndexWithOutcome(project, branch, eventConsumer); + if (deletion.successful()) { deletedBranches.add(branch); } else { failedBranches.add(branch); + if (deletion.shouldStopRemainingBranches()) { + log.info("Stopping stale branch cleanup after a service-wide deletion " + + "failure; {} remaining branch(es) will retry next run", + staleBranches.size() + - deletedBranches.size() + - failedBranches.size()); + break; + } } } catch (Exception e) { log.warn("Failed to delete stale branch {} for project={}: {}", @@ -969,7 +1345,7 @@ public Map cleanupStaleBranches( log.info("Cleanup complete for project={}: deleted={}, failed={}", project.getId(), deletedBranches.size(), failedBranches.size()); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "success", "message", String.format("Cleaned up %d stale branches", deletedBranches.size()))); @@ -1029,7 +1405,7 @@ public boolean ensureRagIndexUpToDate( && !shouldCreateTransientBranchIndex(project, targetBranch)) { log.info("Skipping RAG preparation for non-indexed PR target: project={}, branch={}", project.getId(), targetBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "state", "rag_skipped", "message", "PR target is not configured for retained or temporary RAG indexing")); @@ -1051,7 +1427,7 @@ public boolean ensureRagIndexUpToDate( } catch (Exception e) { log.error("Failed to ensure RAG index up-to-date for project={}, targetBranch={}", project.getId(), targetBranch, e); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "state", "rag_error", "message", "Failed to update RAG index: " + e.getMessage())); @@ -1070,7 +1446,7 @@ private boolean ensureMainIndexUpToDate( String workspaceSlug, String repoSlug, Consumer> eventConsumer) throws IOException { - if (!isRagIndexReady(project)) { + if (!isRagIndexReady(project) && !usesExactGenerations(project)) { log.debug("Main RAG index not ready for project={}", project.getId()); return false; } @@ -1078,6 +1454,21 @@ private boolean ensureMainIndexUpToDate( // Get current commit on branch String currentCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, branchName); + if (usesExactGenerations(project)) { + if (isExactGenerationCurrent( + project, branchName, currentCommit, true)) { + log.debug("Exact main RAG generation and project checkpoint are up-to-date " + + "for project={}, commit={}", + project.getId(), currentCommit); + return true; + } + log.info("Exact main RAG generation requires reconciliation for project={}, " + + "branch={}, target={}", + project.getId(), branchName, currentCommit); + return triggerIncrementalUpdate( + project, branchName, currentCommit, "", eventConsumer); + } + // Get indexed commit from tracking service Optional indexStatus = ragIndexTrackingService.getIndexStatus(project); if (indexStatus.isEmpty()) { @@ -1087,17 +1478,6 @@ private boolean ensureMainIndexUpToDate( String indexedCommit = indexStatus.get().getIndexedCommitHash(); - if (usesExactGenerations(project) - && ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), branchName) - .map(RagBranchIndex::getActiveGeneration) - .isEmpty()) { - log.info("Creating first exact primary generation for project={}, branch={}, commit={}", - project.getId(), branchName, currentCommit); - return triggerIncrementalUpdate( - project, branchName, currentCommit, "", eventConsumer); - } - // If commits match, index is up to date if (currentCommit.equals(indexedCommit)) { log.debug("Main RAG index is up-to-date for project={}, commit={}", project.getId(), currentCommit); @@ -1116,7 +1496,7 @@ private boolean ensureMainIndexUpToDate( return true; } - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "rag_update", "message", String.format("Updating RAG index from %s to %s", @@ -1146,6 +1526,20 @@ private boolean ensureBranchIndexUpToDate( String currentCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); log.info("Current commit on branch '{}': {}", targetBranch, currentCommit); + if (usesExactGenerations(project)) { + if (isExactGenerationCurrent( + project, targetBranch, currentCommit, false)) { + log.info("Exact branch generation is up-to-date for project={}, branch={}, commit={}", + project.getId(), targetBranch, currentCommit); + return true; + } + log.info("Exact branch generation requires reconciliation for project={}, " + + "branch={}, target={}", + project.getId(), targetBranch, currentCommit); + return triggerIncrementalUpdate( + project, targetBranch, currentCommit, "", eventConsumer); + } + // Check if we have branch index tracking Optional branchIndexOpt = ragBranchIndexRepository .findByProjectIdAndBranchName(project.getId(), targetBranch); @@ -1167,11 +1561,6 @@ private boolean ensureBranchIndexUpToDate( log.info("Existing RagBranchIndex for project={}, branch={}: indexedCommit={}", project.getId(), targetBranch, indexedCommit); - if (usesExactGenerations(project) && branchIndex.getActiveGeneration() == null) { - return triggerIncrementalUpdate( - project, targetBranch, currentCommit, "", eventConsumer); - } - // If commits match, index is up to date if (currentCommit.equals(indexedCommit)) { log.info("Branch index is up-to-date for project={}, branch={}, commit={}", @@ -1201,7 +1590,7 @@ private boolean ensureBranchIndexUpToDate( return true; } - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "branch_update", "message", @@ -1214,6 +1603,36 @@ private boolean ensureBranchIndexUpToDate( project, targetBranch, currentCommit, rawDiff, eventConsumer); } + /** + * A cheap exact-generation readiness hint that is safe against transient + * cleanup. The atomic touch must win before the scalar projection is read; + * otherwise the locked trigger owns reconciliation. Primary callers also + * require the independently persisted project checkpoint to be current so + * a publication/status crash is repaired by the trigger's locked no-op. + */ + private boolean isExactGenerationCurrent( + Project project, + String branchName, + String targetRevision, + boolean requireProjectCheckpoint) { + if (ragBranchIndexRepository.markAccessedIfUnclaimed( + project.getId(), branchName, OffsetDateTime.now()) == 0) { + return false; + } + boolean generationCurrent = ragBranchIndexRepository + .findActiveGenerationCoordinates(project.getId(), branchName) + .map(RagBranchIndexRepository.ActiveGenerationCoordinates::getRevision) + .filter(targetRevision::equals) + .isPresent(); + if (!generationCurrent || !requireProjectCheckpoint) { + return generationCurrent; + } + return ragIndexTrackingService.getIndexStatus(project) + .map(RagIndexStatus::getIndexedCommitHash) + .filter(targetRevision::equals) + .isPresent(); + } + private boolean usesExactGenerations(Project project) { return branchIndexRegistryService != null && branchGenerationBuildService != null @@ -1232,4 +1651,19 @@ private RagBranchIndexKind indexKind(Project project, String branchName) { : RagBranchIndexKind.TRANSIENT; } + /** Observer delivery is never part of the durable RAG operation outcome. */ + private static void emitEvent( + Consumer> eventConsumer, + Map event) { + if (eventConsumer == null) { + return; + } + try { + eventConsumer.accept(event); + } catch (RuntimeException observerFailure) { + log.debug("RAG progress observer rejected event state={}: {}", + event.get("state"), observerFailure.getMessage()); + } + } + } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java new file mode 100644 index 00000000..75f53dbf --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java @@ -0,0 +1,153 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagIndexOperation; +import org.rostilos.codecrow.core.service.AnalysisJobService; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class BranchIndexBuildAdmissionServiceTest { + + @Mock private RagBranchIndexRegistryService registryService; + @Mock private AnalysisJobService jobService; + @Mock private RagIndexTrackingService trackingService; + + private BranchIndexBuildAdmissionService service; + private Project project; + private RagBranchIndexRegistryService.BuildRegistration registration; + + @BeforeEach + void setUp() { + service = new BranchIndexBuildAdmissionService( + registryService, jobService, trackingService); + project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + RagBranchIndex branchIndex = new RagBranchIndex( + project, "main", RagBranchIndexKind.PRIMARY); + branchIndex.setId(10L); + RagBranchIndexGeneration source = new RagBranchIndexGeneration( + branchIndex, "revision-a", "source-target", null, null, null); + source.setId(19L); + source.activate("source-manifest", 120, 240); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, "revision-b", "physical-target", source, null, null); + generation.setId(20L); + RagIndexOperation operation = new RagIndexOperation( + project, "main", null, "revision-b", "operation-key"); + operation.setId(30L); + operation.setGeneration(generation); + registration = new RagBranchIndexRegistryService.BuildRegistration( + branchIndex, generation, operation, false); + } + + @Test + void registersThenAtomicallyLinksAndStartsJobAndOperation() { + when(registryService.registerBuild( + eq(project), eq("main"), eq(RagBranchIndexKind.PRIMARY), + isNull(), eq("revision-b"), startsWith("exact-full-snapshot:automatic:"))) + .thenReturn(registration); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + when(jobService.createRagIndexJob( + project, false, JobTriggerSource.WEBHOOK, "main", "revision-b")) + .thenReturn(job); + + var admitted = service.admit( + project, "main", "revision-b", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "lock-owner-123", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + + assertThat(admitted.job()).isSameAs(job); + assertThat(admitted.preparedBuild().operationId()).isEqualTo(30L); + assertThat(admitted.preparedBuild().collectionTarget()) + .isEqualTo("physical-target"); + assertThat(admitted.preparedBuild().analysisLockKey()) + .isEqualTo("lock-owner-123"); + assertThat(admitted.statusAdmission()).isEqualTo( + BranchIndexBuildAdmissionService.ProjectStatusAdmission.UPDATING); + InOrder order = inOrder(registryService, jobService, trackingService); + order.verify(registryService).registerBuild( + eq(project), eq("main"), eq(RagBranchIndexKind.PRIMARY), + isNull(), eq("revision-b"), startsWith("exact-full-snapshot:automatic:")); + order.verify(trackingService).preparePublishedGenerationForUpdate( + project, "main", "revision-a", 120, 240); + order.verify(jobService).createRagIndexJob( + project, false, JobTriggerSource.WEBHOOK, "main", "revision-b"); + order.verify(registryService).startBuild(30L, 77L, "lock-owner-123"); + order.verify(jobService).startJob(job); + order.verify(trackingService).markUpdatingStarted( + project, "main", "revision-b", 77L); + } + + @Test + void rejectsPreviouslyCommittedAdmissionWithoutCreatingAnotherJob() { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), anyString())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + registration.branchIndex(), registration.generation(), + registration.operation(), true)); + + assertThatThrownBy(() -> service.admit( + project, "main", "revision-b", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "same-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already admitted"); + + verifyNoInteractions(jobService); + verify(registryService, never()).startBuild(anyLong(), any(), anyString()); + } + + @Test + void initialPrimaryAdmissionStartsIndexingWithoutInventingASourceCheckpoint() { + RagBranchIndexGeneration initial = new RagBranchIndexGeneration( + registration.branchIndex(), "revision-first", "initial-target", + null, null, null); + initial.setId(21L); + RagIndexOperation initialOperation = new RagIndexOperation( + project, "main", null, "revision-first", "initial-operation"); + initialOperation.setId(31L); + initialOperation.setGeneration(initial); + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + eq("revision-first"), anyString())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + registration.branchIndex(), initial, initialOperation, false)); + Job job = mock(Job.class); + when(job.getId()).thenReturn(78L); + when(jobService.createRagIndexJob( + project, true, JobTriggerSource.WEBHOOK, "main", "revision-first")) + .thenReturn(job); + + var admitted = service.admit( + project, "main", "revision-first", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "initial-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + + assertThat(admitted.statusAdmission()).isEqualTo( + BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING); + verify(trackingService).markIndexingStarted( + project, "main", "revision-first", 78L); + verify(trackingService, never()).preparePublishedGenerationForUpdate( + any(), anyString(), anyString(), any(), any()); + verify(trackingService, never()).markUpdatingStarted( + any(), anyString(), anyString(), any()); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java index 4f19bbfc..555e553f 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java @@ -7,6 +7,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; @@ -34,6 +35,10 @@ class BranchIndexGenerationBuildServiceTest { @Mock private BranchArchiveService archiveService; @Mock private RagPipelineClient pipelineClient; @Mock private RagBranchIndexRegistryService registryService; + @Mock private RagIndexOperationHeartbeatService heartbeatService; + @Mock private RagIndexOperationHeartbeatService.HeartbeatScope heartbeatScope; + @Mock private AnalysisLockService analysisLockService; + @Mock private AnalysisLockService.LockLease lockLease; private BranchIndexGenerationBuildService service; private Project project; @@ -43,7 +48,8 @@ class BranchIndexGenerationBuildServiceTest { @BeforeEach void setUp() { service = new BranchIndexGenerationBuildService( - archiveService, pipelineClient, registryService); + archiveService, pipelineClient, registryService, heartbeatService); + lenient().when(heartbeatService.start(anyLong())).thenReturn(heartbeatScope); project = new Project(); ReflectionTestUtils.setField(project, "id", 42L); RagBranchIndex branchIndex = new RagBranchIndex( @@ -93,10 +99,12 @@ project, new VcsConnection(), "provider-workspace", "repo", assertThat(result).containsEntry( "generation_manifest_sha256", "manifest-400"); verify(registryService).startBuild(30L, 77L, null); + verify(heartbeatService).start(30L); + verify(heartbeatScope).close(); verify(registryService).publish(30L, "manifest-400", 231, 400); verify(pipelineClient).publishGenerationAliases( "workspace", "namespace", "develop", "develop-400", - "opaque-generation-target", true, false); + "opaque-generation-target", "manifest-400", true, false); ArgumentCaptor snapshot = ArgumentCaptor.forClass(Path.class); verify(archiveService).downloadAndExtractSnapshotToDirectory( any(), eq("provider-workspace"), eq("repo"), @@ -104,6 +112,44 @@ project, new VcsConnection(), "provider-workspace", "repo", assertThat(Files.exists(snapshot.getValue())).isFalse(); } + @Test + void runtimeAliasFailureCannotReclassifyAPublishedGenerationAsFailed() + throws Exception { + when(pipelineClient.indexRepository( + anyString(), eq("workspace"), eq("namespace"), + eq("develop"), eq("develop-400"), eq(List.of()), + eq(List.of()), eq("opaque-generation-target"), + eq(false), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400)); + when(registryService.publish(30L, "manifest-400", 231, 400)) + .thenAnswer(ignored -> { + generation.activate("manifest-400", 231, 400); + return generation; + }); + doThrow(new IllegalStateException("observer alias adapter failed")) + .when(pipelineClient).publishGenerationAliases( + anyString(), anyString(), anyString(), anyString(), + anyString(), anyString(), anyBoolean(), anyBoolean()); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 30L, "opaque-generation-target", false, null, null); + + Map result = service.execute( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), prepared, null); + + assertThat(result).containsEntry("generation_manifest_sha256", "manifest-400"); + verify(registryService).publish(30L, "manifest-400", 231, 400); + verify(registryService, never()).fail(anyLong(), anyString()); + } + @Test void missingManifestFailsOperationAndDoesNotPublish() throws Exception { when(registryService.registerBuild(any(), anyString(), any(), isNull(), @@ -128,6 +174,8 @@ project, new VcsConnection(), "provider-workspace", "repo", verify(registryService).fail(30L, "RAG full branch generation has no manifest digest"); + verify(heartbeatService).start(30L); + verify(heartbeatScope).close(); verify(registryService, never()).publish(anyLong(), anyString(), anyInt(), anyInt()); } @@ -155,13 +203,13 @@ project, new VcsConnection(), "provider-workspace", "repo", void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws Exception { when(registryService.registerBuild( eq(project), eq("develop"), eq(RagBranchIndexKind.DURABLE), - isNull(), eq("develop-400"), eq("operator-refresh:77"))) + isNull(), eq("develop-400"), startsWith("full-snapshot:job:77:"))) .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( generation.getBranchIndex(), generation, operation, false)); when(pipelineClient.indexRepository( anyString(), anyString(), anyString(), eq("develop"), eq("develop-400"), anyList(), anyList(), eq("opaque-generation-target"), - eq(false), eq(false), any())) + eq(false), eq(false), eq(true), any(Runnable.class), any())) .thenReturn(Map.of( "generation_manifest_sha256", "fresh-manifest", "document_count", 231, @@ -183,9 +231,83 @@ void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws verify(archiveService).downloadAndExtractSnapshotToDirectory( any(), eq("provider-workspace"), eq("repo"), eq("develop-400"), isNull(), any()); verify(registryService).publish(30L, "fresh-manifest", 231, 400); + verify(pipelineClient).indexRepository( + anyString(), eq("workspace"), eq("namespace"), eq("develop"), + eq("develop-400"), anyList(), anyList(), + eq("opaque-generation-target"), eq(false), eq(false), eq(true), + any(Runnable.class), any()); verify(pipelineClient).publishGenerationAliases( "workspace", "namespace", "develop", "develop-400", - "opaque-generation-target", true, false); + "opaque-generation-target", "fresh-manifest", true, false); + } + + @Test + void streamFailureBeforeOwnershipAdmissionLeavesCleanupWithJava() throws Exception { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), startsWith("full-snapshot:job:77:"))) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + ArgumentCaptor snapshot = ArgumentCaptor.forClass(String.class); + when(pipelineClient.indexRepository( + snapshot.capture(), anyString(), anyString(), anyString(), anyString(), + anyList(), anyList(), anyString(), anyBoolean(), anyBoolean(), + eq(true), any(Runnable.class), any())) + .thenThrow(new IOException("connection refused before admission")); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + assertThatThrownBy(() -> service.rebuild( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), 77L, ignored -> { })) + .isInstanceOf(IOException.class) + .hasMessageContaining("before admission"); + + assertThat(Files.exists(Path.of(snapshot.getValue()))).isFalse(); + verify(registryService).fail(30L, "connection refused before admission"); + } + + @Test + void streamAdmissionRelinquishesJavaSnapshotCleanup() throws Exception { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), startsWith("full-snapshot:job:77:"))) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + ArgumentCaptor snapshot = ArgumentCaptor.forClass(String.class); + when(pipelineClient.indexRepository( + snapshot.capture(), anyString(), anyString(), anyString(), anyString(), + anyList(), anyList(), anyString(), anyBoolean(), anyBoolean(), + eq(true), any(Runnable.class), any())) + .thenAnswer(invocation -> { + ((Runnable) invocation.getArgument(11)).run(); + throw new IOException("stream disconnected after admission"); + }); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + assertThatThrownBy(() -> service.rebuild( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), 77L, ignored -> { })) + .isInstanceOf(IOException.class) + .hasMessageContaining("after admission"); + + Path retainedSnapshot = Path.of(snapshot.getValue()); + assertThat(retainedSnapshot).exists(); + try (var paths = Files.walk(retainedSnapshot)) { + paths.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException cleanupFailure) { + throw new RuntimeException(cleanupFailure); + } + }); + } + verify(registryService).fail(30L, "stream disconnected after admission"); } @Test @@ -215,7 +337,49 @@ void staleCompletedGenerationDoesNotPublishReadableAliases() throws Exception { List.of(), List.of()); verify(pipelineClient, never()).publishGenerationAliases( - anyString(), anyString(), anyString(), anyString(), anyString(), + anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyBoolean(), anyBoolean()); } + + @Test + void admittedBuildMaintainsAndConfirmsBranchLockBeforePublication() throws Exception { + service = new BranchIndexGenerationBuildService( + archiveService, pipelineClient, registryService, heartbeatService, + analysisLockService, 12); + when(analysisLockService.maintainLockLease("rag-lock", 12)) + .thenReturn(lockLease); + when(lockLease.confirmOwnership()).thenReturn(true); + when(pipelineClient.indexRepository( + anyString(), anyString(), anyString(), eq("develop"), eq("develop-400"), + anyList(), anyList(), eq("opaque-generation-target"), + eq(false), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400)); + when(registryService.publish(30L, "manifest-400", 231, 400)) + .thenAnswer(ignored -> { + generation.activate("manifest-400", 231, 400); + return generation; + }); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 30L, "opaque-generation-target", false, null, "rag-lock"); + + service.execute( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), prepared, null); + + var order = inOrder(pipelineClient, lockLease, registryService); + order.verify(pipelineClient).indexRepository( + anyString(), anyString(), anyString(), anyString(), anyString(), + anyList(), anyList(), anyString(), anyBoolean(), anyBoolean()); + order.verify(lockLease).confirmOwnership(); + order.verify(registryService).publish(30L, "manifest-400", 231, 400); + verify(lockLease).close(); + } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java new file mode 100644 index 00000000..512d545a --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java @@ -0,0 +1,103 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; +import org.rostilos.codecrow.core.model.project.config.RagConfig; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; +import org.rostilos.codecrow.core.service.AnalysisJobService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.VcsClientProvider; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class BranchIndexMaintenanceServiceTest { + + @Test + void observerAndLockCleanupFailuresCannotReverseAPublishedBuild() throws Exception { + RagOperationsService ragOperations = mock(RagOperationsService.class); + VcsClientProvider vcsClients = mock(VcsClientProvider.class); + BranchIndexGenerationBuildService builds = mock( + BranchIndexGenerationBuildService.class); + BranchIndexBuildAdmissionService admissions = mock( + BranchIndexBuildAdmissionService.class); + RagIndexTrackingService tracking = mock(RagIndexTrackingService.class); + AnalysisLockService locks = mock(AnalysisLockService.class); + AnalysisJobService jobs = mock(AnalysisJobService.class); + BranchIndexMaintenanceService service = new BranchIndexMaintenanceService( + ragOperations, vcsClients, builds, admissions, tracking, locks, jobs, + Runnable::run, 1); + + Project project = mock(Project.class); + when(project.getId()).thenReturn(42L); + when(project.getConfiguration()).thenReturn(new ProjectConfig( + false, "main", null, + new RagConfig(true, "main", List.of(), List.of()))); + VcsRepoBinding binding = mock(VcsRepoBinding.class); + VcsConnection connection = new VcsConnection(); + when(project.getVcsRepoBinding()).thenReturn(binding); + when(binding.getVcsConnection()).thenReturn(connection); + when(binding.getExternalNamespace()).thenReturn("workspace"); + when(binding.getExternalRepoSlug()).thenReturn("repository"); + VcsClient vcs = mock(VcsClient.class); + when(vcsClients.getClient(connection)).thenReturn(vcs); + when(vcs.getLatestCommitHash("workspace", "repository", "main")) + .thenReturn("revision-a"); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(locks.acquireLock(eq(project), eq("main"), any(), eq("revision-a"), isNull())) + .thenReturn(Optional.of("rag-lock")); + + Job job = mock(Job.class); + when(job.getId()).thenReturn(91L); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 81L, "physical-generation", false, null, "rag-lock"); + when(admissions.admit( + eq(project), eq("main"), eq("revision-a"), + eq(RagBranchIndexKind.PRIMARY), any(), eq("rag-lock"), any())) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); + when(builds.execute( + eq(project), eq(connection), eq("workspace"), eq("repository"), + eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), + eq(List.of()), eq(List.of()), eq(prepared), any())) + .thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + Consumer> progress = invocation.getArgument(10); + progress.accept(Map.of("stage", "indexing", "message", "halfway")); + return Map.of("document_count", 12, "chunk_count", 34); + }); + doThrow(new IllegalStateException("lock database unavailable")) + .when(locks).releaseLock("rag-lock"); + + Map outcome = service.rebuild( + project, + "main", + false, + ignored -> { + throw new IllegalStateException("observer disconnected"); + }); + + assertThat(outcome.get("branches")).isEqualTo(List.of("main")); + assertThat(outcome.get("failedBranches")).isEqualTo(Map.of()); + verify(tracking).reconcilePublishedGeneration( + project, "main", "revision-a", 12, 34, 91L); + verify(jobs).completeJob(job, Map.of("branch", "main", "revision", "revision-a")); + verify(jobs, never()).failJob(any(), anyString()); + verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); + verify(locks).releaseLock("rag-lock"); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseServiceTest.java new file mode 100644 index 00000000..fef214ab --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseServiceTest.java @@ -0,0 +1,105 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.service.JobService; + +import java.time.OffsetDateTime; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +class LegacyRagJobLeaseServiceTest { + + @Test + void renewsSynchronouslyThenHeartbeatsUntilClosed() { + JobService jobs = mock(JobService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + ScheduledFuture scheduled = mock(ScheduledFuture.class); + when(jobs.renewLegacyRagJobLease(eq(91L), any(), any())) + .thenReturn(true); + doReturn(scheduled).when(executor).scheduleWithFixedDelay( + any(Runnable.class), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); + LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( + jobs, executor, 60, 15); + + LegacyRagJobLeaseService.JobLease lease = service.start(91L); + + var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(executor).scheduleWithFixedDelay( + heartbeat.capture(), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); + verify(jobs, times(1)).renewLegacyRagJobLease(eq(91L), any(), any()); + heartbeat.getValue().run(); + verify(jobs, times(2)).renewLegacyRagJobLease(eq(91L), any(), any()); + assertThat(lease.confirmOwnership()).isTrue(); + verify(jobs, times(3)).renewLegacyRagJobLease(eq(91L), any(), any()); + + lease.close(); + verify(scheduled).cancel(false); + } + + @Test + void refusesWorkWhenTheInitialLeaseCannotBeProven() { + JobService jobs = mock(JobService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + when(jobs.renewLegacyRagJobLease(eq(92L), any(), any())) + .thenReturn(false); + LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( + jobs, executor, 60, 15); + + LegacyRagJobLeaseService.JobLease lease = service.start(92L); + + assertThat(lease.isOwnershipLost()).isTrue(); + assertThat(lease.confirmOwnership()).isFalse(); + verifyNoInteractions(executor); + } + + @Test + void transientHeartbeatFailureDoesNotDiscardAStillValidLease() { + JobService jobs = mock(JobService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + ScheduledFuture scheduled = mock(ScheduledFuture.class); + when(jobs.renewLegacyRagJobLease(eq(93L), any(), any())) + .thenReturn(true) + .thenThrow(new IllegalStateException("temporary database outage")) + .thenReturn(true); + doReturn(scheduled).when(executor).scheduleWithFixedDelay( + any(Runnable.class), anyLong(), anyLong(), any()); + LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( + jobs, executor, 60, 15); + + LegacyRagJobLeaseService.JobLease lease = service.start(93L); + var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(executor).scheduleWithFixedDelay( + heartbeat.capture(), anyLong(), anyLong(), any()); + + heartbeat.getValue().run(); + assertThat(lease.isOwnershipLost()).isFalse(); + assertThat(lease.confirmOwnership()).isTrue(); + } + + @Test + void renewalUsesARealLeaseWindow() { + JobService jobs = mock(JobService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + when(jobs.renewLegacyRagJobLease(eq(94L), any(), any())) + .thenReturn(true); + doReturn(mock(ScheduledFuture.class)).when(executor).scheduleWithFixedDelay( + any(Runnable.class), anyLong(), anyLong(), any()); + LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( + jobs, executor, 60, 15); + + service.start(94L); + + var validAfter = org.mockito.ArgumentCaptor.forClass(OffsetDateTime.class); + var renewedAt = org.mockito.ArgumentCaptor.forClass(OffsetDateTime.class); + verify(jobs).renewLegacyRagJobLease( + eq(94L), validAfter.capture(), renewedAt.capture()); + assertThat(validAfter.getValue()).isEqualTo( + renewedAt.getValue().minusSeconds(60)); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryServiceTest.java new file mode 100644 index 00000000..89397664 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryServiceTest.java @@ -0,0 +1,151 @@ +package org.rostilos.codecrow.ragengine.branch; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.service.JobService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class LegacyRagJobRecoveryServiceTest { + private JobService jobs; + private RagIndexTrackingService tracking; + private LegacyRagJobRecoveryService recovery; + private JobRepository.LegacyRagJobRecoveryCoordinates coordinates; + + @BeforeEach + void setUp() { + jobs = mock(JobService.class); + tracking = mock(RagIndexTrackingService.class); + recovery = new LegacyRagJobRecoveryService( + jobs, tracking, 120, 100); + coordinates = mock( + JobRepository.LegacyRagJobRecoveryCoordinates.class); + when(coordinates.getJobId()).thenReturn(91L); + when(coordinates.getProjectId()).thenReturn(42L); + when(coordinates.getBranchName()).thenReturn("main"); + when(coordinates.getCommitHash()).thenReturn("commit-a"); + } + + @Test + void staleProducerIsAtomicallyFailedAndItsOwnedStatusIsRestored() { + Job job = mock(Job.class); + when(jobs.findAbandonedLegacyRagJobs(any(OffsetDateTime.class), eq(100))) + .thenReturn(List.of(coordinates)); + when(jobs.failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), anyString())) + .thenReturn(true); + when(jobs.findById(91L)).thenReturn(Optional.of(job)); + + recovery.failAbandonedJobs(); + + verify(jobs).failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), contains("stopped heartbeating")); + verify(jobs).recordExternallyFailedJob( + eq(job), eq("rag_recovery"), + contains("last completed checkpoint was preserved")); + verify(tracking).recoverAbandonedIncrementalUpdate( + eq(42L), eq(91L), contains("stopped heartbeating")); + } + + @Test + void heartbeatWinningTheCasPreservesTheLiveProducer() { + when(jobs.findAbandonedLegacyRagJobs(any(OffsetDateTime.class), eq(100))) + .thenReturn(List.of(coordinates)); + when(jobs.failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), anyString())) + .thenReturn(false); + + recovery.failAbandonedJobs(); + + verify(jobs, never()).findById(anyLong()); + verifyNoInteractions(tracking); + } + + @Test + void statusOwnedByANewerJobIsPreserved() { + when(jobs.findAbandonedLegacyRagJobs(any(OffsetDateTime.class), eq(100))) + .thenReturn(List.of(coordinates)); + when(jobs.failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), anyString())) + .thenReturn(true); + when(tracking.recoverAbandonedIncrementalUpdate( + eq(42L), eq(91L), anyString())).thenReturn(false); + + recovery.failAbandonedJobs(); + + verify(tracking).recoverAbandonedIncrementalUpdate( + eq(42L), eq(91L), anyString()); + } + + @Test + void failedJobProjectionDriftIsRetriedOnLaterScan() { + when(coordinates.getErrorMessage()).thenReturn("prior producer failure"); + when(jobs.findFailedLegacyRagJobsWithActiveStatus(100)) + .thenReturn(List.of(coordinates)); + + recovery.failAbandonedJobs(); + + verify(tracking).recoverAbandonedIncrementalUpdate( + 42L, 91L, "prior producer failure"); + verify(jobs, never()).failAbandonedLegacyRagJob( + anyLong(), any(), anyString()); + } + + @Test + void scanOutageLogsOneTransitionThenDebugAndOneRecovery() { + Logger logger = (Logger) org.slf4j.LoggerFactory.getLogger( + LegacyRagJobRecoveryService.class); + Level previousLevel = logger.getLevel(); + logger.setLevel(Level.DEBUG); + ListAppender logs = new ListAppender<>(); + logs.start(); + logger.addAppender(logs); + try { + when(jobs.findAbandonedLegacyRagJobs( + any(OffsetDateTime.class), eq(100))) + .thenThrow(new IllegalStateException("database unavailable")) + .thenThrow(new IllegalStateException("database unavailable")) + .thenReturn(List.of()); + + recovery.failAbandonedJobs(); + recovery.failAbandonedJobs(); + recovery.failAbandonedJobs(); + + long warnings = logs.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .filter(event -> event.getFormattedMessage() + .contains("recovery degraded")) + .count(); + long repeats = logs.list.stream() + .filter(event -> event.getLevel() == Level.DEBUG) + .filter(event -> event.getFormattedMessage() + .contains("remains degraded")) + .count(); + long recoveries = logs.list.stream() + .filter(event -> event.getLevel() == Level.INFO) + .filter(event -> event.getFormattedMessage() + .contains("recovery scan recovered")) + .count(); + assertThat(warnings).isEqualTo(1); + assertThat(repeats).isEqualTo(1); + assertThat(recoveries).isEqualTo(1); + } finally { + logger.detachAppender(logs); + logs.stop(); + logger.setLevel(previousLevel); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionServiceTest.java new file mode 100644 index 00000000..d5ae45f6 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionServiceTest.java @@ -0,0 +1,123 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; +import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.persistence.repository.analysis.RagIndexStatusRepository; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.OffsetDateTime; +import java.util.Optional; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class LegacyRagUpdateCompletionServiceTest { + private JobRepository jobs; + private RagIndexTrackingService tracking; + private RagIndexStatusRepository statuses; + private RagBranchIndexRepository branches; + private LegacyRagUpdateCompletionService completion; + private Project project; + + @BeforeEach + void setUp() { + jobs = mock(JobRepository.class); + tracking = mock(RagIndexTrackingService.class); + statuses = mock(RagIndexStatusRepository.class); + branches = mock(RagBranchIndexRepository.class); + completion = new LegacyRagUpdateCompletionService( + jobs, tracking, statuses, branches); + project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + } + + @Test + void recoveryWinningTheJobCasPreventsEveryCheckpointWrite() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) + .thenReturn(0); + + boolean completed = completion.complete( + project, "main", "commit-b", 91L, validAfter, + true, 2, 1, 30, Set.of("old.java")); + + assertThat(completed).isFalse(); + verifyNoInteractions(tracking, branches); + } + + @Test + void ownedJobAndAllCheckpointsAdvanceTogether() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + RagBranchIndex branch = new RagBranchIndex(project, "main"); + branch.setCommitHash("commit-a"); + branch.setDeletedFiles(new java.util.HashSet<>(Set.of("older.java"))); + when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) + .thenReturn(1); + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.UPDATING); + status.setActiveJobId(91L); + when(statuses.findByProjectIdForUpdate(42L)) + .thenReturn(Optional.of(status)); + when(branches.findByProjectIdAndBranchNameForUpdate(42L, "main")) + .thenReturn(Optional.of(branch)); + + boolean completed = completion.complete( + project, "main", "commit-b", 91L, validAfter, + true, 2, 1, 30, Set.of("old.java")); + + assertThat(completed).isTrue(); + verify(tracking).markUpdatingCompleted( + project, "main", "commit-b", 2, 1, 30, 91L); + verify(branches).save(argThat(saved -> + "commit-b".equals(saved.getCommitHash()) + && saved.getDeletedFiles().equals( + Set.of("older.java", "old.java")))); + } + + @Test + void newerProjectStatusOwnerFencesEveryCheckpointWrite() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.UPDATING); + status.setActiveJobId(92L); + when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) + .thenReturn(1); + when(statuses.findByProjectIdForUpdate(42L)) + .thenReturn(Optional.of(status)); + + org.assertj.core.api.Assertions.assertThatThrownBy(() -> completion.complete( + project, "main", "commit-b", 91L, validAfter, + true, 2, 1, 30, Set.of("old.java"))) + .isInstanceOf(LegacyRagUpdateCompletionService + .LegacyRagCompletionConflictException.class); + + verifyNoInteractions(tracking, branches); + } + + @Test + void nonPrimaryBranchDoesNotTouchProjectStatus() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) + .thenReturn(1); + when(branches.findByProjectIdAndBranchNameForUpdate(42L, "feature")) + .thenReturn(Optional.empty()); + + assertThat(completion.complete( + project, "feature", "commit-b", 91L, validAfter, + false, 2, 1, null, Set.of())).isTrue(); + + verifyNoInteractions(tracking); + verify(branches).save(argThat(saved -> + "feature".equals(saved.getBranchName()) + && "commit-b".equals(saved.getCommitHash()))); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java index 7a63647e..865b935d 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java @@ -1,12 +1,20 @@ package org.rostilos.codecrow.ragengine.branch; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.Test; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.List; +import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.*; class RagBranchOperatorAliasReconciliationServiceTest { @@ -21,28 +29,225 @@ void restoresReadableAliasesForDurableAndPrimaryGenerationsOnly() throws Excepti var primary = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); var durable = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); when(repository.findOperatorAliasCandidates()).thenReturn(List.of(primary, durable)); + stubCurrent(repository, primary, durable); service.reconcileActiveGenerationAliases(); verify(client).publishGenerationAliases( - "workspace", "project", "main", "revision", "main-target", true, true); + "workspace", "project", "main", "revision", "main-target", "manifest-main", true, true); verify(client).publishGenerationAliases( - "workspace", "project", "develop", "revision", "develop-target", true, false); + "workspace", "project", "develop", "revision", "develop-target", "manifest-develop", true, false); verifyNoMoreInteractions(client); } + @Test + void transportFailureStopsRemainingCandidatesUntilNextScheduledRun() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var first = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + var second = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(first, second)); + stubCurrent(repository, first, second); + doThrow(new IOException("timeout")).when(client).publishGenerationAliases( + eq("workspace"), eq("project"), eq("main"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + + service.reconcileActiveGenerationAliases(); + + verify(client).publishGenerationAliases( + "workspace", "project", "main", "revision", "main-target", "manifest-main", true, true); + verify(client, never()).publishGenerationAliases( + eq("workspace"), eq("project"), eq("develop"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + } + + @Test + void rateLimitStopsRemainingCandidatesUntilNextScheduledRun() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var first = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + var second = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(first, second)); + stubCurrent(repository, first, second); + doThrow(new RagPipelineClient.RagApiException(429, "rate limited")) + .when(client).publishGenerationAliases( + eq("workspace"), eq("project"), eq("main"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + + service.reconcileActiveGenerationAliases(); + + verify(client, never()).publishGenerationAliases( + eq("workspace"), eq("project"), eq("develop"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + } + + @Test + void generationValidationRejectionDoesNotBlockOtherCandidates() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var first = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + var second = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(first, second)); + stubCurrent(repository, first, second); + doThrow(new RagPipelineClient.RagApiException(409, "manifest mismatch")) + .when(client).publishGenerationAliases( + eq("workspace"), eq("project"), eq("main"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + + service.reconcileActiveGenerationAliases(); + + verify(client).publishGenerationAliases( + "workspace", "project", "develop", "revision", "develop-target", "manifest-develop", true, false); + } + + @Test + void repairsAliasWithNewGenerationWhenRegistryChangesDuringPublication() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var generationA = candidate( + "main", RagBranchIndexKind.PRIMARY, "generation-a", 10L, 100L); + var generationB = candidate( + "main", RagBranchIndexKind.PRIMARY, "generation-b", 10L, 101L); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(generationA)); + when(repository.findOperatorAliasCandidateById(10L)).thenReturn( + Optional.of(generationA), Optional.of(generationB), Optional.of(generationB)); + + service.reconcileActiveGenerationAliases(); + + var inOrder = inOrder(client); + inOrder.verify(client).publishGenerationAliases( + "workspace", "project", "main", "revision", "generation-a", "manifest-main", true, true); + inOrder.verify(client).publishGenerationAliases( + "workspace", "project", "main", "revision", "generation-b", "manifest-main", true, true); + } + + @Test + void persistentOutageWarnsOnceAndLogsRecoveryOnce() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var candidate = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(candidate)); + stubCurrent(repository, candidate); + doThrow(new IOException("timeout")) + .doThrow(new IOException("timeout")) + .doNothing() + .when(client).publishGenerationAliases( + anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + Logger logger = (Logger) LoggerFactory.getLogger( + RagBranchOperatorAliasReconciliationService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + try { + service.reconcileActiveGenerationAliases(); + service.reconcileActiveGenerationAliases(); + service.reconcileActiveGenerationAliases(); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage)) + .containsExactly("RAG alias reconciliation stopped after a transport failure; " + + "remaining candidates will retry next run: timeout"); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.INFO) + .map(ILoggingEvent::getFormattedMessage)) + .anyMatch(message -> message.contains("retry next run: timeout")) + .contains("RAG alias reconciliation recovered"); + } + + @Test + void persistentCandidateRejectionWarnsOnlyOnDegradedTransition() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var candidate = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(candidate)); + stubCurrent(repository, candidate); + doThrow(new RagPipelineClient.RagApiException(409, "manifest mismatch")) + .doThrow(new RagPipelineClient.RagApiException(409, "manifest mismatch")) + .doNothing() + .when(client).publishGenerationAliases( + anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + Logger logger = (Logger) LoggerFactory.getLogger( + RagBranchOperatorAliasReconciliationService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + try { + service.reconcileActiveGenerationAliases(); + service.reconcileActiveGenerationAliases(); + service.reconcileActiveGenerationAliases(); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage)) + .containsExactly("RAG alias reconciliation completed with 1 rejected candidate(s); " + + "they will retry next run"); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.INFO) + .map(ILoggingEvent::getFormattedMessage)) + .contains("RAG alias reconciliation recovered"); + } + + private static void stubCurrent( + RagBranchIndexRepository repository, + RagBranchIndexRepository.OperatorAliasCandidate... candidates) { + for (var candidate : candidates) { + when(repository.findOperatorAliasCandidateById(candidate.getBranchIndexId())) + .thenReturn(Optional.of(candidate)); + } + } + private static RagBranchIndexRepository.OperatorAliasCandidate candidate( String branch, RagBranchIndexKind kind, String target) { var candidate = mock(RagBranchIndexRepository.OperatorAliasCandidate.class); + when(candidate.getBranchIndexId()).thenReturn( + "main".equals(branch) ? 10L : 20L); + when(candidate.getGenerationId()).thenReturn( + "main".equals(branch) ? 100L : 200L); when(candidate.getProjectId()).thenReturn(1L); when(candidate.getWorkspaceName()).thenReturn("workspace"); when(candidate.getProjectNamespace()).thenReturn("project"); when(candidate.getBranchName()).thenReturn(branch); when(candidate.getRevision()).thenReturn("revision"); when(candidate.getCollectionName()).thenReturn(target); + when(candidate.getManifestDigest()).thenReturn("manifest-" + branch); when(candidate.getIndexKind()).thenReturn(kind); return candidate; } + + private static RagBranchIndexRepository.OperatorAliasCandidate candidate( + String branch, + RagBranchIndexKind kind, + String target, + Long branchIndexId, + Long generationId) { + var candidate = candidate(branch, kind, target); + when(candidate.getBranchIndexId()).thenReturn(branchIndexId); + when(candidate.getGenerationId()).thenReturn(generationId); + return candidate; + } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatServiceTest.java new file mode 100644 index 00000000..cc081c31 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatServiceTest.java @@ -0,0 +1,62 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +class RagIndexOperationHeartbeatServiceTest { + + @Test + void scopeSchedulesHeartbeatsAndCancelsOnClose() { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + @SuppressWarnings("unchecked") + ScheduledFuture scheduled = mock(ScheduledFuture.class); + doReturn(scheduled).when(executor).scheduleAtFixedRate( + any(Runnable.class), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); + RagIndexOperationHeartbeatService service = + new RagIndexOperationHeartbeatService(registry, executor, 15L); + + RagIndexOperationHeartbeatService.HeartbeatScope scope = service.start(91L); + + var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(executor).scheduleAtFixedRate( + heartbeat.capture(), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); + heartbeat.getValue().run(); + verify(registry).heartbeatBuild(91L); + + scope.close(); + verify(scheduled).cancel(false); + service.close(); + verify(executor).shutdownNow(); + } + + @Test + void transientHeartbeatFailureDoesNotEscapeSchedulerTask() { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + @SuppressWarnings("unchecked") + ScheduledFuture scheduled = mock(ScheduledFuture.class); + doReturn(scheduled).when(executor).scheduleAtFixedRate( + any(Runnable.class), anyLong(), anyLong(), any()); + doThrow(new IllegalStateException("database unavailable")) + .when(registry).heartbeatBuild(92L); + RagIndexOperationHeartbeatService service = + new RagIndexOperationHeartbeatService(registry, executor, 15L); + + service.start(92L); + + var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(executor).scheduleAtFixedRate( + heartbeat.capture(), anyLong(), anyLong(), any()); + heartbeat.getValue().run(); + verify(registry).heartbeatBuild(92L); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java index fa221152..19af8a79 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java @@ -1,5 +1,9 @@ package org.rostilos.codecrow.ragengine.branch; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; @@ -7,17 +11,21 @@ import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobStatus; +import org.rostilos.codecrow.core.model.job.JobType; import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.rag.RagIndexOperation; import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagIndexOperationRepository; import org.rostilos.codecrow.core.service.JobService; import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.slf4j.LoggerFactory; import java.time.OffsetDateTime; import java.util.List; import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @@ -30,7 +38,7 @@ class RagIndexOperationRecoveryServiceTest { private RagOperationsService ragOperations; private AnalysisLockService locks; private RagIndexOperationRecoveryService recovery; - private RagIndexOperation operation; + private RagIndexOperationRepository.RecoveryOperationProjection operation; private Project project; @BeforeEach @@ -46,20 +54,25 @@ void setUp() { project = mock(Project.class); when(project.getId()).thenReturn(42L); - operation = mock(RagIndexOperation.class); - when(operation.getId()).thenReturn(81L); - when(operation.getProject()).thenReturn(project); + operation = mock( + RagIndexOperationRepository.RecoveryOperationProjection.class); + when(operation.getOperationId()).thenReturn(81L); + when(operation.getProjectId()).thenReturn(42L); when(operation.getBranchName()).thenReturn("main"); when(operation.getToRevision()).thenReturn("commit-a"); when(operation.getJobId()).thenReturn(91L); when(operation.getAnalysisLockKey()).thenReturn("rag-lock-owner-91"); when(registry.findFailedOperationsWithActiveProjections()) .thenReturn(List.of()); + when(registry.findSucceededOperationsWithActiveProjections()) + .thenReturn(List.of()); } @Test void abandonedOperationTerminalizesJobPrimaryStatusAndLock() { Job job = new Job(); + job.setJobType(JobType.RAG_INCREMENTAL_INDEX); + job.setStatus(JobStatus.RUNNING); RagIndexStatus status = new RagIndexStatus(); status.setStatus(RagIndexingStatus.INDEXING); status.setActiveJobId(91L); @@ -168,4 +181,125 @@ void alreadyFailedOperationPreservesStatusOwnedByANewerJob() { verify(tracking, never()).markIncrementalUpdateFailed(any(), anyString(), any()); verify(locks).releaseLock("rag-lock-owner-91"); } + + @Test + void failedOperationNeverGuessesOwnershipOfAnOwnerlessStatus() { + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.UPDATING); + when(registry.findRecoverableOperations(any(OffsetDateTime.class))) + .thenReturn(List.of()); + when(registry.findFailedOperationsWithActiveProjections()) + .thenReturn(List.of(operation)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(tracking.getIndexStatus(project)).thenReturn(Optional.of(status)); + + recovery.failAbandonedOperations(); + + verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); + verify(tracking, never()).markIncrementalUpdateFailed(any(), anyString(), any()); + verify(locks).releaseLock("rag-lock-owner-91"); + } + + @Test + void publishedInitialOperationCompletesJobRepairsPrimaryStatusAndReleasesLock() { + RagIndexOperationRepository.SucceededOperationProjection published = + mock(RagIndexOperationRepository.SucceededOperationProjection.class); + when(published.getOperationId()).thenReturn(82L); + when(published.getProjectId()).thenReturn(42L); + when(published.getBranchName()).thenReturn("main"); + when(published.getToRevision()).thenReturn("commit-published"); + when(published.getJobId()).thenReturn(92L); + when(published.getAnalysisLockKey()).thenReturn("published-lock-owner"); + when(published.getFileCount()).thenReturn(214); + when(published.getChunkCount()).thenReturn(642); + when(published.getActiveGeneration()).thenReturn(true); + Job job = new Job(); + job.setJobType(JobType.RAG_INITIAL_INDEX); + job.setStatus(JobStatus.RUNNING); + when(registry.findRecoverableOperations(any())).thenReturn(List.of()); + when(registry.findSucceededOperationsWithActiveProjections()) + .thenReturn(List.of(published)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(jobs.findById(92L)).thenReturn(Optional.of(job)); + + recovery.failAbandonedOperations(); + + verify(tracking).reconcilePublishedGeneration( + project, "main", "commit-published", 214, 642, 92L); + verify(jobs).completeJob(job); + verify(jobs, never()).failJob(any(), anyString()); + verify(locks).releaseLock("published-lock-owner"); + } + + @Test + void publishedOperationWithOwnerlessStatusIsNotGuessedOrRegressed() { + RagIndexOperationRepository.SucceededOperationProjection published = + mock(RagIndexOperationRepository.SucceededOperationProjection.class); + when(published.getProjectId()).thenReturn(42L); + when(published.getBranchName()).thenReturn("main"); + when(published.getToRevision()).thenReturn("old-commit"); + when(published.getJobId()).thenReturn(92L); + when(published.getActiveGeneration()).thenReturn(true); + when(registry.findRecoverableOperations(any())).thenReturn(List.of()); + when(registry.findSucceededOperationsWithActiveProjections()) + .thenReturn(List.of(published)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(tracking.reconcilePublishedGeneration( + project, "main", "old-commit", 0, 0, 92L)) + .thenReturn(false); + + recovery.failAbandonedOperations(); + + verify(tracking).reconcilePublishedGeneration( + project, "main", "old-commit", 0, 0, 92L); + verifyNoMoreInteractions(tracking); + } + + @Test + void persistentProjectionFailuresEmitOneWarningThenDebugUntilRecovery() { + when(registry.findRecoverableOperations(any())).thenReturn(List.of()); + when(registry.findFailedOperationsWithActiveProjections()) + .thenReturn(List.of(operation), List.of(operation), List.of()); + when(jobs.findById(91L)).thenThrow(new IllegalStateException("database down")); + when(projects.findByIdWithFullDetails(42L)) + .thenThrow(new IllegalStateException("database down")); + doThrow(new IllegalStateException("database down")) + .when(locks).releaseLock("rag-lock-owner-91"); + Logger logger = (Logger) LoggerFactory.getLogger( + RagIndexOperationRecoveryService.class); + Level priorLevel = logger.getLevel(); + logger.setLevel(Level.DEBUG); + ListAppender events = new ListAppender<>(); + events.start(); + logger.addAppender(events); + try { + recovery.failAbandonedOperations(); + recovery.failAbandonedOperations(); + + reset(jobs, projects, locks); + recovery.failAbandonedOperations(); + } finally { + logger.detachAppender(events); + logger.setLevel(priorLevel); + events.stop(); + } + + List transitionEvents = events.list.stream() + .filter(event -> event.getFormattedMessage() + .contains("Exact RAG operation recovery")) + .toList(); + assertThat(transitionEvents.stream() + .filter(event -> event.getLevel() == Level.WARN)).hasSize(1); + assertThat(transitionEvents.stream() + .filter(event -> event.getLevel() == Level.DEBUG)).isNotEmpty(); + assertThat(transitionEvents.stream() + .filter(event -> event.getLevel() == Level.INFO + && event.getFormattedMessage().contains("scan recovered"))) + .hasSize(1); + assertThat(events.list.stream() + .filter(event -> event.getLevel() == Level.ERROR)).isEmpty(); + } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java index 072fe996..e628bb32 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java @@ -1,59 +1,326 @@ package org.rostilos.codecrow.ragengine.branch; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.project.config.RagConfig; -import org.rostilos.codecrow.core.model.rag.RagBranchIndex; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.springframework.test.util.ReflectionTestUtils; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.annotation.Transactional; import java.time.OffsetDateTime; import java.util.List; -import static org.mockito.Mockito.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; class RagTransientBranchIndexCleanupServiceTest { @Test - void deletesExpiredTransientGenerationUsingProjectTenantCoordinates() throws Exception { + void deletesExpiredGenerationOutsideSchedulerTransactionAndRechecksRegistry() throws Exception { RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); RagBranchIndexGenerationRepository generations = mock(RagBranchIndexGenerationRepository.class); RagPipelineClient pipeline = mock(RagPipelineClient.class); - Project project = new Project(); - ReflectionTestUtils.setField(project, "id", 42L); - project.setNamespace("namespace"); - Workspace workspace = new Workspace(); - workspace.setName("workspace"); - project.setWorkspace(workspace); - project.setConfiguration(new ProjectConfig( - false, "master", null, - new RagConfig(true, "master", null, null, - true, 30, List.of("develop"), true))); - RagBranchIndex index = new RagBranchIndex( - project, "release/candidate", RagBranchIndexKind.TRANSIENT); - index.setId(10L); - index.setLastAccessedAt(OffsetDateTime.now().minusDays(31)); - RagBranchIndexGeneration generation = mock(RagBranchIndexGeneration.class); - when(generation.getCollectionName()).thenReturn("opaque-transient-target"); - when(branches.findByIndexKind(RagBranchIndexKind.TRANSIENT)) - .thenReturn(List.of(index)); - when(generations.findByBranchIndexIdOrderByCreatedAtDesc(10L)) + var index = expiredCandidate(); + var generation = generation(100L, "opaque-transient-target"); + when(branches.findTransientCleanupCandidates()).thenReturn(List.of(index)); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) .thenReturn(List.of(generation)); - when(pipeline.deleteBranch( + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(pipeline.deleteBranchWithOutcome( "workspace", "namespace", "release/candidate", - "opaque-transient-target")) - .thenReturn(true); + "opaque-transient-target", "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success( + "opaque-transient-target")); + when(branches.deleteClaimedTransientById(eq(10L), anyString())) + .thenReturn(1); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + verify(branches).deleteClaimedTransientById(eq(10L), anyString()); + verify(pipeline).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", + "opaque-transient-target", "revision-100", "manifest-100"); + assertThat(RagTransientBranchIndexCleanupService.class + .getMethod("cleanupExpired") + .isAnnotationPresent(Transactional.class)) + .isFalse(); + } + + @Test + void serviceFailureStopsRemainingGenerationsAndWarnsOnlyOnDegradedTransition() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var first = generation(100L, "target-a"); + var second = generation(101L, "target-b"); + when(branches.findTransientCleanupCandidates()).thenReturn(List.of(index)); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(first, second)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "target-a", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "target-a", + RagPipelineClient.BranchDeletionFailure.SERVICE, + 503, + "unavailable")); + Logger logger = (Logger) LoggerFactory.getLogger( + RagTransientBranchIndexCleanupService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + try { + RagTransientBranchIndexCleanupService service = + new RagTransientBranchIndexCleanupService(branches, generations, pipeline); + service.cleanupExpired(); + service.cleanupExpired(); + } finally { + logger.detachAppender(appender); + } + + verify(pipeline, times(2)).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "target-a", + "revision-100", "manifest-100"); + verify(pipeline, never()).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "target-b", + "revision-101", "manifest-101"); + verify(branches, never()).deleteClaimedTransientById( + org.mockito.ArgumentMatchers.anyLong(), anyString()); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN + && event.getFormattedMessage().contains("target=target-a"))) + .hasSize(1); + } + + @Test + void refreshedCandidateThatLosesAtomicClaimIsNeverDeleted() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + when(branches.findTransientCleanupCandidates()) + .thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(0); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + verifyNoInteractions(generations, pipeline); + } + + @Test + void partialPhysicalCleanupKeepsDurableClaimAndDeletesActiveGenerationLast() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var superseded = generation(100L, "superseded", RagBranchIndexGenerationStatus.SUPERSEDED); + var active = generation(101L, "active", RagBranchIndexGenerationStatus.ACTIVE); + when(branches.findTransientCleanupCandidates()) + .thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(active, superseded)); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "superseded", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("superseded")); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "active", + "revision-101", "manifest-101")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "active", RagPipelineClient.BranchDeletionFailure.SERVICE, + 503, "unavailable")); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + var ordered = inOrder(pipeline); + ordered.verify(pipeline).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "superseded", + "revision-100", "manifest-100"); + ordered.verify(pipeline).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "active", + "revision-101", "manifest-101"); + verify(branches, never()).cancelTransientDeletion(eq(10L), anyString()); + verify(branches, never()).deleteClaimedTransientById(eq(10L), anyString()); + } + + @Test + void transportFailureKeepsClaimBecauseRemoteDeletionMayHaveCompleted() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var generation = generation(100L, "uncertain-target"); + when(branches.findTransientCleanupCandidates()) + .thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(generation)); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "uncertain-target", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "uncertain-target", + RagPipelineClient.BranchDeletionFailure.TRANSPORT, + null, + "connection reset")); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + verify(branches, never()).cancelTransientDeletion(eq(10L), anyString()); + verify(branches, never()).deleteClaimedTransientById(eq(10L), anyString()); + } + + @Test + void targetRejectionKeepsActiveTargetReadableAndReleasesUnusedClaim() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var rejected = generation( + 100L, "rejected", RagBranchIndexGenerationStatus.SUPERSEDED); + var active = generation( + 101L, "active", RagBranchIndexGenerationStatus.ACTIVE); + when(branches.findTransientCleanupCandidates()).thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(active, rejected)); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "rejected", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "rejected", RagPipelineClient.BranchDeletionFailure.TARGET, + 422, "manifest receipt rejected")); new RagTransientBranchIndexCleanupService( branches, generations, pipeline).cleanupExpired(); - verify(branches).delete(index); + verify(pipeline, never()).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "active", + "revision-101", "manifest-101"); + verify(branches).cancelTransientDeletion(eq(10L), anyString()); + verify(branches, never()).deleteClaimedTransientById(eq(10L), anyString()); + } + + @Test + void globallyDisabledRagSilentlyRetainsRegistryRow() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var generation = generation(100L, "disabled-target"); + when(branches.findTransientCleanupCandidates()) + .thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(generation)); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "disabled-target", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "disabled-target", + RagPipelineClient.BranchDeletionFailure.TARGET, + null, + "RAG disabled")); + Logger logger = (Logger) LoggerFactory.getLogger( + RagTransientBranchIndexCleanupService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + try { + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + } finally { + logger.detachAppender(appender); + } + + verify(branches).cancelTransientDeletion(eq(10L), anyString()); + verify(branches, never()).deleteClaimedTransientById(eq(10L), anyString()); + assertThat(appender.list).noneMatch(event -> event.getLevel() == Level.WARN); + } + + private static RagBranchIndexRepository.TransientCleanupCandidate expiredCandidate() { + var candidate = mock(RagBranchIndexRepository.TransientCleanupCandidate.class); + when(candidate.getBranchIndexId()).thenReturn(10L); + when(candidate.getProjectId()).thenReturn(42L); + when(candidate.getWorkspaceName()).thenReturn("workspace"); + when(candidate.getProjectNamespace()).thenReturn("namespace"); + when(candidate.getBranchName()).thenReturn("release/candidate"); + when(candidate.getLastAccessedAt()).thenReturn(OffsetDateTime.now().minusDays(31)); + when(candidate.getProjectConfiguration()).thenReturn(new ProjectConfig( + false, "master", null, + new RagConfig(true, "master", null, null, + true, 30, List.of("develop"), true))); + return candidate; + } + + private static RagBranchIndexGenerationRepository.CleanupGenerationCandidate generation( + Long id, + String target) { + return generation(id, target, RagBranchIndexGenerationStatus.SUPERSEDED); + } + + private static RagBranchIndexGenerationRepository.CleanupGenerationCandidate generation( + Long id, + String target, + RagBranchIndexGenerationStatus status) { + var candidate = mock( + RagBranchIndexGenerationRepository.CleanupGenerationCandidate.class); + when(candidate.getGenerationId()).thenReturn(id); + when(candidate.getCollectionName()).thenReturn(target); + when(candidate.getRevision()).thenReturn("revision-" + id); + when(candidate.getManifestDigest()).thenReturn("manifest-" + id); + when(candidate.getStatus()).thenReturn(status); + return candidate; } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java index 46d092d5..db5537a6 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -101,6 +102,29 @@ void testDeleteFiles_EmptyList() throws Exception { assertThat(result).containsEntry("status", "success"); } + @Test + @SuppressWarnings("unchecked") + void publishGenerationAliasesIncludesExpectedManifestDigest() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("{\"status\":\"published\"}") + .addHeader("Content-Type", "application/json")); + + client.publishGenerationAliases( + "ws", "proj", "main", "commit", "physical-target", + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + true, true); + + RecordedRequest request = mockWebServer.takeRequest(); + assertThat(request.getPath()).isEqualTo("/index/generation-aliases"); + Map payload = objectMapper.readValue( + request.getBody().readUtf8(), Map.class); + assertThat(payload) + .containsEntry("collection_target", "physical-target") + .containsEntry( + "generation_manifest_sha256", + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); + } + @Test @SuppressWarnings("unchecked") void testApplyChanges_SendsOneCompleteCommitPayload() throws Exception { @@ -482,6 +506,27 @@ void testIndexRepository_StreamForwardsProgressAndReturnsTerminalResult() throws assertThat(request.getHeader("Accept")).isEqualTo("text/event-stream"); } + @Test + void testIndexRepository_StreamAcknowledgesSnapshotOwnershipTransfer() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("data: {\"type\":\"admitted\",\"repositoryOwnershipTransferred\":true}\n\n" + + "data: {\"type\":\"complete\",\"result\":{\"document_count\":2,\"chunk_count\":5}}\n\n") + .addHeader("Content-Type", "text/event-stream")); + AtomicBoolean admitted = new AtomicBoolean(false); + + Map result = client.indexRepository( + repositoryPath.toString(), "ws", "proj", "develop", "abc123", + List.of("src/**"), List.of("vendor/**"), "generation-target", + false, false, true, () -> admitted.set(true), ignored -> { }); + + assertThat(admitted).isTrue(); + assertThat(result).containsEntry("chunk_count", 5); + RecordedRequest request = mockWebServer.takeRequest(); + Map payload = objectMapper.readValue( + request.getBody().readUtf8(), Map.class); + assertThat(payload).containsEntry("transfer_repo_ownership", true); + } + // ── deleteBranch tests ─────────────────────────────────────────────────── @Test @@ -527,6 +572,30 @@ void testDeleteBranch_WithSlashInBranchName() throws Exception { assertThat(request.getPath()).contains("feature%2Fxyz"); } + @Test + void testDeleteBranchStructuredOutcomeClassifiesServiceFailureAndNamesTarget() + throws Exception { + mockWebServer.enqueue(new MockResponse() + .setResponseCode(503) + .setBody("{\"detail\":\"unavailable\"}")); + + RagPipelineClient.BranchDeletionOutcome outcome = + client.deleteBranchWithOutcome( + "ws", "proj", "feature", "physical target/1", + "revision/abc", "manifest+digest"); + + assertThat(outcome.successful()).isFalse(); + assertThat(outcome.failure()) + .isEqualTo(RagPipelineClient.BranchDeletionFailure.SERVICE); + assertThat(outcome.shouldStopRemainingTargets()).isTrue(); + assertThat(outcome.targetLabel()).isEqualTo("physical target/1"); + assertThat(outcome.statusCode()).isEqualTo(503); + assertThat(mockWebServer.takeRequest().getPath()) + .contains("collection_target=physical%20target%2F1") + .contains("generation_revision=revision%2Fabc") + .contains("generation_manifest_sha256=manifest%2Bdigest"); + } + // ── getIndexedBranches tests ───────────────────────────────────────────── @Test @@ -749,15 +818,49 @@ void testDeletePrFiles_Success() throws Exception { assertThat(request.getHeader("x-service-secret")).isEqualTo("test-secret"); } + @Test + void deletePrFilesTargetsExactGenerationAndEncodesQueryParameter() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setResponseCode(200) + .setBody("{\"deleted_count\": 1}")); + + boolean result = client.deletePrFiles( + "ws", "proj", 42, "project-generation/with space"); + + assertThat(result).isTrue(); + RecordedRequest request = mockWebServer.takeRequest(); + assertThat(request.getRequestUrl().queryParameter("collection_target")) + .isEqualTo("project-generation/with space"); + } + @Test void testDeletePrFiles_ServerError_ReturnsFalse() throws Exception { mockWebServer.enqueue(new MockResponse() .setResponseCode(500) .setBody("{\"error\": \"internal\"}")); - boolean result = client.deletePrFiles("ws", "proj", 42); + RagPipelineClient.PrFilesDeletionOutcome result = client.deletePrFilesWithOutcome( + "ws", "proj", 42, "generation-a"); - assertThat(result).isFalse(); + assertThat(result.successful()).isFalse(); + assertThat(result.targetLabel()).isEqualTo("generation-a"); + assertThat(result.statusCode()).isEqualTo(500); + assertThat(result.failure()).isEqualTo(RagPipelineClient.PrFilesDeletionFailure.SERVICE); + assertThat(result.shouldStopRemainingTargets()).isTrue(); + } + + @Test + void deletePrFilesLeaseConflictStopsRemainingTargets() { + mockWebServer.enqueue(new MockResponse() + .setResponseCode(409) + .setBody("{\"error\": \"mutation lease unavailable\"}")); + + RagPipelineClient.PrFilesDeletionOutcome result = client.deletePrFilesWithOutcome( + "ws", "proj", 42, "generation-a"); + + assertThat(result.statusCode()).isEqualTo(409); + assertThat(result.failure()).isEqualTo(RagPipelineClient.PrFilesDeletionFailure.SERVICE); + assertThat(result.shouldStopRemainingTargets()).isTrue(); } @Test @@ -786,8 +889,12 @@ void testDeletePrFiles_WhenDisabled_ReturnsTrue() { void testDeletePrFiles_NetworkError_ReturnsFalse() throws IOException { mockWebServer.shutdown(); - boolean result = client.deletePrFiles("ws", "proj", 42); + RagPipelineClient.PrFilesDeletionOutcome result = client.deletePrFilesWithOutcome( + "ws", "proj", 42, "generation-a"); - assertThat(result).isFalse(); + assertThat(result.successful()).isFalse(); + assertThat(result.targetLabel()).isEqualTo("generation-a"); + assertThat(result.failure()).isEqualTo(RagPipelineClient.PrFilesDeletionFailure.TRANSPORT); + assertThat(result.shouldStopRemainingTargets()).isTrue(); } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java index 59810b9f..4c1a0534 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java @@ -16,6 +16,7 @@ import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; @@ -271,4 +272,48 @@ void abandonmentClaimRechecksAHeartbeatUnderTheOperationLock() { verifyNoInteractions(branchIndexRepository); verify(operationRepository, never()).save(operation); } + + @Test + void cleanupClaimMakesExactGenerationUnavailableWithoutRefreshingIt() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "feature/expired", RagBranchIndexKind.TRANSIENT); + branchIndex.setId(10L); + branchIndex.setCleanupClaimToken("cleanup-owner"); + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate( + 42L, "feature/expired")) + .thenReturn(Optional.of(branchIndex)); + + assertThat(service.findAvailableGeneration( + 42L, "feature/expired", "revision-100")) + .isEmpty(); + + verify(branchIndexRepository, never()).save(branchIndex); + verifyNoInteractions(generationRepository); + } + + @Test + void cleanupClaimRejectsAConcurrentBuildRegistration() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "feature/expired", RagBranchIndexKind.TRANSIENT); + branchIndex.setId(10L); + branchIndex.setCleanupClaimToken("cleanup-owner"); + when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) + .thenReturn(Optional.empty()); + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate( + 42L, "feature/expired")) + .thenReturn(Optional.of(branchIndex)); + + assertThatThrownBy(() -> service.registerBuild( + project, + "feature/expired", + RagBranchIndexKind.TRANSIENT, + "revision-99", + "revision-100", + "representation")) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cleanup owns"); + + verify(branchIndexRepository, never()).save(any()); + verifyNoInteractions(generationRepository); + } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java index ae1fce99..c3c9c209 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java @@ -441,6 +441,43 @@ void staleFailureCannotTerminalizeANewerJobOwner() { // ── canStartIndexing ───────────────────────────────────────────────────── + @Test + void succeededOperationRecoveryRequiresItsExactStatusOwner() { + RagIndexStatus ownerless = new RagIndexStatus(); + ownerless.setProject(testProject); + ownerless.setStatus(RagIndexingStatus.UPDATING); + ownerless.setIndexedCommitHash("newer-checkpoint"); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) + .thenReturn(Optional.of(ownerless)); + + boolean reconciled = service.reconcilePublishedGeneration( + testProject, "main", "old-checkpoint", 10, 20, 91L); + + assertThat(reconciled).isFalse(); + assertThat(ownerless.getIndexedCommitHash()).isEqualTo("newer-checkpoint"); + assertThat(ownerless.getStatus()).isEqualTo(RagIndexingStatus.UPDATING); + verify(ragIndexStatusRepository, never()).save(any()); + } + + @Test + void sameRevisionLockOwnedReconciliationCanCreateMissingStatus() { + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) + .thenReturn(Optional.empty()); + when(ragIndexStatusRepository.save(any(RagIndexStatus.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + + boolean reconciled = service.reconcilePublishedGeneration( + testProject, "main", "published-checkpoint", 10, 20); + + assertThat(reconciled).isTrue(); + ArgumentCaptor saved = ArgumentCaptor.forClass( + RagIndexStatus.class); + verify(ragIndexStatusRepository).save(saved.capture()); + assertThat(saved.getValue().getIndexedCommitHash()) + .isEqualTo("published-checkpoint"); + assertThat(saved.getValue().getStatus()).isEqualTo(RagIndexingStatus.INDEXED); + } + @Test void testCanStartIndexing_NoStatus() { when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.empty()); diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java index 798e5026..f130d89e 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java @@ -1,14 +1,20 @@ package org.rostilos.codecrow.ragengine.service; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.hibernate.LazyInitializationException; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; import org.rostilos.codecrow.core.model.branch.Branch; import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.project.config.RagConfig; @@ -23,14 +29,19 @@ import org.rostilos.codecrow.core.service.AnalysisJobService; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; +import org.rostilos.codecrow.ragengine.branch.BranchIndexBuildAdmissionService; +import org.rostilos.codecrow.ragengine.branch.LegacyRagJobLeaseService; +import org.rostilos.codecrow.ragengine.branch.LegacyRagUpdateCompletionService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.slf4j.LoggerFactory; import org.springframework.test.util.ReflectionTestUtils; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; @@ -65,6 +76,21 @@ class RagOperationsServiceImplTest { @Mock private RagPipelineClient ragPipelineClient; + @Mock + private BranchIndexBuildAdmissionService branchIndexBuildAdmissionService; + + @Mock + private LegacyRagJobLeaseService legacyRagJobLeaseService; + + @Mock + private LegacyRagJobLeaseService.JobLease legacyJobLease; + + @Mock + private AnalysisLockService.LockLease legacyLockLease; + + @Mock + private LegacyRagUpdateCompletionService legacyRagUpdateCompletionService; + private RagOperationsServiceImpl service; private Project testProject; @@ -77,7 +103,21 @@ void setUp() { analysisJobService, ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient); + ragPipelineClient, + null, + null, + null, + legacyRagJobLeaseService, + legacyRagUpdateCompletionService); + lenient().when(legacyRagJobLeaseService.start(anyLong())) + .thenReturn(legacyJobLease); + lenient().when(legacyJobLease.confirmOwnership()).thenReturn(true); + lenient().when(analysisLockService.maintainLockLease(anyString(), anyInt())) + .thenReturn(legacyLockLease); + lenient().when(legacyLockLease.confirmOwnership()).thenReturn(true); + lenient().when(legacyRagUpdateCompletionService.complete( + any(), anyString(), anyString(), anyLong(), any(), anyBoolean(), + anyInt(), anyInt(), any(), anySet())).thenReturn(true); ReflectionTestUtils.setField( service, "branchGenerationRepository", branchGenerationRepository); @@ -102,20 +142,23 @@ void exactFirstUseBuildsTargetSnapshotWithoutPrimaryToDevelopDiff() throws Excep ragIndexTrackingService, incrementalRagUpdateService, analysisLockService, analysisJobService, ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, registry, builder); + ragPipelineClient, registry, builder, branchIndexBuildAdmissionService); setupRagEnabled(); setupVcsBinding(); ReflectionTestUtils.setField(service, "ragApiEnabled", true); when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) .thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag("")) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of(), Set.of(), Set.of())); Job job = mock(Job.class); - when(job.getId()).thenReturn(77L); - when(analysisJobService.createRagIndexJob(any(), eq(false), any())) - .thenReturn(job); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 30L, "physical-target", false, null, "exact-feature-lock"); + when(branchIndexBuildAdmissionService.admit( + testProject, "feature", "develop-400", RagBranchIndexKind.DURABLE, + JobTriggerSource.WEBHOOK, "exact-feature-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.NONE)); when(analysisLockService.acquireLock( eq(testProject), eq("feature"), any(), eq("develop-400"), isNull())) .thenReturn(Optional.of("exact-feature-lock")); @@ -123,13 +166,11 @@ void exactFirstUseBuildsTargetSnapshotWithoutPrimaryToDevelopDiff() throws Excep when(vcsClientProvider.getClient(any())).thenReturn(vcs); when(vcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) .thenReturn("develop-400"); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.empty()); - when(builder.build( + when(builder.execute( eq(testProject), any(), eq("my-workspace"), eq("my-repo"), eq("feature"), eq("develop-400"), eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), - any(), any(), eq(77L), eq("exact-feature-lock"), isNull())) + nullable(List.class), nullable(List.class), eq(prepared), isNull())) .thenReturn(Map.of( "generation_manifest_sha256", "manifest-400", "document_count", 231, @@ -143,11 +184,434 @@ void exactFirstUseBuildsTargetSnapshotWithoutPrimaryToDevelopDiff() throws Excep assertThat(ready).isTrue(); verify(vcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - verify(builder).build( + verify(builder).execute( eq(testProject), any(), eq("my-workspace"), eq("my-repo"), eq("feature"), eq("develop-400"), eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), - any(), any(), eq(77L), eq("exact-feature-lock"), isNull()); + nullable(List.class), nullable(List.class), eq(prepared), isNull()); + verify(branchIndexBuildAdmissionService).admit( + testProject, "feature", "develop-400", RagBranchIndexKind.DURABLE, + JobTriggerSource.WEBHOOK, "exact-feature-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + verifyNoInteractions(registry); + } + + @Test + void exactMismatchUsesScalarProjectionWithoutTouchingDetachedActiveGenerationProxy() + throws Exception { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, registry, builder, branchIndexBuildAdmissionService); + setupRagEnabled(); + setupVcsBinding(); + setupProjectWithWorkspaceAndNamespace(); + + when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "main")) + .thenReturn(true); + RagBranchIndex detachedIndex = mock(RagBranchIndex.class); + RagBranchIndexGeneration detachedGeneration = mock( + RagBranchIndexGeneration.class); + lenient().when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "main")) + .thenReturn(Optional.of(detachedIndex)); + lenient().when(detachedIndex.getActiveGeneration()).thenReturn(detachedGeneration); + lenient().when(detachedGeneration.getRevision()).thenThrow( + new LazyInitializationException("no Session")); + when(ragBranchIndexRepository.markAccessedIfUnclaimed( + eq(100L), eq("main"), any())) + .thenReturn(1); + RagBranchIndexRepository.ActiveGenerationCoordinates source = + mock(RagBranchIndexRepository.ActiveGenerationCoordinates.class); + // Simulate returning A after B became active. An older successful A + // operation may exist, so automatic reconciliation must force a fresh + // generation instead of reusing the superseded one. + when(source.getRevision()).thenReturn("revision-b"); + AtomicBoolean lockAcquired = new AtomicBoolean(); + when(ragBranchIndexRepository.findActiveGenerationCoordinates(100L, "main")) + .thenAnswer(ignored -> { + assertThat(lockAcquired.get()).isTrue(); + return Optional.of(source); + }); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 31L, "target-generation", false, null, "rag-lock"); + when(branchIndexBuildAdmissionService.admit( + testProject, "main", "revision-a", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "rag-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.UPDATING)); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("revision-a"), isNull())) + .thenAnswer(ignored -> { + lockAcquired.set(true); + return Optional.of("rag-lock"); + }); + + when(builder.execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull())) + .thenReturn(Map.of( + "generation_manifest_sha256", "target-manifest", + "document_count", 214, + "chunk_count", 642, + "deletedFiles", 0)); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "revision-a", "caller diff", ignored -> { }); + + assertThat(result).isTrue(); + var sourceBindingOrder = inOrder( + analysisLockService, ragBranchIndexRepository, builder); + sourceBindingOrder.verify(analysisLockService).acquireLock( + eq(testProject), eq("main"), any(), eq("revision-a"), isNull()); + sourceBindingOrder.verify(ragBranchIndexRepository) + .findActiveGenerationCoordinates(100L, "main"); + sourceBindingOrder.verify(builder).execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull()); + verifyNoInteractions(vcsClientProvider, registry); + verify(incrementalRagUpdateService, never()).parseDiffForRag("caller diff"); + verify(analysisJobService).info(eq(job), eq("rag_init"), + contains("Starting exact RAG generation rebuild")); + verify(analysisJobService).info(eq(job), eq("rag_complete"), + contains("214 documents, 642 chunks")); + verify(branchIndexBuildAdmissionService).admit( + testProject, "main", "revision-a", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "rag-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + verify(ragBranchIndexRepository, never()) + .findByProjectIdAndBranchName(anyLong(), anyString()); + verify(detachedIndex, never()).getActiveGeneration(); + verify(detachedGeneration, never()).getRevision(); + verify(ragBranchIndexRepository, never()).save(any()); + } + + @Test + void exactPublicationProjectionFailureLeavesSucceededOperationOwnedForRecovery() + throws Exception { + BranchIndexGenerationBuildService builder = mock( + BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + builder, branchIndexBuildAdmissionService); + setupRagEnabled(); + setupVcsBinding(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("revision-published"), isNull())) + .thenReturn(Optional.of("rag-lock")); + Job job = mock(Job.class); + when(job.getId()).thenReturn(91L); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 81L, "physical-generation", false, null, "rag-lock"); + when(branchIndexBuildAdmissionService.admit( + testProject, "main", "revision-published", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "rag-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); + when(builder.execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-published"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull())) + .thenReturn(Map.of("document_count", 12, "chunk_count", 34)); + when(ragIndexTrackingService.reconcilePublishedGeneration( + testProject, "main", "revision-published", 12, 34, 91L)) + .thenThrow(new IllegalStateException("status database unavailable")); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "revision-published", "ignored", ignored -> { }); + + assertThat(result).isFalse(); + verify(builder).execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-published"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull()); + verify(branchIndexBuildAdmissionService, never()).abortOperation(any(), anyString()); + verify(analysisJobService, never()).failJob(any(), anyString()); + verify(ragIndexTrackingService, never()).markIndexingFailed(any(), anyString(), any()); + verify(ragIndexTrackingService, never()).markIncrementalUpdateFailed( + any(), anyString(), any()); + verify(analysisLockService).releaseLock("rag-lock"); + } + + @Test + void exactLockCleanupFailureCannotReverseSuccessfulPublicationAndJob() + throws Exception { + BranchIndexGenerationBuildService builder = mock( + BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + builder, branchIndexBuildAdmissionService); + setupRagEnabled(); + setupVcsBinding(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("revision-complete"), isNull())) + .thenReturn(Optional.of("rag-lock")); + Job job = mock(Job.class); + when(job.getId()).thenReturn(91L); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 81L, "physical-generation", false, null, "rag-lock"); + when(branchIndexBuildAdmissionService.admit( + testProject, "main", "revision-complete", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "rag-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); + when(builder.execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-complete"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull())) + .thenReturn(Map.of("document_count", 12, "chunk_count", 34)); + doThrow(new IllegalStateException("lock database unavailable")) + .when(analysisLockService).releaseLock("rag-lock"); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "revision-complete", "ignored", ignored -> { }); + + assertThat(result).isTrue(); + verify(ragIndexTrackingService).reconcilePublishedGeneration( + testProject, "main", "revision-complete", 12, 34, 91L); + verify(analysisJobService).completeJob(job, null); + verify(analysisJobService, never()).failJob(any(), anyString()); + verify(analysisLockService).releaseLock("rag-lock"); + } + + @Test + void exactIncrementalLockContentionSkipsBeforeBindingSource() { + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class)); + setupRagEnabled(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("target-revision"), isNull())) + .thenReturn(Optional.empty()); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "target-revision", "caller diff", eventConsumer); + + assertThat(result).isFalse(); + verifyNoInteractions(analysisJobService); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); + verify(eventConsumer).accept(argThat(event -> "rag_skip".equals(event.get("state")))); + } + + @Test + void exactIncrementalCleanupClaimSkipsWithoutReadingOrMutatingGeneration() { + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class)); + setupRagEnabled(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("feature"), any(), eq("target-revision"), isNull())) + .thenReturn(Optional.of("rag-lock")); + when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "feature")) + .thenReturn(true); + when(ragBranchIndexRepository.markAccessedIfUnclaimed( + eq(100L), eq("feature"), any())) + .thenReturn(0); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.triggerIncrementalUpdate( + testProject, "feature", "target-revision", "caller diff", eventConsumer); + + assertThat(result).isFalse(); + verifyNoInteractions(analysisJobService); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); + verify(analysisLockService).releaseLock("rag-lock"); + verify(eventConsumer).accept(argThat(event -> "rag_skipped".equals(event.get("state")))); + } + + @Test + void exactIncrementalAlreadyAtTargetCompletesJobAndRepairsPrimaryCheckpoint() + throws Exception { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, registry, builder); + setupRagEnabled(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("target-revision"), isNull())) + .thenReturn(Optional.of("rag-lock")); + when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "main")) + .thenReturn(true); + when(ragBranchIndexRepository.markAccessedIfUnclaimed( + eq(100L), eq("main"), any())) + .thenReturn(1); + RagBranchIndexRepository.ActiveGenerationCoordinates source = + mock(RagBranchIndexRepository.ActiveGenerationCoordinates.class); + when(source.getRevision()).thenReturn("target-revision"); + when(ragBranchIndexRepository.findActiveGenerationCoordinates(100L, "main")) + .thenReturn(Optional.of(source)); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "target-revision", "stale caller diff", eventConsumer); + + assertThat(result).isTrue(); + verify(ragIndexTrackingService).preparePublishedGenerationForUpdate( + testProject, "main", "target-revision", 0, 0); + verifyNoInteractions(analysisJobService); + verify(analysisLockService).releaseLock("rag-lock"); + verifyNoInteractions(vcsClientProvider, registry, builder); + verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); + verify(incrementalRagUpdateService, never()).performIncrementalUpdate( + any(), any(), anyString(), anyString(), anyString(), anyString(), + anySet(), anySet(), anySet(), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + verify(eventConsumer).accept(argThat(event -> + "rag_complete".equals(event.get("state")))); + } + + @Test + void ensureExactMainAlwaysDelegatesToLockedTriggerWithoutPrediff() throws Exception { + service = spy(new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class))); + setupRagEnabled(); + setupVcsBinding(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + VcsClient vcsClient = mock(VcsClient.class); + when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); + when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "main")) + .thenReturn("main-head"); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("main"), eq("main-head"), eq(""), any()); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); + + assertThat(result).isTrue(); + verify(service).triggerIncrementalUpdate( + testProject, "main", "main-head", "", eventConsumer); + verify(ragIndexTrackingService, never()).getIndexStatus(testProject); + verify(vcsClient, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(ragBranchIndexRepository, never()) + .findByProjectIdAndBranchName(anyLong(), anyString()); + } + + @Test + void ensureExactBranchDelegatesMainAndBranchToLockedTriggersWithoutPrediff() + throws Exception { + service = spy(new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class))); + setupRagEnabled(); + setupVcsBinding(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + VcsClient vcsClient = mock(VcsClient.class); + when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); + when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "main")) + .thenReturn("main-head"); + when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "feature")) + .thenReturn("feature-head"); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("main"), eq("main-head"), eq(""), any()); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("feature"), eq("feature-head"), eq(""), any()); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.ensureRagIndexUpToDate(testProject, "feature", eventConsumer); + + assertThat(result).isTrue(); + verify(service).triggerIncrementalUpdate( + testProject, "feature", "feature-head", "", eventConsumer); + verify(service).triggerIncrementalUpdate( + testProject, "main", "main-head", "", eventConsumer); + verify(ragIndexTrackingService, never()).getIndexStatus(testProject); + verify(vcsClient, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(ragBranchIndexRepository, never()) + .findByProjectIdAndBranchName(anyLong(), anyString()); + verify(ragBranchIndexRepository, never()).save(any()); + } + + @Test + void updateExactBranchDelegatesToLockedTriggerWithoutTrustingMutableCheckpoint() + throws Exception { + service = spy(new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class))); + setupRagEnabled(); + setupVcsBinding(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + VcsClient vcsClient = mock(VcsClient.class); + when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); + when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "feature")) + .thenReturn("feature-head"); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("feature"), eq("feature-head"), eq(""), any()); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); + + assertThat(result).isTrue(); + verify(service).triggerIncrementalUpdate( + testProject, "feature", "feature-head", "", eventConsumer); + verify(ragBranchIndexRepository, never()) + .findByProjectIdAndBranchName(anyLong(), anyString()); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(vcsClient, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); } @Test @@ -398,7 +862,9 @@ void testDeleteBranchIndex_NoVcsBinding() { void testDeleteBranchIndex_Success() throws Exception { setupRagEnabled(); setupVcsBinding(); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "feature")).thenReturn(true); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "feature", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("legacy-alias")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -412,7 +878,11 @@ void testDeleteBranchIndex_Success() throws Exception { void testDeleteBranchIndex_PipelineFailure() throws Exception { setupRagEnabled(); setupVcsBinding(); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "feature")).thenReturn(false); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "feature", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "legacy-alias", RagPipelineClient.BranchDeletionFailure.TARGET, + 404, "not found")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -422,11 +892,86 @@ void testDeleteBranchIndex_PipelineFailure() throws Exception { verify(ragBranchIndexRepository, never()).deleteByProjectIdAndBranchName(anyLong(), anyString()); } + @Test + void globalRagDisablementRetainsBranchWithoutCleanupWarning() { + setupRagEnabled(); + setupVcsBinding(); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "feature", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "legacy-alias", + RagPipelineClient.BranchDeletionFailure.TARGET, + null, + "RAG disabled")); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + Logger logger = (Logger) LoggerFactory.getLogger(RagOperationsServiceImpl.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + boolean result; + try { + result = service.deleteBranchIndex(testProject, "feature", eventConsumer); + } finally { + logger.detachAppender(appender); + } + + assertThat(result).isFalse(); + verify(ragBranchIndexRepository, never()) + .deleteByProjectIdAndBranchName(anyLong(), anyString()); + assertThat(appender.list).noneMatch(event -> event.getLevel() == Level.WARN + && event.getFormattedMessage().contains("delete branch RAG generation")); + } + + @Test + void exactDeletionKeepsActiveTargetWhenOlderTargetIsRejected() { + setupRagEnabled(); + setupVcsBinding(); + setupProjectWithWorkspaceAndNamespace(); + RagBranchIndex branchIndex = new RagBranchIndex( + testProject, "feature", RagBranchIndexKind.DURABLE); + branchIndex.setId(501L); + RagBranchIndexGeneration rejected = new RagBranchIndexGeneration(); + rejected.setCollectionName("superseded-target"); + rejected.setRevision("revision-1"); + rejected.activate("manifest-1", 1, 1); + rejected.supersede(); + RagBranchIndexGeneration active = new RagBranchIndexGeneration(); + active.setCollectionName("active-target"); + active.setRevision("revision-2"); + active.activate("manifest-2", 1, 1); + when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) + .thenReturn(Optional.of(branchIndex)); + when(branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc(501L)) + .thenReturn(List.of(active, rejected)); + when(ragPipelineClient.deleteBranchWithOutcome( + "test-ws", "test-ns", "feature", "superseded-target", + "revision-1", "manifest-1")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "superseded-target", + RagPipelineClient.BranchDeletionFailure.TARGET, + 422, + "manifest receipt rejected")); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.deleteBranchIndex(testProject, "feature", eventConsumer); + + assertThat(result).isFalse(); + verify(ragPipelineClient, never()).deleteBranchWithOutcome( + "test-ws", "test-ns", "feature", "active-target", + "revision-2", "manifest-2"); + verify(ragBranchIndexRepository, never()) + .deleteByProjectIdAndBranchName(anyLong(), anyString()); + } + @Test void testDeleteBranchIndex_PipelineException() throws Exception { setupRagEnabled(); setupVcsBinding(); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "feature")) + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "feature", null)) .thenThrow(new RuntimeException("Connection timeout")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -482,7 +1027,9 @@ void testCleanupStaleBranches_DeletesStaleBranch() throws Exception { setupVcsBinding(); when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) .thenReturn(java.util.List.of("main", "feature", "stale-branch")); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "stale-branch")).thenReturn(true); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale-branch", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("legacy-alias")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -508,18 +1055,24 @@ void cleanupStaleBranchesDeletesEveryRegisteredExactGeneration() throws Exceptio branchIndex.setId(501L); RagBranchIndexGeneration first = new RagBranchIndexGeneration(); first.setCollectionName("cc_generation_1"); + first.setRevision("revision-1"); + first.setManifestDigest("manifest-1"); RagBranchIndexGeneration second = new RagBranchIndexGeneration(); second.setCollectionName("cc_generation_2"); + second.setRevision("revision-2"); + second.setManifestDigest("manifest-2"); when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "stale-exact")) .thenReturn(Optional.of(branchIndex)); when(branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc(501L)) .thenReturn(List.of(first, second)); - when(ragPipelineClient.deleteBranch( - "test-ws", "test-ns", "stale-exact", "cc_generation_1")) - .thenReturn(true); - when(ragPipelineClient.deleteBranch( - "test-ws", "test-ns", "stale-exact", "cc_generation_2")) - .thenReturn(true); + when(ragPipelineClient.deleteBranchWithOutcome( + "test-ws", "test-ns", "stale-exact", "cc_generation_1", + "revision-1", "manifest-1")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("cc_generation_1")); + when(ragPipelineClient.deleteBranchWithOutcome( + "test-ws", "test-ns", "stale-exact", "cc_generation_2", + "revision-2", "manifest-2")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("cc_generation_2")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -529,14 +1082,16 @@ void cleanupStaleBranchesDeletesEveryRegisteredExactGeneration() throws Exceptio assertThat(result).containsEntry("status", "success"); assertThat(result).containsEntry("total_deleted", 1); assertThat(result.get("deleted_branches")).isEqualTo(List.of("stale-exact")); - verify(ragPipelineClient).deleteBranch( - "test-ws", "test-ns", "stale-exact", "cc_generation_1"); - verify(ragPipelineClient).deleteBranch( - "test-ws", "test-ns", "stale-exact", "cc_generation_2"); + verify(ragPipelineClient).deleteBranchWithOutcome( + "test-ws", "test-ns", "stale-exact", "cc_generation_1", + "revision-1", "manifest-1"); + verify(ragPipelineClient).deleteBranchWithOutcome( + "test-ws", "test-ns", "stale-exact", "cc_generation_2", + "revision-2", "manifest-2"); verify(ragBranchIndexRepository) .deleteByProjectIdAndBranchName(100L, "stale-exact"); - verify(ragPipelineClient, never()) - .deleteBranch("my-workspace", "my-repo", "stale-exact"); + verify(ragPipelineClient, never()).deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale-exact", null); } @Test @@ -656,12 +1211,14 @@ void testTriggerIncrementalUpdate_FullSuccessFlow() throws Exception { @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(77L); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); when(incrementalRagUpdateService.parseDiffForRag("diff content")) .thenReturn(new IncrementalRagUpdateService.DiffResult(Set.of("src/A.java"), java.util.Collections.emptySet(), Set.of("src/B.java"))); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())).thenReturn(mockJob); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); when(analysisLockService.acquireLock(any(), eq("feature"), any(), eq("commit1"), isNull())) .thenReturn(Optional.of("lock-key")); when(incrementalRagUpdateService.performIncrementalUpdate( @@ -677,8 +1234,103 @@ void testTriggerIncrementalUpdate_FullSuccessFlow() throws Exception { assertThat(result).isTrue(); verifyNoInteractions(ragIndexTrackingService); verify(analysisLockService).releaseLock("lock-key"); - verify(analysisJobService).completeJob(eq(mockJob), isNull()); - verify(ragBranchIndexRepository).save(any(RagBranchIndex.class)); + verify(analysisJobService, never()).completeJob(eq(mockJob), isNull()); + verify(analysisJobService).recordExternallyCompletedJob( + eq(mockJob), eq("rag_complete"), contains("RAG index updated")); + verify(legacyRagUpdateCompletionService).complete( + eq(testProject), eq("feature"), eq("commit1"), eq(77L), + any(), eq(false), eq(0), eq(1), isNull(), eq(Set.of("src/B.java"))); + verify(legacyRagJobLeaseService).start(mockJob.getId()); + verify(analysisLockService).maintainLockLease("lock-key", 360); + verify(legacyJobLease).confirmOwnership(); + verify(legacyLockLease).confirmOwnership(); + verify(legacyJobLease).close(); + verify(legacyLockLease).close(); + } + + @Test + void legacyOwnershipLossBeforeRemoteWorkDoesNotMutateOrOverwriteRecovery() + throws Exception { + setupRagEnabled(); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag("diff")) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of("src/A.java"), Set.of(), Set.of())); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())) + .thenReturn(job); + when(analysisLockService.acquireLock( + any(), eq("feature"), any(), eq("commit1"), isNull())) + .thenReturn(Optional.of("lock-key")); + when(legacyJobLease.isOwnershipLost()).thenReturn(true); + @SuppressWarnings("unchecked") + Consumer> events = mock(Consumer.class); + + boolean result = service.triggerIncrementalUpdate( + testProject, "feature", "commit1", "diff", events); + + assertThat(result).isFalse(); + verify(incrementalRagUpdateService, never()).performIncrementalUpdate( + any(), any(), anyString(), anyString(), anyString(), anyString(), + any(), any(), any()); + verify(analysisJobService, never()).failJob(any(), anyString()); + verifyNoInteractions(ragIndexTrackingService); + verify(legacyJobLease).close(); + verify(legacyLockLease).close(); + verify(analysisLockService).releaseLock("lock-key"); + verify(events).accept(argThat(event -> + "rag_error".equals(event.get("state")) + && String.valueOf(event.get("message")) + .contains("lost durable ownership"))); + } + + @Test + void legacyOwnershipIsReconfirmedBeforeCheckpointAndJobCompletion() + throws Exception { + setupRagEnabled(); + setupVcsBinding(); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag("diff")) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of("src/A.java"), Set.of(), Set.of())); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())) + .thenReturn(job); + when(analysisLockService.acquireLock( + any(), eq("main"), any(), eq("commit1"), isNull())) + .thenReturn(Optional.of("lock-key")); + when(incrementalRagUpdateService.performIncrementalUpdate( + any(), any(), anyString(), anyString(), anyString(), anyString(), + any(), any(), any())) + .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0)); + when(legacyLockLease.confirmOwnership()).thenReturn(false); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "commit1", "diff", ignored -> { }); + + assertThat(result).isFalse(); + verify(incrementalRagUpdateService).performIncrementalUpdate( + any(), any(), anyString(), anyString(), eq("main"), eq("commit1"), + any(), any(), any()); + verify(legacyJobLease).confirmOwnership(); + verify(legacyLockLease).confirmOwnership(); + verify(ragIndexTrackingService, never()).markUpdatingCompleted( + any(), anyString(), anyString(), any(), any(), any(), any()); + verify(legacyRagUpdateCompletionService, never()).complete( + any(), anyString(), anyString(), anyLong(), any(), anyBoolean(), + anyInt(), anyInt(), any(), anySet()); + verify(analysisJobService, never()).completeJob(any(), any()); + verify(analysisJobService, never()).failJob(any(), anyString()); + verify(ragIndexTrackingService, never()).markIncrementalUpdateFailed( + any(), anyString(), any()); + verify(legacyJobLease).close(); + verify(legacyLockLease).close(); } @Test @@ -710,7 +1362,8 @@ void testTriggerIncrementalUpdate_LockNotAcquired() { when(incrementalRagUpdateService.parseDiffForRag(anyString())) .thenReturn(new IncrementalRagUpdateService.DiffResult(Set.of("a.java"), java.util.Collections.emptySet(), java.util.Collections.emptySet())); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())).thenReturn(mockJob); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) .thenReturn(Optional.empty()); @@ -718,10 +1371,40 @@ void testTriggerIncrementalUpdate_LockNotAcquired() { service.triggerIncrementalUpdate(testProject, "feature", "c1", "diff", eventConsumer); assertThat(result).isFalse(); - verify(analysisJobService).failJob(eq(mockJob), anyString()); + verify(analysisJobService).skipJob( + eq(mockJob), contains("previous checkpoint is retained for the next trigger")); + verify(analysisJobService, never()).failJob(any(), anyString()); + verify(eventConsumer).accept(argThat(event -> + "rag_skip".equals(event.get("state")) + && String.valueOf(event.get("message")).contains("next trigger"))); verifyNoInteractions(ragIndexTrackingService); } + @Test + void lockContentionObserverFailureCannotChangeSkippedTerminalState() { + setupRagEnabled(); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + Job mockJob = mock(Job.class); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag(anyString())) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of("a.java"), Set.of(), Set.of())); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); + when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) + .thenReturn(Optional.empty()); + doThrow(new IllegalStateException("observer disconnected")) + .when(eventConsumer).accept(anyMap()); + + boolean result = service.triggerIncrementalUpdate( + testProject, "feature", "c1", "diff", eventConsumer); + + assertThat(result).isFalse(); + verify(analysisJobService).skipJob(eq(mockJob), anyString()); + verify(analysisJobService, never()).failJob(any(), anyString()); + } + @Test void testTriggerIncrementalUpdate_IncrementalUpdateThrows() throws Exception { setupRagEnabled(); @@ -729,12 +1412,14 @@ void testTriggerIncrementalUpdate_IncrementalUpdateThrows() throws Exception { @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(78L); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); when(incrementalRagUpdateService.parseDiffForRag(anyString())) .thenReturn(new IncrementalRagUpdateService.DiffResult(Set.of("a.java"), java.util.Collections.emptySet(), java.util.Collections.emptySet())); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())).thenReturn(mockJob); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) .thenReturn(Optional.of("lock-key")); when(incrementalRagUpdateService.performIncrementalUpdate( @@ -749,6 +1434,12 @@ void testTriggerIncrementalUpdate_IncrementalUpdateThrows() throws Exception { // A retained branch has its own durable operation state and cannot // overwrite the primary branch's project-level status. verifyNoInteractions(ragIndexTrackingService); + verify(analysisJobService).failJob( + eq(mockJob), contains("RAG incremental update failed: Pipeline down")); + verify(analysisJobService, never()).error(eq(mockJob), eq("rag_error"), anyString()); + verify(eventConsumer).accept(argThat(event -> + "rag_error".equals(event.get("state")) + && "warning".equals(event.get("type")))); verify(analysisLockService).releaseLock("lock-key"); } @@ -760,6 +1451,7 @@ void testTriggerIncrementalUpdate_UsesCompletedMainCheckpointAfterEarlierFailure @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(79L); VcsClient vcsClient = mock(VcsClient.class); RagIndexStatus completedStatus = new RagIndexStatus(); completedStatus.setIndexedBranch("main"); @@ -778,7 +1470,8 @@ void testTriggerIncrementalUpdate_UsesCompletedMainCheckpointAfterEarlierFailure when(incrementalRagUpdateService.parseDiffForRag(catchUpDiff)) .thenReturn(new IncrementalRagUpdateService.DiffResult( Set.of("src/Recovered.java"), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())) + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())) .thenReturn(mockJob); when(analysisLockService.acquireLock( any(), eq("main"), any(), eq("current-head"), isNull())) @@ -787,9 +1480,6 @@ void testTriggerIncrementalUpdate_UsesCompletedMainCheckpointAfterEarlierFailure any(), any(), anyString(), anyString(), anyString(), anyString(), any(), any(), any())) .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0)); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "main")) - .thenReturn(Optional.empty()); - boolean result = service.triggerIncrementalUpdate( testProject, "main", "current-head", "diff --git a/src/OnlyLatest.java b/src/OnlyLatest.java\n+latest\n", @@ -801,8 +1491,9 @@ void testTriggerIncrementalUpdate_UsesCompletedMainCheckpointAfterEarlierFailure eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), eq("main"), eq("current-head"), eq(Set.of("src/Recovered.java")), eq(Set.of()), eq(Set.of())); - verify(ragIndexTrackingService).markUpdatingCompleted( - testProject, "main", "current-head", 0, 0, null, 0L); + verify(legacyRagUpdateCompletionService).complete( + eq(testProject), eq("main"), eq("current-head"), eq(79L), + any(), eq(true), eq(0), eq(0), isNull(), eq(Set.of())); } @Test @@ -812,12 +1503,14 @@ void testTriggerIncrementalUpdate_TrackBranchIndex_MergesDeletedFiles() throws E @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(80L); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); when(incrementalRagUpdateService.parseDiffForRag(anyString())) .thenReturn(new IncrementalRagUpdateService.DiffResult(java.util.Collections.emptySet(), java.util.Collections.emptySet(), Set.of("old.java"))); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())).thenReturn(mockJob); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) .thenReturn(Optional.of("lock-key")); when(incrementalRagUpdateService.performIncrementalUpdate( @@ -833,8 +1526,9 @@ void testTriggerIncrementalUpdate_TrackBranchIndex_MergesDeletedFiles() throws E service.triggerIncrementalUpdate(testProject, "feature", "c1", "diff", eventConsumer); - verify(ragBranchIndexRepository).save(argThat( - idx -> idx.getDeletedFiles().contains("old.java") && idx.getDeletedFiles().contains("prev.java"))); + verify(legacyRagUpdateCompletionService).complete( + eq(testProject), eq("feature"), eq("c1"), eq(80L), + any(), eq(false), eq(0), eq(1), isNull(), eq(Set.of("old.java"))); } // ── updateBranchIndex full-flow tests ─────────────────────────────── @@ -868,7 +1562,8 @@ void updateBranchIndexUsesCompletedBranchCheckpointWithoutComparingMain() throws when(incrementalRagUpdateService.parseDiffForRag("checkpoint diff")) .thenReturn(new IncrementalRagUpdateService.DiffResult( Set.of(), Set.of("src/Changed.java"), Set.of())); - when(analysisJobService.createRagIndexJob(eq(testProject), eq(false), any())).thenReturn(mock(Job.class)); + when(analysisJobService.createRagIndexJob( + eq(testProject), eq(false), any(), anyString(), anyString())).thenReturn(mock(Job.class)); when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) .thenReturn(Optional.of("lock")); when(incrementalRagUpdateService.performIncrementalUpdate( @@ -924,7 +1619,6 @@ void updateBranchIndexEmptyDiffSeedsExactSnapshot() throws Exception { when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); VcsClient mockVcs = mock(VcsClient.class); when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getBranchDiff("my-workspace", "my-repo", "main", "feature")).thenReturn(""); when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) .thenReturn("feature-head"); doReturn(true).when(service).triggerIncrementalUpdate( @@ -935,7 +1629,6 @@ void updateBranchIndexEmptyDiffSeedsExactSnapshot() throws Exception { boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); assertThat(result).isTrue(); - verify(eventConsumer).accept(argThat(m -> "info".equals(m.get("type")))); verify(service).triggerIncrementalUpdate( testProject, "feature", "feature-head", "", eventConsumer); } @@ -1119,12 +1812,14 @@ void testEnsureRagIndexUpToDate_DifferentBranch_NoBranchIndex() throws Exception private void stubSuccessfulIncrementalUpdate(String branchName, String commitHash) throws Exception { Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(81L); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) .thenReturn(true); when(incrementalRagUpdateService.parseDiffForRag(anyString())) .thenReturn(new IncrementalRagUpdateService.DiffResult( Set.of("src/A.java"), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())) + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())) .thenReturn(mockJob); when(analysisLockService.acquireLock( any(), eq(branchName), any(), eq(commitHash), isNull())) @@ -1258,8 +1953,14 @@ void testCleanupStaleBranches_PartialFailure() throws Exception { setupVcsBinding(); when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) .thenReturn(List.of("main", "stale1", "stale2")); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "stale1")).thenReturn(true); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "stale2")).thenReturn(false); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale1", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("legacy-alias")); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale2", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "legacy-alias", RagPipelineClient.BranchDeletionFailure.TARGET, + 404, "not found")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -1279,7 +1980,8 @@ void testCleanupStaleBranches_DeleteThrows() throws Exception { setupVcsBinding(); when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) .thenReturn(List.of("main", "stale1")); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "stale1")) + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale1", null)) .thenThrow(new RuntimeException("Connection error")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -1325,19 +2027,119 @@ private void setupProjectWithWorkspaceAndNamespace() { void testDeletePrFiles_Success() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); setupProjectWithWorkspaceAndNamespace(); - when(ragPipelineClient.deletePrFiles("test-ws", "test-ns", 42)).thenReturn(true); + when(ragPipelineClient.deletePrFilesWithOutcome("test-ws", "test-ns", 42, null)) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("legacy-alias")); + + boolean result = service.deletePrFiles(testProject, 42); + + assertThat(result).isTrue(); + verify(ragPipelineClient).deletePrFilesWithOutcome("test-ws", "test-ns", 42, null); + } + + @Test + void deletePrFilesCleansEveryPublishedGenerationAndDeduplicatesTargets() { + ReflectionTestUtils.setField(service, "ragApiEnabled", true); + setupProjectWithWorkspaceAndNamespace(); + when(branchGenerationRepository.findCollectionNamesByProjectIdAndStatusIn( + eq(100L), + eq(List.of( + org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus.ACTIVE, + org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus.SUPERSEDED)))) + .thenReturn(List.of("generation-a", "generation-a", " ", "generation-b")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("generation-a")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("generation-b")); boolean result = service.deletePrFiles(testProject, 42); assertThat(result).isTrue(); - verify(ragPipelineClient).deletePrFiles("test-ws", "test-ns", 42); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a"); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b"); + verify(ragPipelineClient, never()).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, null); + } + + @Test + void deletePrFilesContinuesAfterOneTargetSpecificRejection() { + ReflectionTestUtils.setField(service, "ragApiEnabled", true); + setupProjectWithWorkspaceAndNamespace(); + when(branchGenerationRepository.findCollectionNamesByProjectIdAndStatusIn( + eq(100L), anyList())) + .thenReturn(List.of("generation-a", "generation-b")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.failure( + "generation-a", + RagPipelineClient.PrFilesDeletionFailure.TARGET, + 404, + "collection missing")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("generation-b")); + + boolean result = service.deletePrFiles(testProject, 42); + + assertThat(result).isFalse(); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a"); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b"); + } + + @Test + void deletePrFilesStopsAfterServiceFailureAndNamesFailingTarget() { + ReflectionTestUtils.setField(service, "ragApiEnabled", true); + setupProjectWithWorkspaceAndNamespace(); + when(branchGenerationRepository.findCollectionNamesByProjectIdAndStatusIn( + eq(100L), anyList())) + .thenReturn(List.of("generation-a", "generation-b")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.failure( + "generation-a", + RagPipelineClient.PrFilesDeletionFailure.SERVICE, + 409, + "mutation lease unavailable")); + + Logger logger = (Logger) LoggerFactory.getLogger(RagOperationsServiceImpl.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + boolean result; + try { + result = service.deletePrFiles(testProject, 42); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + + assertThat(result).isFalse(); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a"); + verify(ragPipelineClient, never()).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b"); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage)) + .containsExactly("Failed to delete PR #42 files for project=100 " + + "target=generation-a: status=409 detail=mutation lease unavailable"); } @Test void testDeletePrFiles_PipelineReturnsFalse() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); setupProjectWithWorkspaceAndNamespace(); - when(ragPipelineClient.deletePrFiles("test-ws", "test-ns", 42)).thenReturn(false); + when(ragPipelineClient.deletePrFilesWithOutcome("test-ws", "test-ns", 42, null)) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.failure( + "legacy-alias", + RagPipelineClient.PrFilesDeletionFailure.TARGET, + 404, + "not found")); boolean result = service.deletePrFiles(testProject, 42); @@ -1348,7 +2150,7 @@ void testDeletePrFiles_PipelineReturnsFalse() { void testDeletePrFiles_PipelineThrowsException() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); setupProjectWithWorkspaceAndNamespace(); - when(ragPipelineClient.deletePrFiles("test-ws", "test-ns", 42)) + when(ragPipelineClient.deletePrFilesWithOutcome("test-ws", "test-ns", 42, null)) .thenThrow(new RuntimeException("Connection timeout")); boolean result = service.deletePrFiles(testProject, 42); diff --git a/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/BranchResolverFlowIT.java b/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/BranchResolverFlowIT.java index dae90b21..bc8dd379 100644 --- a/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/BranchResolverFlowIT.java +++ b/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/BranchResolverFlowIT.java @@ -53,6 +53,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anySet; @@ -91,9 +92,12 @@ class BranchResolverFlowIT extends BasePipelineAgentIT { void configureMocks() throws Exception { vcsAiClientService = mock(VcsAiClientService.class); vcsClient = mock(VcsClient.class); + AnalysisLockService.LockLease lockLease = mock(AnalysisLockService.LockLease.class); when(analysisLockService.acquireLockWithWait(any(Project.class), anyString(), any(), anyString(), any(), any())) .thenReturn(Optional.of("branch-resolver-it-lock")); + when(analysisLockService.maintainLockLease(anyString(), anyInt())).thenReturn(lockLease); + when(lockLease.confirmOwnership()).thenReturn(true); when(analysisLockService.isLocked(any(), anyString(), any())).thenReturn(false); when(vcsServiceFactory.getAiClientService(EVcsProvider.GITHUB)).thenReturn(vcsAiClientService); diff --git a/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/LineTrackingFlowIT.java b/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/LineTrackingFlowIT.java index 8f7eeaf0..4714b113 100644 --- a/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/LineTrackingFlowIT.java +++ b/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/LineTrackingFlowIT.java @@ -85,9 +85,12 @@ void configureMocks() throws Exception { vcsAiClientService = mock(VcsAiClientService.class); vcsReportingService = mock(VcsReportingService.class); vcsClient = mock(VcsClient.class); + AnalysisLockService.LockLease lockLease = mock(AnalysisLockService.LockLease.class); when(analysisLockService.acquireLockWithWait(any(Project.class), anyString(), any(), anyString(), any(), any())) .thenReturn(Optional.of("it-lock")); + when(analysisLockService.maintainLockLease(anyString(), anyInt())).thenReturn(lockLease); + when(lockLease.confirmOwnership()).thenReturn(true); when(analysisLockService.isLocked(anyLong(), anyString(), any())).thenReturn(false); when(vcsServiceFactory.getAiClientService(EVcsProvider.GITHUB)).thenReturn(vcsAiClientService); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java index a8b330a2..2fc22991 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java @@ -266,7 +266,8 @@ private void cleanupPrRagData(WebhookPayload payload, Project project) { if (deleted) { log.info("Cleaned up PR #{} RAG data for project {} on merge", prNumber, project.getId()); } else { - log.warn("Failed to cleanup PR #{} RAG data for project {}", prNumber, project.getId()); + log.info("PR #{} RAG cleanup did not complete for project {}; " + + "the RAG client recorded the failure detail", prNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up PR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudPullRequestWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudPullRequestWebhookHandler.java index 5379f641..2786fb5d 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudPullRequestWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudPullRequestWebhookHandler.java @@ -308,7 +308,8 @@ private void cleanupPrRagData(WebhookPayload payload, Project project) { if (deleted) { log.info("Cleaned up PR #{} RAG data for project {} on close/merge", prNumber, project.getId()); } else { - log.warn("Failed to cleanup PR #{} RAG data for project {}", prNumber, project.getId()); + log.info("PR #{} RAG cleanup did not complete for project {}; " + + "the cleanup operation recorded the failure detail", prNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up PR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessor.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessor.java index e6ca5580..412ddf0a 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessor.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessor.java @@ -36,9 +36,7 @@ import org.rostilos.codecrow.taskmanagement.model.TaskDetails; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; -import org.rostilos.codecrow.security.oauth.TokenEncryptionService; import org.rostilos.codecrow.vcsclient.utils.VcsConnectionCredentialsExtractor; -import org.rostilos.codecrow.vcsclient.utils.VcsConnectionCredentialsExtractor.VcsConnectionCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -75,7 +73,6 @@ public class QaDocCommandProcessor implements CommentCommandProcessor { private final QaDocDocumentService qaDocDocumentService; private final QaDocPublicPreviewService qaDocPublicPreviewService; private final PrFileEnrichmentService enrichmentService; - private final VcsConnectionCredentialsExtractor credentialsExtractor; public QaDocCommandProcessor( TaskManagementConnectionRepository connectionRepository, @@ -86,8 +83,7 @@ public QaDocCommandProcessor( QaDocStateRepository qaDocStateRepository, QaDocDocumentService qaDocDocumentService, QaDocPublicPreviewService qaDocPublicPreviewService, - PrFileEnrichmentService enrichmentService, - TokenEncryptionService tokenEncryptionService + PrFileEnrichmentService enrichmentService ) { this.connectionRepository = connectionRepository; this.clientFactory = clientFactory; @@ -98,7 +94,6 @@ public QaDocCommandProcessor( this.qaDocDocumentService = qaDocDocumentService; this.qaDocPublicPreviewService = qaDocPublicPreviewService; this.enrichmentService = enrichmentService; - this.credentialsExtractor = new VcsConnectionCredentialsExtractor(tokenEncryptionService); } @Override @@ -276,22 +271,11 @@ public WebhookResult process( vcsConnection, workspace, repoSlug, commitHash, changedFilePaths); } - // 5e. Extract VCS credentials for Python-side RAG access - String oauthKey = null; - String oauthSecret = null; - String bearerToken = null; - String vcsProviderStr = null; - if (vcsConnection != null) { - try { - VcsConnectionCredentials creds = credentialsExtractor.extractCredentials(vcsConnection); - oauthKey = creds.oAuthClient(); - oauthSecret = creds.oAuthSecret(); - bearerToken = creds.accessToken(); - vcsProviderStr = creds.vcsProviderString(); - } catch (Exception e) { - log.warn("qa-doc command: failed to extract VCS credentials: {}", e.getMessage()); - } - } + // 5e. Keep only the non-secret provider identifier used by the prompt. + String vcsProviderStr = vcsConnection == null + ? null + : VcsConnectionCredentialsExtractor.getVcsProviderString( + vcsConnection.getProviderType()); // 6. Load server-side state and check for existing Jira comment QaDocState state = (prNumber != null) @@ -377,9 +361,6 @@ public WebhookResult process( .repoSlug(repoSlug) .sourceBranch(sourceBranch) .targetBranch(targetBranch) - .oauthKey(oauthKey) - .oauthSecret(oauthSecret) - .bearerToken(bearerToken) .build(); String qaDocument; diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthScheduler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthScheduler.java index 5deaafeb..a62936dd 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthScheduler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthScheduler.java @@ -1,9 +1,7 @@ package org.rostilos.codecrow.pipelineagent.generic.service; -import org.rostilos.codecrow.core.model.branch.Branch; import org.rostilos.codecrow.core.model.branch.BranchHealthStatus; import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; -import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.persistence.repository.branch.BranchRepository; import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; import org.rostilos.codecrow.analysisengine.processor.analysis.BranchAnalysisProcessor; @@ -11,11 +9,12 @@ import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.Map; +import java.util.function.Consumer; /** * Scheduled service that retries analysis for STALE branches. @@ -30,7 +29,7 @@ * - 2 failures → retry after 20 min * - 3 failures → retry after 30 min * - ... - * - MAX_CONSECUTIVE_FAILURES (10) → marked CRITICAL, no more retries + * - MAX_CONSECUTIVE_FAILURES (10) → left STALE until the next push * * Batch size is limited to prevent overwhelming the system. */ @@ -42,11 +41,14 @@ public class BranchHealthScheduler { /** Base backoff in minutes per consecutive failure. */ private static final long BASE_BACKOFF_MINUTES = 10; - /** Maximum consecutive failures before promoting to CRITICAL (stops retrying). */ + /** Maximum consecutive failures before scheduled retries stop. */ private static final int MAX_CONSECUTIVE_FAILURES = 10; /** Maximum branches to retry per scheduler run to avoid overloading. */ private static final int BATCH_SIZE = 5; + + /** Scheduled retries have no request stream, but downstream stages may emit. */ + private static final Consumer> NO_OP_OBSERVER = ignored -> { }; private final BranchRepository branchRepository; private final BranchAnalysisProcessor branchAnalysisProcessor; @@ -65,9 +67,12 @@ public BranchHealthScheduler( * based on its consecutiveFailures count. */ @Scheduled(fixedDelayString = "${branch.health.retry.interval.ms:600000}") - @Transactional public void retryStaleBranches() { - List staleBranches = branchRepository.findByHealthStatusWithProject(BranchHealthStatus.STALE); + // The repository materializes scalar data in its own short read + // transaction. Never hold a scheduler transaction/connection while the + // processor performs VCS, RAG, or AI calls. + List staleBranches = + branchRepository.findStaleRetryCandidates(BranchHealthStatus.STALE); if (staleBranches.isEmpty()) { return; @@ -78,7 +83,7 @@ public void retryStaleBranches() { OffsetDateTime now = OffsetDateTime.now(); int retried = 0; - for (Branch branch : staleBranches) { + for (BranchRepository.StaleRetryCandidate branch : staleBranches) { if (retried >= BATCH_SIZE) { log.info("Batch size limit reached ({}), remaining branches will be retried next cycle", BATCH_SIZE); break; @@ -88,8 +93,9 @@ public void retryStaleBranches() { // The branch stays STALE but backoff is so large it effectively won't retry // until a new push event resets the failure counter. if (branch.getConsecutiveFailures() >= MAX_CONSECUTIVE_FAILURES) { - log.warn("Branch {} (project={}, branch='{}') exceeded max retries ({}) — skipping until next push event", - branch.getId(), branch.getProject().getId(), branch.getBranchName(), + log.debug("Branch {} (project={}, branch='{}') remains above max retries ({}) — " + + "waiting for the next push event", + branch.getBranchId(), branch.getProjectId(), branch.getBranchName(), branch.getConsecutiveFailures()); continue; } @@ -99,22 +105,20 @@ public void retryStaleBranches() { continue; } - Project project = branch.getProject(); - // Skip if branch analysis is disabled for this project - if (!project.isBranchAnalysisEnabled()) { + if (!branch.getBranchAnalysisEnabled()) { log.debug("Skipping STALE branch {} — branch analysis disabled for project {}", - branch.getBranchName(), project.getId()); + branch.getBranchName(), branch.getProjectId()); continue; } // Skip if no commit hash to retry with if (branch.getCommitHash() == null || branch.getCommitHash().isBlank()) { - log.warn("Skipping STALE branch {} — no commit hash available", branch.getId()); + log.debug("Skipping STALE branch {} — no commit hash available", branch.getBranchId()); continue; } - retryBranchAnalysis(branch, project); + retryBranchAnalysis(branch); retried++; } @@ -127,7 +131,9 @@ public void retryStaleBranches() { * Check if a branch is eligible for retry based on backoff. * A branch must wait (consecutiveFailures * BASE_BACKOFF_MINUTES) since its last health check. */ - private boolean isEligibleForRetry(Branch branch, OffsetDateTime now) { + private boolean isEligibleForRetry( + BranchRepository.StaleRetryCandidate branch, + OffsetDateTime now) { OffsetDateTime lastCheck = branch.getLastHealthCheckAt(); if (lastCheck == null) { // Never checked — eligible @@ -139,7 +145,7 @@ private boolean isEligibleForRetry(Branch branch, OffsetDateTime now) { if (minutesSinceLastCheck < requiredWaitMinutes) { log.debug("Branch {} backoff not elapsed: waited {}m of required {}m (failures={})", - branch.getId(), minutesSinceLastCheck, requiredWaitMinutes, + branch.getBranchId(), minutesSinceLastCheck, requiredWaitMinutes, branch.getConsecutiveFailures()); return false; } @@ -150,29 +156,29 @@ private boolean isEligibleForRetry(Branch branch, OffsetDateTime now) { /** * Retry branch analysis for a single STALE branch. */ - private void retryBranchAnalysis(Branch branch, Project project) { + private void retryBranchAnalysis(BranchRepository.StaleRetryCandidate branch) { try { log.info("Retrying STALE branch: id={}, project={}, branch='{}', commit={}, failures={}", - branch.getId(), project.getId(), branch.getBranchName(), + branch.getBranchId(), branch.getProjectId(), branch.getBranchName(), branch.getCommitHash(), branch.getConsecutiveFailures()); BranchProcessRequest request = new BranchProcessRequest(); - request.projectId = project.getId(); + request.projectId = branch.getProjectId(); request.targetBranchName = branch.getBranchName(); request.commitHash = branch.getCommitHash(); request.analysisType = AnalysisType.BRANCH_ANALYSIS; // No sourcePrNumber — this is a scheduled retry, not triggered by a PR merge - branchAnalysisProcessor.process(request, null); + branchAnalysisProcessor.process(request, NO_OP_OBSERVER); log.info("STALE branch retry completed successfully: branch={}, project={}", - branch.getBranchName(), project.getId()); + branch.getBranchName(), branch.getProjectId()); } catch (Exception e) { // The processor itself handles markStale() on failure, // so we just log the error and move on to the next branch. log.error("STALE branch retry failed: branch={}, project={}, error={}", - branch.getBranchName(), project.getId(), e.getMessage()); + branch.getBranchName(), branch.getProjectId(), e.getMessage()); } } } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java index 89af6e8c..4c982464 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java @@ -368,6 +368,32 @@ public void failJob(Job job, String errorMessage) { } } + @Override + public void skipJob(Job job, String reason) { + if (job == null) return; + + try { + jobService.skipJob(job, reason); + } catch (Exception e) { + log.error("Error skipping redundant job {}", job.getExternalId(), e); + } + } + + @Override + public void recordExternallyCompletedJob( + Job job, + String state, + String message) { + if (job == null) return; + + try { + jobService.recordExternallyCompletedJob(job, state, message); + } catch (Exception e) { + log.warn("Could not record terminal notification for completed job {}: {}", + job.getExternalId(), e.getMessage()); + } + } + public JobService getJobService() { return jobService; } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoveryScheduler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoveryScheduler.java index 8913d457..35fc85ac 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoveryScheduler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoveryScheduler.java @@ -52,6 +52,9 @@ public void recoverAcceptedJobs() { OffsetDateTime pendingThreshold = OffsetDateTime.now().minusSeconds(30); List candidates = jobService.findRecoverableWebhookJobs(pendingThreshold, 50); for (Job candidate : candidates) { + if (!isSupportedWebhookAnalysis(candidate)) { + continue; + } if (!jobService.claimRecoverableWebhookJob( candidate.getId(), pendingThreshold)) { continue; @@ -63,6 +66,9 @@ public void recoverAcceptedJobs() { List abandoned = jobService.findAbandonedRunningWebhookJobs( abandonedThreshold, 20); for (Job candidate : abandoned) { + if (!isSupportedWebhookAnalysis(candidate)) { + continue; + } if (!jobService.claimAbandonedRunningWebhookJob( candidate.getId(), abandonedThreshold)) { continue; @@ -73,6 +79,11 @@ public void recoverAcceptedJobs() { } } + private static boolean isSupportedWebhookAnalysis(Job job) { + return job != null && (job.getJobType() == JobType.PR_ANALYSIS + || job.getJobType() == JobType.BRANCH_ANALYSIS); + } + private void recover(Job job) { try { Optional persistedPayload = diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java index 80ad0679..78271b3c 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java @@ -400,7 +400,8 @@ private void cleanupPrRagData(WebhookPayload payload, Project project) { if (deleted) { log.info("Cleaned up PR #{} RAG data for project {} on close/merge", prNumber, project.getId()); } else { - log.warn("Failed to cleanup PR #{} RAG data for project {}", prNumber, project.getId()); + log.info("PR #{} RAG cleanup did not complete for project {}; " + + "the cleanup operation recorded the failure detail", prNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up PR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMergeRequestWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMergeRequestWebhookHandler.java index e7b42f2a..bee07e1e 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMergeRequestWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMergeRequestWebhookHandler.java @@ -309,7 +309,8 @@ private void cleanupPrRagData(WebhookPayload payload, Project project) { if (deleted) { log.info("Cleaned up MR !{} RAG data for project {} on close/merge", prNumber, project.getId()); } else { - log.warn("Failed to cleanup MR !{} RAG data for project {}", prNumber, project.getId()); + log.info("MR !{} RAG cleanup did not complete for project {}; " + + "the cleanup operation recorded the failure detail", prNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up MR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java index 51993f7d..165aa3a6 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java @@ -181,7 +181,8 @@ private void cleanupPrRagData(Project project, Long mrNumber) { if (deleted) { log.info("Cleaned up MR !{} RAG data for project {} on merge", mrNumber, project.getId()); } else { - log.warn("Failed to cleanup MR !{} RAG data for project {}", mrNumber, project.getId()); + log.info("MR !{} RAG cleanup did not complete for project {}; " + + "the cleanup operation recorded the failure detail", mrNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up MR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListener.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListener.java index fcbf0508..4853b522 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListener.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListener.java @@ -30,9 +30,7 @@ import org.rostilos.codecrow.taskmanagement.model.TaskDetails; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; -import org.rostilos.codecrow.security.oauth.TokenEncryptionService; import org.rostilos.codecrow.vcsclient.utils.VcsConnectionCredentialsExtractor; -import org.rostilos.codecrow.vcsclient.utils.VcsConnectionCredentialsExtractor.VcsConnectionCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.event.EventListener; @@ -76,7 +74,6 @@ public class QaAutoDocListener { private final QaDocDocumentService qaDocDocumentService; private final QaDocPublicPreviewService qaDocPublicPreviewService; private final PrFileEnrichmentService enrichmentService; - private final VcsConnectionCredentialsExtractor credentialsExtractor; public QaAutoDocListener(ProjectRepository projectRepository, TaskManagementConnectionRepository connectionRepository, @@ -87,8 +84,7 @@ public QaAutoDocListener(ProjectRepository projectRepository, QaDocStateRepository qaDocStateRepository, QaDocDocumentService qaDocDocumentService, QaDocPublicPreviewService qaDocPublicPreviewService, - PrFileEnrichmentService enrichmentService, - TokenEncryptionService tokenEncryptionService) { + PrFileEnrichmentService enrichmentService) { this.projectRepository = projectRepository; this.connectionRepository = connectionRepository; this.clientFactory = clientFactory; @@ -99,7 +95,6 @@ public QaAutoDocListener(ProjectRepository projectRepository, this.qaDocDocumentService = qaDocDocumentService; this.qaDocPublicPreviewService = qaDocPublicPreviewService; this.enrichmentService = enrichmentService; - this.credentialsExtractor = new VcsConnectionCredentialsExtractor(tokenEncryptionService); } @Async @@ -265,22 +260,11 @@ private void processQaAutoDoc(AnalysisCompletedEvent event) throws Exception { currentCommitHash, changedFilePaths); } - // 5e. Extract VCS credentials for Python-side RAG queries - String oauthKey = null; - String oauthSecret = null; - String bearerToken = null; - String vcsProviderStr = null; - if (vcsConnection != null) { - try { - VcsConnectionCredentials creds = credentialsExtractor.extractCredentials(vcsConnection); - oauthKey = creds.oAuthClient(); - oauthSecret = creds.oAuthSecret(); - bearerToken = creds.accessToken(); - vcsProviderStr = creds.vcsProviderString(); - } catch (Exception e) { - log.warn("QA auto-doc: failed to extract VCS credentials (non-critical): {}", e.getMessage()); - } - } + // 5e. Keep only the non-secret provider identifier used by the prompt. + String vcsProviderStr = vcsConnection == null + ? null + : VcsConnectionCredentialsExtractor.getVcsProviderString( + vcsConnection.getProviderType()); // 6. Resolve task management connection + fetch task details TaskManagementConnection connection = connectionRepository @@ -367,9 +351,6 @@ private void processQaAutoDoc(AnalysisCompletedEvent event) throws Exception { .repoSlug(repoSlug) .sourceBranch(sourceBranch) .targetBranch(targetBranch) - .oauthKey(oauthKey) - .oauthSecret(oauthSecret) - .bearerToken(bearerToken) .build(); String qaDocument; diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationContext.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationContext.java index fcf7d21c..fb3c4ead 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationContext.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationContext.java @@ -49,15 +49,7 @@ public record QaDocGenerationContext( /** Source branch of the PR. May be null. */ String sourceBranch, /** Target branch of the PR. May be null. */ - String targetBranch, - - // ── VCS credentials (for Python-side RAG deterministic API) ── - /** OAuth consumer key (Bitbucket). May be null. */ - String oauthKey, - /** OAuth consumer secret (Bitbucket). May be null. */ - String oauthSecret, - /** Bearer / PAT token (GitHub, GitLab, Bitbucket APP). May be null. */ - String bearerToken + String targetBranch ) { /** * Builder for constructing a context step-by-step as data becomes available. @@ -81,9 +73,6 @@ public static final class Builder { private String repoSlug; private String sourceBranch; private String targetBranch; - private String oauthKey; - private String oauthSecret; - private String bearerToken; private Builder() {} @@ -101,9 +90,6 @@ private Builder() {} public Builder repoSlug(String v) { this.repoSlug = v; return this; } public Builder sourceBranch(String v) { this.sourceBranch = v; return this; } public Builder targetBranch(String v) { this.targetBranch = v; return this; } - public Builder oauthKey(String v) { this.oauthKey = v; return this; } - public Builder oauthSecret(String v) { this.oauthSecret = v; return this; } - public Builder bearerToken(String v) { this.bearerToken = v; return this; } public QaDocGenerationContext build() { return new QaDocGenerationContext( @@ -111,8 +97,7 @@ public QaDocGenerationContext build() { diff, deltaDiff, enrichmentData, changedFilePaths, previousDocumentation, isSamePrRerun, - vcsProvider, workspaceSlug, repoSlug, sourceBranch, targetBranch, - oauthKey, oauthSecret, bearerToken); + vcsProvider, workspaceSlug, repoSlug, sourceBranch, targetBranch); } } } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationService.java index d88b778c..f14e53ff 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationService.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationService.java @@ -67,6 +67,9 @@ public QaDocGenerationService( this.tokenEncryptionService = tokenEncryptionService; this.objectMapper = new ObjectMapper(); this.httpClient = HttpClient.newBuilder() + // Uvicorn serves HTTP/1.1 on the internal clear-text endpoint. + // Prevent the JDK client from attempting an h2c upgrade. + .version(HttpClient.Version.HTTP_1_1) .connectTimeout(Duration.ofSeconds(30)) .build(); } @@ -252,7 +255,7 @@ private Map buildPayloadFromContext(Project project, payload.put("changed_file_paths", ctx.changedFilePaths()); } - // ── VCS connection info (for Python-side RAG queries) ── + // ── Non-secret VCS identifiers used in the generated document ── if (ctx.vcsProvider() != null) { payload.put("vcs_provider", ctx.vcsProvider()); } @@ -269,17 +272,6 @@ private Map buildPayloadFromContext(Project project, payload.put("target_branch", ctx.targetBranch()); } - // ── OAuth credentials (for Python-side VCS/RAG access) ── - if (ctx.oauthKey() != null) { - payload.put("oauth_key", ctx.oauthKey()); - } - if (ctx.oauthSecret() != null) { - payload.put("oauth_secret", ctx.oauthSecret()); - } - if (ctx.bearerToken() != null) { - payload.put("bearer_token", ctx.bearerToken()); - } - // ── PR metadata enriched with analysis summary ── Map prMeta = (prMetadata != null) ? new LinkedHashMap<>(prMetadata) diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessorTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessorTest.java index d7ed377e..caebc2e0 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessorTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessorTest.java @@ -32,7 +32,6 @@ import org.rostilos.codecrow.pipelineagent.qadoc.QaDocPublicPreviewService; import org.rostilos.codecrow.core.persistence.repository.qadoc.QaDocStateRepository; import org.rostilos.codecrow.analysisengine.service.pr.PrFileEnrichmentService; -import org.rostilos.codecrow.security.oauth.TokenEncryptionService; import org.rostilos.codecrow.taskmanagement.ETaskManagementPlatform; import org.rostilos.codecrow.taskmanagement.TaskManagementClient; import org.rostilos.codecrow.taskmanagement.TaskManagementClientFactory; @@ -65,7 +64,6 @@ class QaDocCommandProcessorTest { @Mock private QaDocDocumentService qaDocDocumentService; @Mock private QaDocPublicPreviewService qaDocPublicPreviewService; @Mock private PrFileEnrichmentService enrichmentService; - @Mock private TokenEncryptionService tokenEncryptionService; private QaDocCommandProcessor processor; private Project project; @@ -107,8 +105,7 @@ void setUp() { qaDocStateRepository, qaDocDocumentService, qaDocPublicPreviewService, - enrichmentService, - tokenEncryptionService + enrichmentService ); project = new Project(); diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthSchedulerTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthSchedulerTest.java new file mode 100644 index 00000000..c2d91458 --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthSchedulerTest.java @@ -0,0 +1,134 @@ +package org.rostilos.codecrow.pipelineagent.generic.service; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; +import org.rostilos.codecrow.analysisengine.processor.analysis.BranchAnalysisProcessor; +import org.rostilos.codecrow.core.model.branch.BranchHealthStatus; +import org.rostilos.codecrow.core.persistence.repository.branch.BranchRepository; +import org.rostilos.codecrow.events.EventNotificationEmitter; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.notNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class BranchHealthSchedulerTest { + + @Test + void retryRunsOutsideTransactionUsingMaterializedCandidate() throws Exception { + BranchRepository repository = mock(BranchRepository.class); + BranchAnalysisProcessor processor = mock(BranchAnalysisProcessor.class); + BranchRepository.StaleRetryCandidate candidate = candidate(2, true, "abc123"); + when(repository.findStaleRetryCandidates(BranchHealthStatus.STALE)) + .thenReturn(List.of(candidate)); + + new BranchHealthScheduler(repository, processor).retryStaleBranches(); + + verify(processor).process( + any(BranchProcessRequest.class), + notNull()); + assertThat(BranchHealthScheduler.class + .getMethod("retryStaleBranches") + .isAnnotationPresent(Transactional.class)) + .isFalse(); + } + + @Test + void scheduledRetryCanEmitProgressWithoutAnAttachedObserver() throws Exception { + BranchRepository repository = mock(BranchRepository.class); + BranchAnalysisProcessor processor = mock(BranchAnalysisProcessor.class); + BranchRepository.StaleRetryCandidate retryable = candidate(2, true, "abc123"); + when(repository.findStaleRetryCandidates(BranchHealthStatus.STALE)) + .thenReturn(List.of(retryable)); + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + java.util.function.Consumer> observer = invocation.getArgument(1); + EventNotificationEmitter.emitStatus(observer, "started", "Started"); + return Map.of("status", "accepted"); + }).when(processor).process(any(BranchProcessRequest.class), notNull()); + + new BranchHealthScheduler(repository, processor).retryStaleBranches(); + + verify(processor).process(any(BranchProcessRequest.class), notNull()); + } + + @Test + void retryCeilingDoesNotWarnEveryScheduledRun() throws Exception { + BranchRepository repository = mock(BranchRepository.class); + BranchAnalysisProcessor processor = mock(BranchAnalysisProcessor.class); + BranchRepository.StaleRetryCandidate atCeiling = candidate(10, true, "abc123"); + when(repository.findStaleRetryCandidates(BranchHealthStatus.STALE)) + .thenReturn(List.of(atCeiling)); + Logger logger = (Logger) LoggerFactory.getLogger(BranchHealthScheduler.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + try { + BranchHealthScheduler scheduler = new BranchHealthScheduler(repository, processor); + scheduler.retryStaleBranches(); + scheduler.retryStaleBranches(); + } finally { + logger.detachAppender(appender); + } + + verify(processor, never()).process(any(), any()); + assertThat(appender.list) + .noneMatch(event -> event.getLevel() == Level.WARN + && event.getFormattedMessage().contains("max retries")); + } + + @Test + void missingCommitDoesNotWarnEveryScheduledRun() throws Exception { + BranchRepository repository = mock(BranchRepository.class); + BranchAnalysisProcessor processor = mock(BranchAnalysisProcessor.class); + BranchRepository.StaleRetryCandidate missingCommit = candidate(2, true, " "); + when(repository.findStaleRetryCandidates(BranchHealthStatus.STALE)) + .thenReturn(List.of(missingCommit)); + Logger logger = (Logger) LoggerFactory.getLogger(BranchHealthScheduler.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + try { + BranchHealthScheduler scheduler = new BranchHealthScheduler(repository, processor); + scheduler.retryStaleBranches(); + scheduler.retryStaleBranches(); + } finally { + logger.detachAppender(appender); + } + + verify(processor, never()).process(any(), any()); + assertThat(appender.list) + .noneMatch(event -> event.getLevel() == Level.WARN + && event.getFormattedMessage().contains("no commit hash")); + } + + private static BranchRepository.StaleRetryCandidate candidate( + int failures, + boolean analysisEnabled, + String commitHash) { + BranchRepository.StaleRetryCandidate candidate = + mock(BranchRepository.StaleRetryCandidate.class); + when(candidate.getBranchId()).thenReturn(7L); + when(candidate.getProjectId()).thenReturn(42L); + when(candidate.getBranchName()).thenReturn("main"); + when(candidate.getCommitHash()).thenReturn(commitHash); + when(candidate.getConsecutiveFailures()).thenReturn(failures); + when(candidate.getBranchAnalysisEnabled()).thenReturn(analysisEnabled); + return candidate; + } +} diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoverySchedulerTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoverySchedulerTest.java index 8e73c437..c9aa49bf 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoverySchedulerTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoverySchedulerTest.java @@ -4,6 +4,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -45,6 +46,7 @@ void replaysPersistedWebhookPayloadAfterLostDispatch() throws Exception { when(job.getId()).thenReturn(10L); when(job.getExternalId()).thenReturn("job-10"); + when(job.getJobType()).thenReturn(JobType.BRANCH_ANALYSIS); when(job.getProject()).thenReturn(project); when(project.getId()).thenReturn(20L); when(jobService.findWebhookDispatchPayload(10L)) @@ -95,4 +97,45 @@ void restoresLegacyPendingBranchJobWithoutStoredPayload() { assertThat(payload.getValue().sourceBranch()).isEqualTo("release/10x"); assertThat(payload.getValue().commitHash()).isEqualTo("abc123"); } + + @Test + void neverClaimsPendingRagChildrenOrConsultsPersistedPayload() { + Job initial = mock(Job.class); + Job incremental = mock(Job.class); + when(initial.getId()).thenReturn(12L); + when(initial.getJobType()).thenReturn(JobType.RAG_INITIAL_INDEX); + when(incremental.getId()).thenReturn(13L); + when(incremental.getJobType()).thenReturn(JobType.RAG_INCREMENTAL_INDEX); + when(jobService.findRecoverableWebhookJobs(any(), eq(50))) + .thenReturn(List.of(initial, incremental)); + scheduler.recoverAcceptedJobs(); + + verify(jobService, never()).claimRecoverableWebhookJob(eq(12L), any()); + verify(jobService, never()).claimRecoverableWebhookJob(eq(13L), any()); + verify(jobService, never()).findWebhookDispatchPayload(12L); + verify(jobService, never()).findWebhookDispatchPayload(13L); + verify(asyncProcessor, never()).processWebhookAsync( + any(), any(), any(), any(), any()); + } + + @Test + void neverClaimsAbandonedRunningRagChildren() { + Job initial = mock(Job.class); + Job incremental = mock(Job.class); + when(initial.getId()).thenReturn(14L); + when(initial.getJobType()).thenReturn(JobType.RAG_INITIAL_INDEX); + when(incremental.getId()).thenReturn(15L); + when(incremental.getJobType()).thenReturn(JobType.RAG_INCREMENTAL_INDEX); + when(jobService.findAbandonedRunningWebhookJobs(any(), eq(20))) + .thenReturn(List.of(initial, incremental)); + + scheduler.recoverAcceptedJobs(); + + verify(jobService, never()).claimAbandonedRunningWebhookJob(eq(14L), any()); + verify(jobService, never()).claimAbandonedRunningWebhookJob(eq(15L), any()); + verify(jobService, never()).findWebhookDispatchPayload(14L); + verify(jobService, never()).findWebhookDispatchPayload(15L); + verify(asyncProcessor, never()).processWebhookAsync( + any(), any(), any(), any(), any()); + } } diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListenerTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListenerTest.java index c6be90ea..5f8e5fa3 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListenerTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListenerTest.java @@ -11,7 +11,6 @@ import org.rostilos.codecrow.core.service.CodeAnalysisService; import org.rostilos.codecrow.core.service.QaDocDocumentService; import org.rostilos.codecrow.events.analysis.AnalysisCompletedEvent; -import org.rostilos.codecrow.security.oauth.TokenEncryptionService; import org.rostilos.codecrow.taskmanagement.TaskManagementClientFactory; import org.rostilos.codecrow.vcsclient.VcsClientProvider; @@ -47,9 +46,6 @@ class QaAutoDocListenerTest { private QaDocPublicPreviewService qaDocPublicPreviewService; @Mock private PrFileEnrichmentService enrichmentService; - @Mock - private TokenEncryptionService tokenEncryptionService; - @Test void loadsProjectWithVcsConnectionsForAsyncProcessing() { QaAutoDocListener listener = new QaAutoDocListener( @@ -62,8 +58,7 @@ void loadsProjectWithVcsConnectionsForAsyncProcessing() { qaDocStateRepository, qaDocDocumentService, qaDocPublicPreviewService, - enrichmentService, - tokenEncryptionService); + enrichmentService); AnalysisCompletedEvent event = new AnalysisCompletedEvent( this, "correlation-id", diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationServiceTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationServiceTest.java index 51c2d04b..fa38553c 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationServiceTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationServiceTest.java @@ -17,6 +17,7 @@ import org.springframework.test.util.ReflectionTestUtils; import java.io.IOException; +import java.net.http.HttpClient; import java.time.OffsetDateTime; import java.util.Base64; import java.util.LinkedHashMap; @@ -86,6 +87,14 @@ private QaAutoDocConfig customTemplateConfig() { ); } + @Test + void internalInferenceClientUsesHttp11WithoutH2cUpgrade() { + HttpClient configuredClient = (HttpClient) ReflectionTestUtils.getField(service, "httpClient"); + + assertThat(configuredClient).isNotNull(); + assertThat(configuredClient.version()).isEqualTo(HttpClient.Version.HTTP_1_1); + } + private TaskDetails sampleTaskDetails() { return new TaskDetails( "PROJ-123", "Implement feature", "Full description", @@ -150,6 +159,9 @@ void shouldSendCorrectPayloadAndReturnDoc() throws Exception { assertThat(body.get("template_mode").asText()).isEqualTo("BASE"); assertThat(body.has("pr_metadata")).isTrue(); assertThat(body.get("pr_metadata").get("sourceBranch").asText()).isEqualTo("feature/PROJ-123"); + assertThat(body.has("oauth_key")).isFalse(); + assertThat(body.has("oauth_secret")).isFalse(); + assertThat(body.has("bearer_token")).isFalse(); // Verify task context (keys must match Python placeholder names) JsonNode taskContext = body.get("task_context"); diff --git a/python-ecosystem/inference-orchestrator/integration/test_rag_client_extended.py b/python-ecosystem/inference-orchestrator/integration/test_rag_client_extended.py index c82f46f3..dff06c8e 100644 --- a/python-ecosystem/inference-orchestrator/integration/test_rag_client_extended.py +++ b/python-ecosystem/inference-orchestrator/integration/test_rag_client_extended.py @@ -314,14 +314,14 @@ async def test_delete_pr_files_success(rag_client): @pytest.mark.asyncio(loop_scope="function") @respx.mock async def test_delete_pr_files_not_found(rag_client): - """Collection doesn't exist → skipped status → False.""" + """An already-absent overlay is an idempotent cleanup success.""" respx.delete("http://rag-pipeline:8001/index/pr-files/ws/proj/99").mock( return_value=httpx.Response(200, json={"status": "skipped"}) ) result = await rag_client.delete_pr_files( workspace="ws", project="proj", pr_number=99, ) - assert result is False # status != "deleted" + assert result is True await rag_client.close() diff --git a/python-ecosystem/inference-orchestrator/src/api/app.py b/python-ecosystem/inference-orchestrator/src/api/app.py index db79f4c6..52e653da 100644 --- a/python-ecosystem/inference-orchestrator/src/api/app.py +++ b/python-ecosystem/inference-orchestrator/src/api/app.py @@ -4,6 +4,7 @@ Creates and configures the FastAPI application with all routers. Uses lifespan context manager for proper startup/shutdown of shared resources. """ +import asyncio import os import logging from contextlib import asynccontextmanager @@ -48,11 +49,22 @@ async def lifespan(app: FastAPI): # --- Shutdown --- logger.info("Shutting down application services...") + consumer_stops = [] if hasattr(app.state, "queue_consumer"): - await app.state.queue_consumer.stop() - + consumer_stops.append(app.state.queue_consumer.stop()) if hasattr(app.state, "command_queue_consumer"): - await app.state.command_queue_consumer.stop() + consumer_stops.append(app.state.command_queue_consumer.stop()) + if consumer_stops: + # Stop both intake loops immediately. A sequential drain could let the + # second consumer keep admitting work for the full duration of a long + # review shutdown. + stop_results = await asyncio.gather( + *consumer_stops, + return_exceptions=True, + ) + for stop_result in stop_results: + if isinstance(stop_result, BaseException): + logger.warning("Error stopping queue consumer: %s", stop_result) # Close the RagClient HTTP pools owned by each service try: diff --git a/python-ecosystem/inference-orchestrator/src/api/routers/commands.py b/python-ecosystem/inference-orchestrator/src/api/routers/commands.py index de6046e6..58f9794b 100644 --- a/python-ecosystem/inference-orchestrator/src/api/routers/commands.py +++ b/python-ecosystem/inference-orchestrator/src/api/routers/commands.py @@ -3,6 +3,7 @@ """ import json import asyncio +import logging from typing import Dict, Any from fastapi import APIRouter, Request from starlette.responses import StreamingResponse @@ -14,6 +15,7 @@ from service.command.command_service import CommandService router = APIRouter(tags=["commands"]) +logger = logging.getLogger(__name__) def get_command_service(request: Request) -> CommandService: @@ -50,29 +52,52 @@ async def summarize_endpoint(req: SummarizeRequestDto, request: Request): # Streaming behavior async def event_stream(): queue = asyncio.Queue() - - yield _json_event({"type": "status", "state": "queued", "message": "summarize request received"}) - - def event_callback(event: Dict[str, Any]): - try: - queue.put_nowait(event) - except asyncio.QueueFull: - pass - - async def runner(): - try: - result = await command_service.process_summarize(req, event_callback=event_callback) - await queue.put({ - "type": "final", - "result": result - }) - except Exception as e: - await queue.put({"type": "error", "message": str(e)}) - - task = asyncio.create_task(runner()) - - async for event in _drain_queue_until_final(queue, task): - yield _json_event(event) + task = None + terminal_event_type = None + + try: + yield _json_event({"type": "status", "state": "queued", "message": "summarize request received"}) + + def event_callback(event: Dict[str, Any]): + nonlocal terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring streamed summarize event type=%s after " + "terminal type=%s", + event_type, + terminal_event_type, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type + try: + queue.put_nowait(event) + except asyncio.QueueFull: + pass + + async def runner(): + try: + result = await command_service.process_summarize(req, event_callback=event_callback) + event_callback({ + "type": "final", + "result": result + }) + except Exception as e: + event_callback({"type": "error", "message": str(e)}) + + task = asyncio.create_task(runner()) + + async for event in _drain_queue_until_final(queue, task): + yield _json_event(event) + finally: + if task is not None: + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass return StreamingResponse(event_stream(), media_type="application/x-ndjson") @@ -107,29 +132,52 @@ async def ask_endpoint(req: AskRequestDto, request: Request): # Streaming behavior async def event_stream(): queue = asyncio.Queue() - - yield _json_event({"type": "status", "state": "queued", "message": "ask request received"}) - - def event_callback(event: Dict[str, Any]): - try: - queue.put_nowait(event) - except asyncio.QueueFull: - pass - - async def runner(): - try: - result = await command_service.process_ask(req, event_callback=event_callback) - await queue.put({ - "type": "final", - "result": result - }) - except Exception as e: - await queue.put({"type": "error", "message": str(e)}) - - task = asyncio.create_task(runner()) - - async for event in _drain_queue_until_final(queue, task): - yield _json_event(event) + task = None + terminal_event_type = None + + try: + yield _json_event({"type": "status", "state": "queued", "message": "ask request received"}) + + def event_callback(event: Dict[str, Any]): + nonlocal terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring streamed ask event type=%s after terminal " + "type=%s", + event_type, + terminal_event_type, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type + try: + queue.put_nowait(event) + except asyncio.QueueFull: + pass + + async def runner(): + try: + result = await command_service.process_ask(req, event_callback=event_callback) + event_callback({ + "type": "final", + "result": result + }) + except Exception as e: + event_callback({"type": "error", "message": str(e)}) + + task = asyncio.create_task(runner()) + + async for event in _drain_queue_until_final(queue, task): + yield _json_event(event) + finally: + if task is not None: + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass return StreamingResponse(event_stream(), media_type="application/x-ndjson") diff --git a/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py b/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py index 615bbb77..a04a0242 100644 --- a/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py +++ b/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py @@ -59,18 +59,13 @@ class QaDocumentationRequest(BaseModel): # Changed file paths extracted from diff changed_file_paths: Optional[List[str]] = None - # VCS connection info (for RAG queries) + # Non-secret VCS identifiers used in document context vcs_provider: Optional[str] = None workspace_slug: Optional[str] = None repo_slug: Optional[str] = None source_branch: Optional[str] = None target_branch: Optional[str] = None - # OAuth credentials (for Python-side RAG/VCS access) - oauth_key: Optional[str] = None - oauth_secret: Optional[str] = None - bearer_token: Optional[str] = None - @field_validator("task_context", mode="before") @classmethod def normalize_task_context_values(cls, value): diff --git a/python-ecosystem/inference-orchestrator/src/api/routers/review.py b/python-ecosystem/inference-orchestrator/src/api/routers/review.py index 194ec3e6..d6244f5c 100644 --- a/python-ecosystem/inference-orchestrator/src/api/routers/review.py +++ b/python-ecosystem/inference-orchestrator/src/api/routers/review.py @@ -53,41 +53,67 @@ async def review_endpoint(req: ReviewRequestDto, request: Request): # Streaming behavior async def event_stream(): queue = asyncio.Queue() - - # Emit initial queued status - yield _json_event({"type": "status", "state": "queued", "message": "request received"}) - - # Event callback to capture service events - def event_callback(event: Dict[str, Any]): - try: - queue.put_nowait(event) - except asyncio.QueueFull: - pass # Skip if queue is full - - # Run processing in background - async def runner(): - try: - result = await review_service.process_review_request( - req, - event_callback=event_callback - ) - # Emit final event with result - final_event = { - "type": "final", - "result": result.get("result") - } - await queue.put(final_event) - except Exception as e: - await queue.put({ - "type": "error", - "message": str(e) - }) - - task = asyncio.create_task(runner()) - - # Drain queue and yield events - async for event in _drain_queue_until_final(queue, task): - yield _json_event(event) + task = None + terminal_event_type = None + + try: + # Emit initial queued status + yield _json_event({"type": "status", "state": "queued", "message": "request received"}) + + # Event callback to capture service events + def event_callback(event: Dict[str, Any]): + nonlocal terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring streamed review event type=%s after " + "terminal type=%s", + event_type, + terminal_event_type, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type + try: + queue.put_nowait(event) + except asyncio.QueueFull: + pass # Skip if queue is full + + # Run processing in background + async def runner(): + try: + result = await review_service.process_review_request( + req, + event_callback=event_callback + ) + # Emit final event with result + final_event = { + "type": "final", + "result": result.get("result") + } + event_callback(final_event) + except Exception as e: + event_callback({ + "type": "error", + "message": str(e) + }) + + task = asyncio.create_task(runner()) + + # Drain queue and yield events + async for event in _drain_queue_until_final(queue, task): + yield _json_event(event) + finally: + # A disconnected streaming client no longer owns useful work. + # Retire its runner before application shutdown can close the + # shared RagClient underneath it. + if task is not None: + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass return StreamingResponse(event_stream(), media_type="application/x-ndjson") diff --git a/python-ecosystem/inference-orchestrator/src/server/command_queue_consumer.py b/python-ecosystem/inference-orchestrator/src/server/command_queue_consumer.py index fb488468..a7952110 100644 --- a/python-ecosystem/inference-orchestrator/src/server/command_queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/src/server/command_queue_consumer.py @@ -2,7 +2,6 @@ import json import logging import os -import traceback from typing import Dict, Any, Optional import redis.asyncio as redis from pydantic import ValidationError @@ -35,6 +34,22 @@ def __init__(self, command_service: CommandService): self.is_running = False self._redis: Optional[redis.Redis] = None self._task: Optional[asyncio.Task] = None + self._consumer_heartbeat_task: Optional[asyncio.Task] = None + self._job_tasks: set[asyncio.Task] = set() + self._redis_outage_channels: set[str] = set() + self.consumer_heartbeat_key = "codecrow:commands:consumer:heartbeat" + self.consumer_heartbeat_seconds = max( + 1.0, + float(os.environ.get("COMMAND_CONSUMER_HEARTBEAT_SECONDS", "5")), + ) + self.consumer_heartbeat_ttl_seconds = max( + 15, + int(self.consumer_heartbeat_seconds * 3), + ) + self.event_ttl_seconds = max( + 60, + int(os.environ.get("COMMAND_EVENT_TTL_SECONDS", "3600")), + ) max_concurrent = int(os.environ.get("MAX_CONCURRENT_COMMANDS", "10")) self._job_semaphore = asyncio.Semaphore(max_concurrent) @@ -52,6 +67,10 @@ async def start(self): health_check_interval=30, ) self.is_running = True + await self._publish_consumer_heartbeat() + self._consumer_heartbeat_task = asyncio.create_task( + self._consumer_heartbeat_loop() + ) self._task = asyncio.create_task(self._consume_loop()) async def stop(self): @@ -63,32 +82,94 @@ async def stop(self): await self._task except asyncio.CancelledError: pass + if self._consumer_heartbeat_task: + self._consumer_heartbeat_task.cancel() + try: + await self._consumer_heartbeat_task + except asyncio.CancelledError: + pass + + # A dequeued command is admitted work. Do not close Redis or the + # shared RagClient underneath an in-flight command. + active_jobs = tuple(self._job_tasks) + if active_jobs: + logger.info( + "Waiting for %s admitted command jobs before shutdown", + len(active_jobs), + ) + await asyncio.gather(*active_jobs, return_exceptions=True) if self._redis: await self._redis.aclose() logger.info("Command Queue Consumer stopped") + async def _publish_consumer_heartbeat(self): + if self._redis: + try: + await self._redis.set( + self.consumer_heartbeat_key, + "alive", + ex=self.consumer_heartbeat_ttl_seconds, + ) + self._record_redis_success("command consumer heartbeat") + except Exception as error: + self._record_redis_failure("command consumer heartbeat", error) + raise + + async def _consumer_heartbeat_loop(self): + while self.is_running: + try: + await self._publish_consumer_heartbeat() + except asyncio.CancelledError: + raise + except Exception: + pass + await asyncio.sleep(self.consumer_heartbeat_seconds) + async def _consume_loop(self): """Infinite loop blocking on the Redis queue for new jobs.""" logger.info(f"Listening for jobs on '{self.job_queue_key}'...") while self.is_running: + permit_acquired = False try: + # Reserve capacity before removing durable work from Redis. + await self._job_semaphore.acquire() + permit_acquired = True + if not self.is_running: + break + result = await self._redis.brpop([self.job_queue_key], timeout=1) + self._record_redis_success("command queue read") if not result: continue queue_name, payload_str = result logger.debug(f"Received raw command job payload from {queue_name}") - asyncio.create_task(self._bounded_handle_job(payload_str)) + job_task = asyncio.create_task( + self._handle_admitted_job(payload_str) + ) + self._job_tasks.add(job_task) + job_task.add_done_callback(self._job_tasks.discard) + permit_acquired = False except asyncio.CancelledError: break except RedisTimeoutError as error: - logger.warning("Redis command queue read timed out; retrying: %s", error) + self._record_redis_failure("command queue read", error) await asyncio.sleep(1) except Exception as e: - logger.error(f"Error in Command Queue consume loop: {e}", exc_info=True) + self._record_redis_failure("command queue read", e) await asyncio.sleep(2) + finally: + if permit_acquired: + self._job_semaphore.release() + + async def _handle_admitted_job(self, payload_str: str): + """Process a job using the capacity reserved before dequeue.""" + try: + await self._handle_job(payload_str) + finally: + self._job_semaphore.release() async def _bounded_handle_job(self, payload_str: str): """Acquire the concurrency semaphore before processing a job.""" @@ -100,6 +181,8 @@ async def _handle_job(self, payload_str: str): job_id = "UNKNOWN" event_queue_key = None command_type = "UNKNOWN" + publish_tail: Optional[asyncio.Future] = None + terminal_event_type: Optional[str] = None try: payload = json.loads(payload_str) @@ -113,9 +196,34 @@ async def _handle_job(self, payload_str: str): event_queue_key = f"codecrow:analysis:events:{job_id}" logger.info(f"Processing Command Job ID: {job_id} (Type: {command_type})") - + + # Serialize callback publications so progress cannot overtake the + # terminal result, and retain the tail until the job completes. + loop = asyncio.get_running_loop() + publish_tail = loop.create_future() + publish_tail.set_result(None) + def event_callback(event: Dict[str, Any]): - asyncio.create_task(self._publish_event(event_queue_key, event)) + nonlocal publish_tail, terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring command event type=%s after terminal type=%s " + "for job=%s", + event_type, + terminal_event_type, + job_id, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type + previous = publish_tail + + async def publish_after_previous(): + await previous + await self._publish_event(event_queue_key, event) + + publish_tail = asyncio.create_task(publish_after_previous()) event_callback({ "type": "status", @@ -135,11 +243,12 @@ def event_callback(event: Dict[str, Any]): if self._has_error(result): error_message = self._get_result_value(result, "error", "AI command failed") - await self._publish_event(event_queue_key, { + event_callback({ "type": "error", "message": str(error_message) }) - logger.warning(f"Command Job ID {job_id} failed: {error_message}") + await publish_tail + logger.info(f"Command Job ID {job_id} failed: {error_message}") return # Format output correctly depending on command type based on their DTO responses @@ -147,11 +256,12 @@ def event_callback(event: Dict[str, Any]): if command_type == "summarize": summary = self._get_result_value(result, "summary") if not self._has_usable_text(summary): - await self._publish_event(event_queue_key, { + event_callback({ "type": "error", "message": "AI service returned an empty summary" }) - logger.warning(f"Command Job ID {job_id} failed: empty summarize result") + await publish_tail + logger.info(f"Command Job ID {job_id} failed: empty summarize result") return final_payload = { @@ -162,11 +272,12 @@ def event_callback(event: Dict[str, Any]): elif command_type == "ask": answer = self._get_result_value(result, "answer") if not self._has_usable_text(answer): - await self._publish_event(event_queue_key, { + event_callback({ "type": "error", "message": "AI service returned an empty answer" }) - logger.warning(f"Command Job ID {job_id} failed: empty ask result") + await publish_tail + logger.info(f"Command Job ID {job_id} failed: empty ask result") return final_payload = { @@ -174,23 +285,33 @@ def event_callback(event: Dict[str, Any]): } event_callback({"type": "final", "result": final_payload}) - + await publish_tail logger.info(f"Command Job ID {job_id} processing completed successfully.") except ValidationError as ve: logger.error(f"Command Job ID {job_id} Validation Error: {ve}") if event_queue_key: - await self._publish_event(event_queue_key, { + event = { "type": "error", "message": f"Input validation error: {str(ve)}" - }) + } + if publish_tail is None: + await self._publish_event(event_queue_key, event) + else: + event_callback(event) + await publish_tail except Exception as e: logger.error(f"Command Job ID {job_id} Unhandled Error: {e}", exc_info=True) if event_queue_key: - await self._publish_event(event_queue_key, { + event = { "type": "error", "message": f"Internal orchestrator command error: {str(e)}" - }) + } + if publish_tail is None: + await self._publish_event(event_queue_key, event) + else: + event_callback(event) + await publish_tail async def _publish_event(self, key: str, event: Dict[str, Any]): """Publish an event back to the job's specific event list. LPUSH (Java uses rightPop).""" @@ -198,9 +319,42 @@ async def _publish_event(self, key: str, event: Dict[str, Any]): if not self._redis: return event_json = json.dumps(event) - await self._redis.lpush(key, event_json) + pipeline = self._redis.pipeline() + pipeline.lpush(key, event_json) + pipeline.expire(key, self.event_ttl_seconds) + await pipeline.execute() + self._record_redis_success("command event publication") except Exception as e: - logger.error(f"Failed to publish event to {key}: {e}") + self._record_redis_failure("command event publication", e) + + def _record_redis_failure(self, operation: str, error: Exception) -> None: + """Emit one actionable diagnostic per continuous Redis outage.""" + channel = self._redis_diagnostic_channel(operation) + if channel not in self._redis_outage_channels: + self._redis_outage_channels.add(channel) + logger.warning( + "Redis unavailable during %s; queue/event delivery is " + "degraded: %s", + operation, + error, + ) + return + logger.debug( + "Redis remains unavailable during %s: %s", + operation, + error, + ) + + def _record_redis_success(self, operation: str) -> None: + channel = self._redis_diagnostic_channel(operation) + if channel not in self._redis_outage_channels: + return + self._redis_outage_channels.discard(channel) + logger.info("Redis connectivity restored during %s", operation) + + @staticmethod + def _redis_diagnostic_channel(operation: str) -> str: + return "read" if operation.endswith("queue read") else "write" @staticmethod def _get_result_value(result: Any, key: str, default: Any = None) -> Any: diff --git a/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py b/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py index 63a4bb07..110ae9d7 100644 --- a/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py @@ -2,7 +2,6 @@ import json import logging import os -import traceback from typing import Dict, Any, Optional import redis.asyncio as redis from pydantic import ValidationError @@ -31,6 +30,8 @@ def __init__(self, review_service: ReviewService): self._redis: Optional[redis.Redis] = None self._task: Optional[asyncio.Task] = None self._consumer_heartbeat_task: Optional[asyncio.Task] = None + self._job_tasks: set[asyncio.Task] = set() + self._redis_outage_channels: set[str] = set() self.consumer_heartbeat_key = "codecrow:analysis:consumer:heartbeat" self.consumer_heartbeat_seconds = max( 1.0, @@ -86,6 +87,16 @@ async def stop(self): await self._consumer_heartbeat_task except asyncio.CancelledError: pass + + # Removing a job from Redis admits durable work. Keep its shared + # Redis/RAG clients alive until every admitted review has finished. + active_jobs = tuple(self._job_tasks) + if active_jobs: + logger.info( + "Waiting for %s admitted review jobs before shutdown", + len(active_jobs), + ) + await asyncio.gather(*active_jobs, return_exceptions=True) if self._redis: await self._redis.aclose() @@ -93,11 +104,19 @@ async def stop(self): async def _publish_consumer_heartbeat(self): if self._redis: - await self._redis.set( - self.consumer_heartbeat_key, - "alive", - ex=self.consumer_heartbeat_ttl_seconds, - ) + try: + await self._redis.set( + self.consumer_heartbeat_key, + "alive", + ex=self.consumer_heartbeat_ttl_seconds, + ) + self._record_redis_success("review consumer heartbeat") + except Exception as error: + self._record_redis_failure( + "review consumer heartbeat", + error, + ) + raise async def _consumer_heartbeat_loop(self): while self.is_running: @@ -106,7 +125,9 @@ async def _consumer_heartbeat_loop(self): except asyncio.CancelledError: raise except Exception: - logger.exception("Failed to publish review consumer heartbeat") + # The transition diagnostic is owned by + # _publish_consumer_heartbeat; keep the loop alive quietly. + pass await asyncio.sleep(self.consumer_heartbeat_seconds) async def is_healthy(self) -> bool: @@ -143,6 +164,7 @@ async def _consume_loop(self): # Block until a job is available or timeout (1 second for graceful shutdown check) result = await self._redis.brpop([self.job_queue_key], timeout=1) + self._record_redis_success("review queue read") if not result: continue @@ -151,16 +173,20 @@ async def _consume_loop(self): logger.debug(f"Received raw job payload from {queue_name}") # Transfer ownership of the reserved permit to the job task. - asyncio.create_task(self._handle_admitted_job(payload_str)) + job_task = asyncio.create_task( + self._handle_admitted_job(payload_str) + ) + self._job_tasks.add(job_task) + job_task.add_done_callback(self._job_tasks.discard) permit_acquired = False except asyncio.CancelledError: break except RedisTimeoutError as error: - logger.warning("Redis review queue read timed out; retrying: %s", error) + self._record_redis_failure("review queue read", error) await asyncio.sleep(1) except Exception as e: - logger.error(f"Error in Redis consume loop: {e}", exc_info=True) + self._record_redis_failure("review queue read", e) await asyncio.sleep(2) # Backoff on error finally: if permit_acquired: @@ -183,6 +209,7 @@ async def _handle_job(self, payload_str: str): job_id = "UNKNOWN" event_queue_key = None publish_tail: Optional[asyncio.Future] = None + terminal_event_type: Optional[str] = None try: payload = json.loads(payload_str) @@ -216,7 +243,19 @@ async def _handle_job(self, payload_str: str): publish_tail.set_result(None) def event_callback(event: Dict[str, Any]): - nonlocal publish_tail + nonlocal publish_tail, terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring review event type=%s after terminal type=%s " + "for job=%s", + event_type, + terminal_event_type, + job_id, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type previous = publish_tail async def publish_after_previous(): @@ -300,5 +339,37 @@ async def _publish_event(self, key: str, event: Dict[str, Any]): pipeline.lpush(key, event_str) pipeline.expire(key, 3600) await pipeline.execute() + self._record_redis_success("review event publication") except Exception as e: - logger.error(f"Failed to publish event to {key}: {e}") + self._record_redis_failure("review event publication", e) + + def _record_redis_failure(self, operation: str, error: Exception) -> None: + """Emit one actionable diagnostic per continuous Redis outage.""" + channel = self._redis_diagnostic_channel(operation) + if channel not in self._redis_outage_channels: + self._redis_outage_channels.add(channel) + logger.warning( + "Redis unavailable during %s; queue/event delivery is " + "degraded: %s", + operation, + error, + ) + return + logger.debug( + "Redis remains unavailable during %s: %s", + operation, + error, + ) + + def _record_redis_success(self, operation: str) -> None: + channel = self._redis_diagnostic_channel(operation) + if channel not in self._redis_outage_channels: + return + self._redis_outage_channels.discard(channel) + logger.info("Redis connectivity restored during %s", operation) + + @staticmethod + def _redis_diagnostic_channel(operation: str) -> str: + # A successful blocking read does not prove that Redis accepts event + # writes (for example during READONLY/OOM states). + return "read" if operation.endswith("queue read") else "write" diff --git a/python-ecosystem/inference-orchestrator/src/service/command/command_service.py b/python-ecosystem/inference-orchestrator/src/service/command/command_service.py index 037409ac..b69cf0f6 100644 --- a/python-ecosystem/inference-orchestrator/src/service/command/command_service.py +++ b/python-ecosystem/inference-orchestrator/src/service/command/command_service.py @@ -20,6 +20,14 @@ logger = logging.getLogger(__name__) +def _rag_response_error(response: Any) -> Optional[str]: + if not isinstance(response, dict) or response.get("status") != "error": + return None + detail = response.get("error") or response.get("detail") or "unknown failure" + status_code = response.get("status_code") + return f"status={status_code} detail={detail}" if status_code else str(detail) + + class CommandService: """Service class for handling CodeCrow commands with AI integration.""" @@ -122,6 +130,7 @@ async def process_summarize( result = self._normalize_summarize_result(result, supports_mermaid=False) if "error" in result: + logger.error("Summarize failed: %s", result["error"]) self._emit_event(event_callback, {"type": "error", "message": result["error"]}) return result @@ -237,6 +246,7 @@ async def process_ask( result = self._normalize_ask_result(result) if "error" in result: + logger.error("Ask failed: %s", result["error"]) self._emit_event(event_callback, {"type": "error", "message": result["error"]}) return result @@ -385,6 +395,19 @@ async def _fetch_rag_context_for_summarize( base_branch=request.get_rag_base_branch(), ) + if rag_error := _rag_response_error(rag_response): + logger.warning( + "Optional RAG context unavailable for summarize; " + "continuing without it: %s", + rag_error, + ) + self._emit_event(event_callback, { + "type": "status", + "state": "rag_skipped", + "message": "Codebase context unavailable; summary generation continues", + }) + return None + if rag_response and rag_response.get("context"): self._emit_event(event_callback, { "type": "status", @@ -739,7 +762,7 @@ async def _execute_summarize( if "error" not in result: return result - logger.warning("Summarize streaming produced an empty final summary; retrying without output_schema") + logger.info("Summarize streaming produced an empty final summary; retrying without output_schema") self._emit_event(event_callback, { "type": "status", "state": "retrying", @@ -756,7 +779,7 @@ async def _execute_summarize( if "error" not in result: return result - logger.warning("Summarize agent retry also produced an empty summary; trying direct LLM fallback") + logger.info("Summarize agent retry also produced an empty summary; trying direct LLM fallback") direct_response = await llm.ainvoke( prompt + "\n\nIf tool calls are unavailable, summarize from the context already provided. " @@ -765,7 +788,7 @@ async def _execute_summarize( return self._coerce_summarize_final_result(direct_response, supports_mermaid) except Exception as e: - logger.warning(f"Summarize streaming failed, retrying without output_schema: {e}", exc_info=True) + logger.info("Summarize streaming failed; retrying without output_schema: %s", e) self._emit_event(event_callback, { "type": "status", "state": "retrying", @@ -789,7 +812,7 @@ async def _execute_summarize( ) return self._coerce_summarize_final_result(direct_response, supports_mermaid) except Exception as retry_error: - logger.error(f"Summarize agent error: {retry_error}", exc_info=True) + logger.debug("Summarize retries exhausted: %s", retry_error, exc_info=True) sanitized_msg = create_user_friendly_error(retry_error) return {"error": sanitized_msg} @@ -966,7 +989,7 @@ async def _execute_ask( if "error" not in result: return result - logger.warning("Ask streaming produced an empty final answer; retrying without output_schema") + logger.info("Ask streaming produced an empty final answer; retrying without output_schema") self._emit_event(event_callback, { "type": "status", "state": "retrying", @@ -983,7 +1006,7 @@ async def _execute_ask( if "error" not in result: return result - logger.warning("Ask agent retry also produced an empty answer; trying direct LLM fallback") + logger.info("Ask agent retry also produced an empty answer; trying direct LLM fallback") direct_response = await llm.ainvoke( prompt + "\n\nIf tool calls are unavailable, answer from the context already provided. " @@ -992,7 +1015,7 @@ async def _execute_ask( return self._coerce_ask_final_result(direct_response) except Exception as e: - logger.error(f"Ask agent error: {e}", exc_info=True) + logger.debug("Ask agent retries exhausted: %s", e, exc_info=True) sanitized_msg = create_user_friendly_error(e) return {"error": sanitized_msg} @@ -1153,7 +1176,7 @@ def _parse_json_response(self, response: str) -> Optional[Dict[str, Any]]: except json.JSONDecodeError: pass - logger.warning(f"Failed to parse JSON from response: {response[:200]}...") + logger.debug("Failed to parse JSON from response: %s...", response[:200]) return None def _extract_json_object(self, text: str) -> Optional[str]: diff --git a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/base_orchestrator.py b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/base_orchestrator.py index d9ba4ee7..70e39df4 100644 --- a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/base_orchestrator.py +++ b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/base_orchestrator.py @@ -3,16 +3,15 @@ Provides shared infrastructure for multi-stage LLM pipelines: - LLM instance management -- RAG indexing / cleanup lifecycle - Smart dependency-aware batching via DependencyGraphBuilder - Diff filtering for per-batch file subsets - Event emission helpers -Subclasses: MultiStageReviewOrchestrator, QaDocOrchestrator +Subclass: QaDocOrchestrator """ import re import logging -from abc import ABC, abstractmethod +from abc import ABC from typing import Dict, Any, List, Optional, Callable, Set from model.enrichment import PrEnrichmentDataDto @@ -55,9 +54,7 @@ class BaseOrchestrator(ABC): Provides: - ``self.llm`` — the LangChain LLM instance - - ``self.rag_client`` — optional RAG client for hybrid queries - ``self.event_callback`` — optional SSE/WS event emitter - - ``index_pr_files()`` / ``cleanup_pr_files()`` — RAG lifecycle - ``build_dependency_batches()`` — smart batching from enrichment data - ``filter_diff_for_files()`` — per-batch diff slicing """ @@ -65,103 +62,10 @@ class BaseOrchestrator(ABC): def __init__( self, llm, - rag_client=None, event_callback: Optional[Callable[[Dict], None]] = None, ): self.llm = llm - self.rag_client = rag_client self.event_callback = event_callback - self._pr_number: Optional[int] = None - self._pr_indexed: bool = False - - # ── RAG lifecycle ──────────────────────────────────────────────── - - async def index_pr_files( - self, - *, - workspace: str, - project: str, - pr_number: int, - branch: str, - enrichment_data: Optional[PrEnrichmentDataDto], - changed_file_paths: Optional[List[str]], - diff: Optional[str] = None, - ) -> None: - """ - Index PR files into RAG for hybrid context queries. - - Uses full file content from enrichment data when available; - falls back to diff hunks otherwise. - """ - if not self.rag_client or not pr_number: - return - - # Build enrichment lookup: path → full file content - enrichment_lookup: Dict[str, str] = {} - if enrichment_data and enrichment_data.fileContents: - for fc in enrichment_data.fileContents: - if fc.content and not fc.skipped: - enrichment_lookup[fc.path] = fc.content - parts = fc.path.split("/", 1) - if len(parts) > 1: - enrichment_lookup[parts[1]] = fc.content - - # Build file list for indexing - files: List[Dict[str, str]] = [] - paths_to_index = changed_file_paths or [] - for path in paths_to_index: - content = enrichment_lookup.get(path, "") - if not content: - # Suffix matching for path variations - for ep, ec in enrichment_lookup.items(): - if path.endswith(ep) or ep.endswith(path): - content = ec - break - if content: - files.append({"path": path, "content": content, "change_type": "MODIFIED"}) - - if not files: - logger.info("No files to index for PR #%s", pr_number) - return - - self._pr_number = pr_number - try: - result = await self.rag_client.index_pr_files( - workspace=workspace, - project=project, - pr_number=pr_number, - branch=branch, - files=files, - ) - if result.get("status") in {"indexed", "reused"}: - self._pr_indexed = True - logger.info( - "%s PR #%s overlay: %s chunks", - "Reused" if result.get("status") == "reused" else "Indexed", - pr_number, - result.get("chunks_indexed", 0), - ) - else: - logger.warning("Failed to index PR files: %s", result) - except Exception as e: - logger.warning("Error indexing PR files: %s", e) - - async def cleanup_pr_files(self, workspace: str, project: str) -> None: - """Delete PR-indexed data (idempotent).""" - if not self._pr_number or not self.rag_client: - return - try: - await self.rag_client.delete_pr_files( - workspace=workspace, - project=project, - pr_number=self._pr_number, - ) - logger.info("Cleaned up PR #%s indexed data", self._pr_number) - except Exception as e: - logger.warning("Failed to cleanup PR files: %s", e) - finally: - self._pr_number = None - self._pr_indexed = False # ── Smart batching ─────────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_orchestrator.py b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_orchestrator.py index ddbff833..659f5adb 100644 --- a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_orchestrator.py +++ b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_orchestrator.py @@ -6,7 +6,7 @@ - Stage 2: Cross-file impact analysis (how changes interact for testing) - Stage 3: Aggregation into polished QA document (or delta update for re-runs) -Extends BaseOrchestrator for shared RAG, batching, and LLM infrastructure. +Extends BaseOrchestrator for shared batching and LLM infrastructure. """ import asyncio import json @@ -57,10 +57,9 @@ class QaDocOrchestrator(BaseOrchestrator): def __init__( self, llm, - rag_client=None, event_callback: Optional[Callable[[Dict], None]] = None, ): - super().__init__(llm, rag_client, event_callback) + super().__init__(llm, event_callback) async def run( self, @@ -132,10 +131,6 @@ async def run( changed_file_paths=changed_file_paths or [], previous_documentation=previous_documentation, is_same_pr_rerun=is_same_pr_rerun, - workspace_slug=workspace_slug, - repo_slug=repo_slug, - pr_number=pr_number, - source_branch=source_branch, ) else: logger.info( @@ -188,25 +183,9 @@ async def _run_multi_stage( changed_file_paths: List[str], previous_documentation: Optional[str], is_same_pr_rerun: bool, - workspace_slug: Optional[str], - repo_slug: Optional[str], - pr_number: Optional[int], - source_branch: Optional[str], ) -> str: """Execute the 3-stage ULTRATHINKING pipeline.""" try: - # Index files into RAG if available - if self.rag_client and pr_number and workspace_slug and repo_slug: - await self.index_pr_files( - workspace=workspace_slug, - project=repo_slug, - pr_number=pr_number, - branch=source_branch or "unknown", - enrichment_data=enrichment_data, - changed_file_paths=changed_file_paths, - diff=diff, - ) - # For same-PR re-runs, analyze the delta diff — but only if # it contains actual hunks (@@). A delta that is truthy but # header-only (no @@) would starve the whole pipeline of context. @@ -298,9 +277,6 @@ async def _run_multi_stage( placeholders=placeholders, previous_documentation=previous_documentation, ) - finally: - if workspace_slug and repo_slug: - await self.cleanup_pr_files(workspace_slug, repo_slug) # ── Stage 1: Batch Analysis ────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_service.py b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_service.py index beb9e838..51c406bc 100644 --- a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_service.py +++ b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_service.py @@ -19,7 +19,6 @@ from llm.llm_factory import LLMFactory from model.enrichment import PrEnrichmentDataDto from service.qa_documentation.qa_doc_orchestrator import QaDocOrchestrator -from service.rag.rag_client import RagClient logger = logging.getLogger(__name__) @@ -28,8 +27,8 @@ class QaDocumentationService: """ Generates QA-oriented documentation for completed PR analyses. - Creates the LLM and RAG client, then delegates to QaDocOrchestrator - which runs the 3-stage pipeline or single-pass depending on PR size. + Creates the LLM, then delegates to QaDocOrchestrator which runs the + 3-stage pipeline or single-pass depending on PR size. """ def __init__(self): @@ -37,7 +36,6 @@ def __init__(self): self._ai_provider = os.environ.get("QA_DOC_AI_PROVIDER", os.environ.get("AI_PROVIDER", "openrouter")) self._ai_model = os.environ.get("QA_DOC_AI_MODEL", os.environ.get("AI_MODEL", "google/gemini-2.0-flash")) self._ai_api_key = os.environ.get("QA_DOC_AI_API_KEY", os.environ.get("AI_API_KEY", "")) - self._rag_pipeline_url = os.environ.get("RAG_PIPELINE_URL", "http://rag-pipeline:8020") # ------------------------------------------------------------------ # Public API @@ -88,12 +86,7 @@ async def generate( max_tokens=16_384, # QA docs need room for structured JSON output ) - rag_client = self._create_rag_client() - - orchestrator = QaDocOrchestrator( - llm=llm, - rag_client=rag_client, - ) + orchestrator = QaDocOrchestrator(llm=llm) result = await orchestrator.run( project_name=project_name, @@ -143,12 +136,3 @@ def _create_llm(self, ai_provider=None, ai_model=None, ai_api_key=None, ai_base_ ai_base_url=ai_base_url, max_tokens=max_tokens, ) - - def _create_rag_client(self) -> Optional[RagClient]: - """Create RAG client if the RAG pipeline URL is configured.""" - try: - if self._rag_pipeline_url: - return RagClient(base_url=self._rag_pipeline_url) - except Exception as e: - logger.warning("Failed to create RAG client (non-critical): %s", e) - return None diff --git a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py index 49895c8c..901968b1 100644 --- a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py +++ b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py @@ -83,6 +83,7 @@ def __init__(self, base_url: Optional[str] = None, enabled: Optional[bool] = Non os.environ.get("SERVICE_SECRET") or os.environ.get("CODECROW_RAG_API_SECRET", "") ) + self._cleanup_degraded = False if self.enabled: logger.info(f"RAG client initialized: {self.base_url}") @@ -108,6 +109,18 @@ async def close(self): await self._client.aclose() self._client = None + def _record_cleanup_failure(self, detail: str) -> None: + if not self._cleanup_degraded: + logger.warning("RAG PR cleanup is degraded: %s", detail) + self._cleanup_degraded = True + else: + logger.debug("RAG PR cleanup remains degraded: %s", detail) + + def _record_cleanup_success(self) -> None: + if self._cleanup_degraded: + logger.info("RAG PR cleanup recovered") + self._cleanup_degraded = False + async def get_pr_context( self, workspace: str, @@ -232,7 +245,7 @@ async def get_pr_context( except httpx.HTTPError as e: status_code, detail = _http_error_detail(e) - logger.warning( + logger.debug( "Failed to retrieve PR context from RAG: status=%s detail=%s", status_code or "transport-error", detail, @@ -243,7 +256,7 @@ async def get_pr_context( "error": detail, } except Exception as e: - logger.error(f"Unexpected error querying RAG: {e}") + logger.debug("Unexpected error querying RAG: %s", e, exc_info=True) return { "status": "error", "status_code": None, @@ -308,7 +321,7 @@ async def semantic_search( except httpx.HTTPError as e: status_code, detail = _http_error_detail(e) - logger.warning( + logger.debug( "Semantic search failed: status=%s detail=%s", status_code or "transport-error", detail, @@ -320,7 +333,7 @@ async def semantic_search( "results": [], } except Exception as e: - logger.error(f"Unexpected error in semantic search: {e}") + logger.debug("Unexpected error in semantic search: %s", e, exc_info=True) return { "status": "error", "status_code": None, @@ -497,7 +510,7 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: "revision-bound duplication search failed: " f"{type(e).__name__}: {e}" ) from e - logger.warning(f"Failed duplication search: {e}") + logger.debug("Failed duplication search: %s", e, exc_info=True) return [] async def get_deterministic_context( @@ -600,7 +613,7 @@ async def get_deterministic_context( except httpx.HTTPError as e: status_code, detail = _http_error_detail(e) - logger.warning( + logger.debug( "Failed to retrieve deterministic context: status=%s detail=%s", status_code or "transport-error", detail, @@ -611,7 +624,11 @@ async def get_deterministic_context( "error": detail, } except Exception as e: - logger.error(f"Unexpected error in deterministic RAG query: {e}") + logger.debug( + "Unexpected error in deterministic RAG query: %s", + e, + exc_info=True, + ) return { "status": "error", "status_code": None, @@ -719,7 +736,7 @@ async def index_pr_files( except httpx.HTTPError as e: status_code, detail = _http_error_detail(e) - logger.warning( + logger.debug( "Failed to index PR files: status=%s detail=%s timeout=%.1fs", status_code or "transport-error", detail, @@ -731,7 +748,11 @@ async def index_pr_files( "error": detail, } except Exception as e: - logger.error(f"Unexpected error indexing PR files: {e}") + logger.debug( + "Unexpected error indexing PR files: %s", + e, + exc_info=True, + ) return {"status": "error", "error": str(e)} async def delete_pr_files( @@ -768,13 +789,35 @@ async def delete_pr_files( ) response.raise_for_status() result = response.json() - - logger.info(f"Deleted PR #{pr_number} indexed data") - return result.get("status") == "deleted" + + status = result.get("status") + if status == "deleted": + self._record_cleanup_success() + logger.info("Deleted PR #%s indexed data", pr_number) + return True + if status == "skipped": + self._record_cleanup_success() + logger.info( + "PR #%s indexed-data cleanup was already complete: %s", + pr_number, + result.get("message") or "nothing to delete", + ) + return True + self._record_cleanup_failure( + "PR #%s returned unexpected status %s" + % (pr_number, status or "missing") + ) + return False except httpx.HTTPError as e: - logger.warning(f"Failed to delete PR files: {e}") + status_code, detail = _http_error_detail(e) + self._record_cleanup_failure( + "status=%s detail=%s" + % (status_code or "transport-error", detail) + ) return False except Exception as e: - logger.error(f"Unexpected error deleting PR files: {e}") + self._record_cleanup_failure( + f"unexpected {type(e).__name__}: {e}" + ) return False diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py index 888736fb..eb2a2e73 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py @@ -61,7 +61,6 @@ execute_stage_3_aggregation, _emit_status, _emit_progress, - _emit_error, ) from service.review.plugin_context import ( apply_effective_project_capabilities, @@ -168,6 +167,23 @@ def _resolve_enrichment_content( INTERNAL_PR_INDEX_ENABLED = _env_bool("REVIEW_INTERNAL_PR_INDEX_ENABLED", True) VERIFICATION_ENABLED = _env_bool("REVIEW_VERIFICATION_ENABLED", True) +_REQUEST_RAG_BINDING_FIELDS = ( + "ragCollectionTarget", + "ragBaseGenerationManifestSha256", + "ragPrGenerationFingerprint", + "ragPrOverlayGenerationManifestSha256", + "ragBasePluginFingerprint", + "ragBasePluginDescriptorFingerprint", + "ragBasePluginImplementationFingerprint", + "ragBaseIndexRepresentationFingerprint", +) + + +def _clear_request_rag_bindings(request: ReviewRequestDto) -> None: + """Remove host-provided RAG bindings when the project disables RAG.""" + for field_name in _REQUEST_RAG_BINDING_FIELDS: + setattr(request, field_name, None) + def _review_log_id(request: ReviewRequestDto) -> str: return ( @@ -318,7 +334,11 @@ async def _index_pr_files( as a complete repository artifact. """ self._repository_review_groups = () + self._pr_indexed = False + request.ragPrGenerationFingerprint = None + request.ragPrOverlayGenerationManifestSha256 = None if not request.ragEnabled: + _clear_request_rag_bindings(request) logger.info("PR file indexing skipped because project RAG is disabled") return if not INTERNAL_PR_INDEX_ENABLED: @@ -453,16 +473,21 @@ async def _index_pr_files( request, result.get("effective_project_capabilities"), ) - request.ragBaseGenerationManifestSha256 = ( + base_generation_manifest = ( result.get("base_generation_manifest_sha256") or request.ragBaseGenerationManifestSha256 ) - request.ragPrGenerationFingerprint = result.get( + pr_generation_fingerprint = result.get( "generation_fingerprint" ) - request.ragPrOverlayGenerationManifestSha256 = result.get( + overlay_generation_manifest = result.get( "overlay_generation_manifest_sha256" ) + request.ragBaseGenerationManifestSha256 = ( + base_generation_manifest + ) + request.ragPrGenerationFingerprint = None + request.ragPrOverlayGenerationManifestSha256 = None request.ragBasePluginFingerprint = ( result.get("plugin_fingerprint") or request.ragBasePluginFingerprint @@ -479,7 +504,6 @@ async def _index_pr_files( result.get("index_representation_fingerprint") or request.ragBaseIndexRepresentationFingerprint ) - self._pr_indexed = True self._repository_review_groups = tuple( tuple( path for path in group @@ -488,21 +512,51 @@ async def _index_pr_files( for group in (result.get("review_groups") or ()) if isinstance(group, (list, tuple)) ) - logger.info( - "%s PR #%s overlay: %s chunks, %s partial files, " - "%s repository review groups", - "Reused" if result.get("status") == "reused" else "Indexed", - pr_number, - result.get("chunks_indexed", 0), - len(result.get("partial_files") or ()), - len(self._repository_review_groups), + complete_overlay_binding = all( + isinstance(value, str) and bool(value.strip()) + for value in ( + identity.head_revision, + identity.base_revision, + request.ragCollectionTarget, + base_generation_manifest, + pr_generation_fingerprint, + overlay_generation_manifest, + ) ) + if complete_overlay_binding: + request.ragPrGenerationFingerprint = ( + pr_generation_fingerprint + ) + request.ragPrOverlayGenerationManifestSha256 = ( + overlay_generation_manifest + ) + self._pr_indexed = True + logger.info( + "%s PR #%s overlay: %s chunks, %s partial files, " + "%s repository review groups", + "Reused" if result.get("status") == "reused" else "Indexed", + pr_number, + result.get("chunks_indexed", 0), + len(result.get("partial_files") or ()), + len(self._repository_review_groups), + ) + else: + self._pr_indexed = False + self._repository_review_groups = () + logger.info( + "PR #%s overlay was prepared without a complete generation " + "lease; continuing with target-branch and local evidence", + pr_number, + ) elif result.get("status") == "skipped": logger.info("PR indexing skipped: %s", result) else: status_code = result.get("status_code") detail = result.get("error") or result - logger.warning( + log_unavailable = ( + logger.info if status_code == 409 else logger.warning + ) + log_unavailable( "PR context indexing unavailable%s; continuing review without " "the PR overlay: %s", f" (HTTP {status_code})" if status_code else "", @@ -529,13 +583,20 @@ async def _cleanup_pr_files(self, request: ReviewRequestDto) -> None: return try: - await self.rag_client.delete_pr_files( + deleted = await self.rag_client.delete_pr_files( workspace=request.projectWorkspace, project=request.projectNamespace, pr_number=self._pr_number, collection_target=request.ragCollectionTarget, ) - logger.info(f"Cleaned up PR #{self._pr_number} indexed data") + if deleted: + logger.info("Cleaned up PR #%s indexed data", self._pr_number) + else: + logger.info( + "PR #%s indexed-data cleanup did not complete; the RAG " + "client recorded the failure detail", + self._pr_number, + ) except Exception as e: logger.warning(f"Failed to cleanup PR files: {e}") finally: @@ -889,6 +950,11 @@ async def orchestrate_review( Main entry point for the multi-stage review. Supports both FULL (initial review) and INCREMENTAL (follow-up review) modes. """ + request_rag_client = self.rag_client if request.ragEnabled else None + request_rag_context = rag_context if request.ragEnabled else None + if not request.ragEnabled: + _clear_request_rag_bindings(request) + snapshot_identity = validate_review_snapshot_identity(request) validate_acquired_diff_manifest( request.changedFiles or (), @@ -1052,10 +1118,10 @@ async def orchestrate_review( ) _emit_progress(self.event_callback, 10, stage_0_message) - if not inference_profile.fast_check_enabled and self.rag_client: + if not inference_profile.fast_check_enabled and request_rag_client: stage_2_context_task = asyncio.create_task( prefetch_stage_2_cross_module_context( - self.rag_client, + request_rag_client, request, processed_diff=processed_diff, visible_evidence_by_id=stage_2_visible_evidence_by_id, @@ -1072,8 +1138,8 @@ async def orchestrate_review( with_stage_output_cap(self.llm, "stage_1", inference_profile), request, review_plan, - self.rag_client, - rag_context, + request_rag_client, + request_rag_context, processed_diff, is_incremental, self.max_parallel_stage_1, @@ -1169,7 +1235,7 @@ async def orchestrate_review( file_issues, review_plan, processed_diff=processed_diff, - rag_client=self.rag_client, + rag_client=request_rag_client, fallback_llm=self.llm, prefetched_cross_module_context=prefetched_cross_module_context, visible_evidence_by_id=stage_2_visible_evidence_by_id, @@ -1489,8 +1555,14 @@ async def orchestrate_review( return response except Exception as e: - logger.error(f"Multi-stage review failed: {e}", exc_info=True) - _emit_error(self.event_callback, str(e)) + # ReviewService owns the single terminal diagnostic and error + # event. Logging/emitting here as well duplicated the same failure + # at both the orchestration and transport boundaries. + logger.debug( + "Multi-stage review failed; propagating to ReviewService: %s", + e, + exc_info=True, + ) raise finally: if stage_2_context_task and not stage_2_context_task.done(): diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_0_planning.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_0_planning.py index b2ee7f1f..a43581e6 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_0_planning.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_0_planning.py @@ -141,7 +141,7 @@ async def execute_stage_0_planning( logger.info("Stage 0 planning completed with structured output") return result except Exception as e: - logger.warning(f"Structured output failed for Stage 0: {e}") + logger.debug("Structured output failed for Stage 0: %s", e) else: logger.info("Structured output skipped for Stage 0; using prompt JSON parsing") @@ -150,7 +150,7 @@ async def execute_stage_0_planning( content = extract_llm_response_text(response) return await parse_llm_response(content, ReviewPlan, llm) except Exception as e: - logger.error(f"Stage 0 planning failed, using local fallback plan: {e}") + logger.info("Stage 0 planning unavailable; using local fallback plan: %s", e) return _build_fallback_review_plan(request, processed_diff) diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py index f293e565..ee7f9623 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py @@ -129,6 +129,8 @@ class Stage1PreparedContext: @dataclass class Stage1RagState: """Per-review RAG state shared across Stage 1 batches.""" + context_disabled: bool = False + context_disable_reason: str = "" semantic_disabled: bool = False semantic_failures: int = 0 semantic_disable_reason: str = "" @@ -1562,16 +1564,72 @@ def _is_exact_revision_bound( request: ReviewRequestDto, pr_indexed: bool, ) -> bool: + pr_number = getattr(request, "pullRequestId", None) return bool( pr_indexed - and (request.currentCommitHash or request.commitHash) - and request.baseCommitHash - and request.ragBaseGenerationManifestSha256 - and request.ragPrGenerationFingerprint - and request.ragPrOverlayGenerationManifestSha256 + and isinstance(pr_number, int) + and pr_number > 0 + and all( + isinstance(value, str) and bool(value.strip()) + for value in ( + getattr(request, "currentCommitHash", None) + or getattr(request, "commitHash", None), + getattr(request, "baseCommitHash", None), + getattr(request, "ragCollectionTarget", None), + getattr(request, "ragBaseGenerationManifestSha256", None), + getattr(request, "ragPrGenerationFingerprint", None), + getattr( + request, + "ragPrOverlayGenerationManifestSha256", + None, + ), + ) + ) + ) + + +def _has_exact_base_binding(request: ReviewRequestDto) -> bool: + return all( + isinstance(value, str) and bool(value.strip()) + for value in ( + getattr(request, "baseCommitHash", None), + getattr(request, "ragCollectionTarget", None), + getattr(request, "ragBaseGenerationManifestSha256", None), + ) ) +def _disable_rag_context( + rag_state: Optional[Stage1RagState], + reason: str, +) -> bool: + """Open the optional-context circuit once and report whether it changed.""" + if rag_state is None: + return True + if rag_state.context_disabled: + return False + rag_state.context_disabled = True + rag_state.context_disable_reason = reason + rag_state.semantic_disabled = True + rag_state.semantic_disable_reason = reason + return True + + +def _disable_semantic_rag( + rag_state: Optional[Stage1RagState], + reason: str, +) -> bool: + """Open the semantic-filler circuit once across concurrent batches.""" + if rag_state is None: + return True + if rag_state.semantic_disabled: + return False + rag_state.semantic_failures += 1 + rag_state.semantic_disabled = True + rag_state.semantic_disable_reason = reason + return True + + async def fetch_batch_rag_context( rag_client, request: ReviewRequestDto, @@ -1586,10 +1644,23 @@ async def fetch_batch_rag_context( rag_state: Optional[Stage1RagState] = None, ) -> Optional[Dict[str, Any]]: exact_revision_bound = _is_exact_revision_bound(request, pr_indexed) + exact_base_bound = _has_exact_base_binding(request) + exact_context_bound = exact_revision_bound or exact_base_bound + if rag_state and rag_state.context_disabled: + logger.debug( + "Per-batch RAG context skipped after an earlier optional-context " + "failure: %s", + rag_state.context_disable_reason, + ) + return None if not rag_client: - if exact_revision_bound: - raise RuntimeError( - "revision-bound Stage 1 retrieval requires a RAG client" + if exact_context_bound and _disable_rag_context( + rag_state, + "revision-bound Stage 1 retrieval has no RAG client", + ): + logger.info( + "Optional revision-bound RAG context is unavailable; " + "continuing with local review evidence" ) return None @@ -1600,13 +1671,16 @@ async def fetch_batch_rag_context( base_branch = request.get_rag_base_branch() if not rag_branch: message = "Missing authoritative target branch for Stage 1 RAG retrieval" - logger.warning(message) _capture_deterministic_retrieval_state( {"status": "error", "error": message}, rag_state, ) - if exact_revision_bound: - raise RuntimeError(message) + if _disable_rag_context(rag_state, message): + (logger.info if exact_context_bound else logger.warning)( + "%s; disabling optional RAG context for the remaining " + "Stage 1 batches", + message, + ) return None # Scale top_k based on batch priority to ensure adequate context @@ -1616,29 +1690,34 @@ async def fetch_batch_rag_context( logger.info(f"Fetching per-batch RAG context for {len(batch_file_paths)} files " f"(priority={priority_upper}, top_k={top_k})") - pr_number = request.pullRequestId if pr_indexed else None - all_pr_files = request.changedFiles if pr_indexed else None + pr_number = request.pullRequestId if exact_revision_bound else None + all_pr_files = request.changedFiles if exact_revision_bound else None source_revision = ( request.currentCommitHash or request.commitHash - if pr_indexed + if exact_revision_bound else None ) - base_revision = request.baseCommitHash if pr_indexed else None + base_revision = ( + request.baseCommitHash if exact_context_bound else None + ) base_generation_receipt = ( request.ragBaseGenerationManifestSha256 - if pr_indexed + if exact_context_bound else None ) pr_generation_fingerprint = ( request.ragPrGenerationFingerprint - if pr_indexed + if exact_revision_bound else None ) pr_overlay_generation_manifest_sha256 = ( request.ragPrOverlayGenerationManifestSha256 - if pr_indexed + if exact_revision_bound else None ) + collection_target = ( + request.ragCollectionTarget if exact_context_bound else None + ) context = None @@ -1647,11 +1726,15 @@ async def _fetch_deterministic_context() -> Optional[Dict[str, Any]]: return await rag_client.get_deterministic_context( workspace=request.projectWorkspace, project=request.projectNamespace, - branches=list(dict.fromkeys( - branch - for branch in (rag_branch, base_branch) - if branch - )), + branches=( + [rag_branch] + if exact_context_bound + else list(dict.fromkeys( + branch + for branch in (rag_branch, base_branch) + if branch + )) + ), file_paths=batch_file_paths, limit_per_file=5, pr_number=pr_number, @@ -1666,11 +1749,13 @@ async def _fetch_deterministic_context() -> Optional[Dict[str, Any]]: pr_overlay_generation_manifest_sha256=( pr_overlay_generation_manifest_sha256 ), - collection_target=request.ragCollectionTarget, + collection_target=collection_target, ) except Exception as det_err: - logger.warning("Deterministic RAG lookup failed: %s", det_err) - return {"status": "error", "error": str(det_err)} + return { + "status": "error", + "error": f"{type(det_err).__name__}: {det_err}", + } async def _fetch_semantic_context() -> Optional[Dict[str, Any]]: if not SEMANTIC_RAG_FILLER_ENABLED: @@ -1691,10 +1776,10 @@ async def _fetch_semantic_context() -> Optional[Dict[str, Any]]: pr_title=request.prTitle, pr_description=request.prDescription, top_k=semantic_top_k, - base_branch=base_branch, + base_branch=(base_branch if pr_number else None), pr_number=pr_number, all_pr_changed_files=all_pr_files, - deleted_files=request.deletedFiles or None, + deleted_files=(request.deletedFiles or None) if pr_number else None, source_revision=source_revision, base_revision=base_revision, base_generation_manifest_sha256=base_generation_receipt, @@ -1702,7 +1787,7 @@ async def _fetch_semantic_context() -> Optional[Dict[str, Any]]: pr_overlay_generation_manifest_sha256=( pr_overlay_generation_manifest_sha256 ), - collection_target=request.ragCollectionTarget, + collection_target=collection_target, ) async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: @@ -1743,12 +1828,10 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: repository_generation_manifest_sha256=( base_generation_receipt ), - collection_target=request.ragCollectionTarget, + collection_target=collection_target, ) except Exception as dup_err: - if base_revision or base_generation_receipt: - raise - logger.debug(f"Duplication search skipped: {dup_err}") + logger.debug("Duplication search skipped: %s", dup_err) return None deterministic_task = asyncio.create_task(_fetch_deterministic_context()) @@ -1757,32 +1840,37 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: # 1. Deterministic lookup FIRST — structural deps are highest-value context deterministic_response = await deterministic_task deterministic_error = _rag_response_error(deterministic_response) - if deterministic_error: - logger.warning( - "Deterministic RAG lookup returned a failed state: %s", - deterministic_error, - ) deterministic_chunks = _flatten_deterministic_context(deterministic_response) _capture_deterministic_retrieval_state( deterministic_response, rag_state, ) - if deterministic_error and exact_revision_bound: - raise RuntimeError( - "revision-bound deterministic RAG retrieval failed: " - f"{deterministic_error}" - ) deterministic_retrieval_state = _deterministic_retrieval_state( deterministic_response ) - if ( - exact_revision_bound - and deterministic_retrieval_state != "complete" - ): - raise RuntimeError( - "revision-bound deterministic RAG retrieval is not complete: " - f"{deterministic_retrieval_state}" + context_error = ( + deterministic_error + or ( + f"deterministic retrieval state is {deterministic_retrieval_state}" + if exact_context_bound + and deterministic_retrieval_state != "complete" + else None ) + ) + if context_error: + if duplication_task is not None and not duplication_task.done(): + duplication_task.cancel() + if duplication_task is not None: + await asyncio.gather(duplication_task, return_exceptions=True) + if _disable_rag_context(rag_state, context_error): + (logger.info if exact_context_bound else logger.warning)( + "Optional %sRAG context is unavailable; disabling it for " + "the remaining Stage 1 batches and continuing with local " + "review evidence: %s", + "revision-bound " if exact_context_bound else "", + context_error, + ) + return None if deterministic_chunks: context = {"relevant_code": deterministic_chunks} logger.info( @@ -1805,11 +1893,6 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: ) elif semantic_fill_enabled and rag_state and rag_state.semantic_disabled: logger.info("Semantic RAG filler skipped: %s", rag_state.semantic_disable_reason) - if exact_revision_bound: - raise RuntimeError( - "revision-bound semantic RAG retrieval is disabled after " - f"a prior failure: {rag_state.semantic_disable_reason}" - ) elif semantic_fill_enabled: try: rag_response = await asyncio.wait_for( @@ -1821,38 +1904,21 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: raise RuntimeError(semantic_error) except asyncio.TimeoutError: rag_response = None - if rag_state: - rag_state.semantic_failures += 1 - rag_state.semantic_disabled = True - rag_state.semantic_disable_reason = ( - f"timed out after {SEMANTIC_RAG_TIMEOUT_SECONDS}s" - ) - logger.warning( - "Semantic RAG filler timed out after %ss; disabling for remaining Stage 1 batches", - SEMANTIC_RAG_TIMEOUT_SECONDS, - ) - if exact_revision_bound: - raise RuntimeError( - "revision-bound semantic RAG retrieval timed out" + reason = f"timed out after {SEMANTIC_RAG_TIMEOUT_SECONDS}s" + if _disable_semantic_rag(rag_state, reason): + logger.warning( + "Semantic RAG filler timed out after %ss; disabling " + "for remaining Stage 1 batches", + SEMANTIC_RAG_TIMEOUT_SECONDS, ) except Exception as sem_err: rag_response = None - if rag_state: - rag_state.semantic_failures += 1 - rag_state.semantic_disabled = True - rag_state.semantic_disable_reason = str(sem_err) - logger.warning("Semantic RAG filler failed; disabling for remaining Stage 1 batches: %s", sem_err) - if exact_revision_bound: - raise - - if ( - semantic_fill_enabled - and exact_revision_bound - and rag_response is None - ): - raise RuntimeError( - "revision-bound semantic RAG retrieval returned no response" - ) + if _disable_semantic_rag(rag_state, str(sem_err)): + logger.warning( + "Semantic RAG filler failed; disabling for remaining " + "Stage 1 batches: %s", + sem_err, + ) if semantic_fill > 0 and rag_response: sem_context = _unwrap_rag_context(rag_response) @@ -1945,7 +2011,7 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: f"{rerank_result.original_count}→{rerank_result.reranked_count} chunks)" ) except Exception as rerank_err: - logger.warning(f"Per-batch reranking failed (non-critical): {rerank_err}") + logger.info(f"Per-batch reranking skipped (non-critical): {rerank_err}") return context @@ -1956,9 +2022,12 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: duplication_task.cancel() if duplication_task is not None: await asyncio.gather(duplication_task, return_exceptions=True) - logger.warning(f"Failed to fetch per-batch RAG context: {e}") - if exact_revision_bound: - raise + if _disable_rag_context(rag_state, str(e)): + logger.warning( + "Failed to fetch optional per-batch RAG context; disabling it " + "for the remaining Stage 1 batches: %s", + e, + ) return None @@ -2204,7 +2273,7 @@ async def _run_batch( else: logger.info(f"Batch {batch_num} completed: no issues found") except Exception as exc: - logger.error(f"Error reviewing Stage 1 batch: {exc}") + logger.debug("Stage 1 batch failed; cancelling sibling batches: %s", exc) for task in tasks: if not task.done(): task.cancel() @@ -2268,7 +2337,7 @@ async def _review_batch_with_timing( return result except Exception as e: elapsed = time.time() - start_time - logger.error(f"[Batch {batch_idx}] FAILED after {elapsed:.2f}s: {e}") + logger.debug(f"[Batch {batch_idx}] FAILED after {elapsed:.2f}s: {e}") raise @@ -2401,7 +2470,11 @@ async def review_file_batch( str, tuple[Dict[str, Any], ...] ] = {} - if rag_client or _is_exact_revision_bound(request, pr_indexed): + exact_context_bound = ( + _is_exact_revision_bound(request, pr_indexed) + or _has_exact_base_binding(request) + ) + if rag_client or exact_context_bound: batch_rag_context = await fetch_batch_rag_context( rag_client, request, batch_file_paths, batch_diff_snippets, pr_indexed, llm_reranker=llm_reranker, @@ -2423,15 +2496,25 @@ async def review_file_batch( visible_evidence_by_id=batch_visible_evidence_by_id, ) else: - resolved_fallback_rag_context = await _resolve_fallback_rag_context( - fallback_rag_context + fallback_context_allowed = not ( + exact_context_bound + or (rag_state is not None and rag_state.context_disabled) ) - if resolved_fallback_rag_context: - scoped_fallback_rag_context = _scope_fallback_rag_context_to_batch( - resolved_fallback_rag_context, - batch_file_paths, + if fallback_context_allowed: + resolved_fallback_rag_context = await _resolve_fallback_rag_context( + fallback_rag_context ) + if resolved_fallback_rag_context: + scoped_fallback_rag_context = _scope_fallback_rag_context_to_batch( + resolved_fallback_rag_context, + batch_file_paths, + ) + else: + scoped_fallback_rag_context = None else: + # Global fallback is alias/branch scoped and cannot satisfy an + # immutable generation receipt. It must not be awaited after an + # exact-context failure, because doing so could inject stale code. scoped_fallback_rag_context = None if scoped_fallback_rag_context: @@ -2545,7 +2628,7 @@ async def review_file_batch( return issues if fallback_llm is not None and fallback_llm is not llm: - logger.warning( + logger.info( "Stage 1 batch failed with capped LLM for %s; retrying without output cap", batch_file_paths, ) @@ -2565,7 +2648,7 @@ async def review_file_batch( ) return issues - logger.error( + logger.debug( "Batch review parse failure for %s after capped%s attempts. " "The batch will fail so missing results cannot be published as a clean review.", batch_file_paths, @@ -2689,9 +2772,9 @@ async def _invoke_stage_1_batch_llm( result = await structured_llm.ainvoke(prompt) if result: return _extract_calibrated_issues(result) - logger.warning("Structured output returned empty Stage 1 result for %s (%s)", batch_file_paths, label) + logger.debug("Structured output returned empty Stage 1 result for %s (%s)", batch_file_paths, label) except Exception as e: - logger.warning("Structured output failed for Stage 1 batch %s (%s): %s", batch_file_paths, label, e) + logger.debug("Structured output failed for Stage 1 batch %s (%s): %s", batch_file_paths, label, e) else: logger.info( "Structured output skipped for Stage 1 batch %s (%s); using prompt JSON parsing", @@ -2705,7 +2788,7 @@ async def _invoke_stage_1_batch_llm( data = await parse_llm_response(content, FileReviewBatchOutput, llm) return _extract_calibrated_issues(data) except Exception as parse_err: - logger.warning("Stage 1 batch parse failed for %s (%s): %s", batch_file_paths, label, parse_err) + logger.debug("Stage 1 batch parse failed for %s (%s): %s", batch_file_paths, label, parse_err) return None diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py index 880cb3e7..31e34ce2 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py @@ -136,7 +136,7 @@ async def execute_stage_2_cross_file( return result if fallback_llm is not None and fallback_llm is not llm: - logger.warning("Stage 2 failed with capped LLM; retrying without output cap") + logger.info("Stage 2 failed with capped LLM; retrying without output cap") result = await _invoke_stage_2_llm(fallback_llm, prompt, label="uncapped retry") if result is not None: return result @@ -168,9 +168,9 @@ async def _invoke_stage_2_llm(llm, prompt: str, label: str) -> Optional[CrossFil if result: logger.info("Stage 2 cross-file analysis completed with structured output (%s)", label) return result - logger.warning("Structured output returned empty Stage 2 result (%s)", label) + logger.debug("Structured output returned empty Stage 2 result (%s)", label) except Exception as e: - logger.warning("Structured output failed for Stage 2 (%s): %s", label, e) + logger.debug("Structured output failed for Stage 2 (%s): %s", label, e) else: logger.info("Structured output skipped for Stage 2 (%s); using prompt JSON parsing", label) @@ -179,7 +179,7 @@ async def _invoke_stage_2_llm(llm, prompt: str, label: str) -> Optional[CrossFil content = extract_llm_response_text(response) return await parse_llm_response(content, CrossFileAnalysisResult, llm) except Exception as e: - logger.warning("Stage 2 cross-file analysis failed (%s): %s", label, e) + logger.debug("Stage 2 cross-file analysis failed (%s): %s", label, e) return None @@ -711,7 +711,7 @@ async def _fetch_cross_module_context( ) return "" if not base_revision or not base_generation_receipt: - logger.warning( + logger.info( "Stage 2 cross-module RAG requires both immutable target revision " "and generation receipt" ) @@ -721,7 +721,7 @@ async def _fetch_cross_module_context( rag_branch = request.get_rag_branch() base_branch = request.get_rag_base_branch() if not rag_branch: - logger.warning( + logger.info( "Stage 2 cross-module RAG skipped: missing authoritative target branch" ) return "" @@ -788,7 +788,7 @@ async def _fetch_cross_module_context( return formatted except Exception as e: - logger.warning( + logger.info( "Revision-bound cross-module context unavailable for Stage 2: %s: %s", type(e).__name__, e, diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py index cdb5576b..b4a685d7 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py @@ -106,7 +106,7 @@ def _review_revision(request: ReviewRequestDto) -> str: async def _invoke_stage_3_report(llm, prompt: str, fallback_llm=None) -> Dict[str, Any]: response = await llm.ainvoke(prompt) if _response_finished_by_length(response) and fallback_llm is not None and fallback_llm is not llm: - logger.warning("Stage 3 report hit output cap; retrying without output cap") + logger.info("Stage 3 report hit output cap; retrying without output cap") response = await fallback_llm.ainvoke(prompt) return {"report": extract_llm_response_text(response), "dismissed_issue_ids": []} @@ -449,7 +449,7 @@ async def _stage_3_with_mcp( tool_calls = getattr(response, 'tool_calls', None) if not tool_calls: if _response_finished_by_length(response) and fallback_llm is not None and fallback_llm is not llm: - logger.warning("MCP Stage 3 report hit output cap; retrying without output cap") + logger.info("MCP Stage 3 report hit output cap; retrying without output cap") return await _stage_3_with_mcp( fallback_llm, request, @@ -492,8 +492,8 @@ async def _stage_3_with_mcp( }) except Exception as e: - logger.warning(f"[MCP Stage 3] Iteration {iteration + 1} failed: {e}") + logger.info(f"[MCP Stage 3] Iteration {iteration + 1} failed: {e}") break - logger.warning("[MCP Stage 3] Agentic loop exhausted, falling back to plain call") + logger.info("[MCP Stage 3] Agentic loop exhausted, falling back to plain call") return await _invoke_stage_3_report(llm, prompt, fallback_llm=fallback_llm) diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/verification_agent.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/verification_agent.py index 7593e5cb..69ee559a 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/verification_agent.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/verification_agent.py @@ -1383,7 +1383,8 @@ async def run_verification_agent( result = await _run_verification_tool_loop(llm, prompt) except Exception as exception: failed_batches += 1 - logger.error( + log = logger.info if failed_batches == 1 else logger.debug + log( "Stage 1.5 verification batch %d/%d failed; retaining its " "%d issue(s): %s", batch_index, @@ -1438,7 +1439,7 @@ async def run_verification_agent( ] except Exception as e: - logger.error(f"Stage 1.5 Verification failed: {e}") + logger.info("Stage 1.5 verification unavailable; retaining all issues: %s", e) # Fallback: keep all issues if verification fails final_issues = issues finally: diff --git a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py index 6e384fe6..f7af1dbc 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py @@ -655,6 +655,28 @@ async def _fetch_rag_context( timeout=self.GLOBAL_RAG_QUERY_TIMEOUT_SECONDS, ) + if ( + isinstance(rag_response, dict) + and rag_response.get("status") == "error" + ): + logger.info( + "Global fallback RAG context unavailable; per-batch " + "retrieval remains authoritative: status=%s detail=%s", + rag_response.get("status_code") or "transport-error", + rag_response.get("error") + or rag_response.get("detail") + or "unknown failure", + ) + self._emit_event(event_callback, { + "type": "status", + "state": "rag_skipped", + "message": ( + "Global RAG fallback unavailable; per-batch retrieval " + "remains active" + ), + }) + return None + if rag_response and rag_response.get("context"): context = rag_response.get("context") relevant_code = context.get("relevant_code", []) diff --git a/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py b/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py index 82352e3f..83b4c312 100644 --- a/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py +++ b/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py @@ -115,6 +115,8 @@ def neutral_request( pullRequestId=42, currentCommitHash=HEAD_REVISION, baseCommitHash=BASE_REVISION, + ragCollectionTarget="cc_workspace_project_main_generation", + ragBaseGenerationManifestSha256=BASE_GENERATION_MANIFEST, changedFiles=paths, rawDiff="\n".join(diffs) + "\n", enrichmentData=PrEnrichmentDataDto(fileContents=contents), diff --git a/python-ecosystem/inference-orchestrator/tests/test_command_queue_consumer.py b/python-ecosystem/inference-orchestrator/tests/test_command_queue_consumer.py index c0c151df..d58079a3 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_command_queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/tests/test_command_queue_consumer.py @@ -8,12 +8,34 @@ from server.command_queue_consumer import CommandQueueConsumer +class FakePipeline: + def __init__(self, redis): + self.redis = redis + self.pending = [] + + def lpush(self, key, value): + self.pending.append(("lpush", key, value)) + return self + + def expire(self, key, ttl): + self.pending.append(("expire", key, ttl)) + return self + + async def execute(self): + for operation, key, value in self.pending: + if operation == "lpush": + self.redis.events.append((key, json.loads(value))) + else: + self.redis.expiries.append((key, value)) + + class FakeRedis: def __init__(self): self.events = [] + self.expiries = [] - async def lpush(self, key, value): - self.events.append((key, json.loads(value))) + def pipeline(self): + return FakePipeline(self) def _ask_request(): @@ -61,20 +83,32 @@ def _consumer(command_service): async def _handle_and_collect_events(consumer, payload): await consumer._handle_job(payload) - await asyncio.sleep(0) - await asyncio.sleep(0) return [event for _, event in consumer._redis.events] @pytest.mark.asyncio(loop_scope="function") async def test_error_result_is_published_as_error_without_final(): command_service = MagicMock() - command_service.process_ask = AsyncMock(return_value={"error": "provider failed"}) + + async def process(_request, callback): + callback({"type": "error", "message": "provider failed"}) + callback({"type": "status", "state": "late_diagnostic"}) + return {"error": "provider failed"} + + command_service.process_ask = AsyncMock(side_effect=process) consumer = _consumer(command_service) events = await _handle_and_collect_events(consumer, _payload("ask", _ask_request())) - assert any(event["type"] == "error" and event["message"] == "provider failed" for event in events) + assert events == [ + { + "type": "status", + "state": "acknowledged", + "message": "Orchestrator picked up ask command from queue", + }, + {"type": "error", "message": "provider failed"}, + ] + assert sum(event["type"] in {"error", "final"} for event in events) == 1 assert not any(event["type"] == "final" for event in events) @@ -105,6 +139,52 @@ async def test_successful_ask_answer_is_published_as_final(): assert not any(event["type"] == "error" for event in events) +@pytest.mark.asyncio(loop_scope="function") +async def test_progress_and_final_events_are_ordered_before_job_returns(): + command_service = MagicMock() + + async def process(_request, callback): + callback({"type": "status", "state": "answering"}) + return {"answer": "42"} + + command_service.process_ask = AsyncMock(side_effect=process) + consumer = _consumer(command_service) + + events = await _handle_and_collect_events( + consumer, + _payload("ask", _ask_request()), + ) + + assert [event.get("state") for event in events[:-1]] == [ + "acknowledged", + "answering", + ] + assert events[-1] == {"type": "final", "result": {"answer": "42"}} + assert consumer._redis.expiries == [ + ("codecrow:analysis:events:job-ask", 3600), + ("codecrow:analysis:events:job-ask", 3600), + ("codecrow:analysis:events:job-ask", 3600), + ] + + +@pytest.mark.asyncio(loop_scope="function") +async def test_command_event_publish_uses_configured_expiry(monkeypatch): + monkeypatch.setenv("COMMAND_EVENT_TTL_SECONDS", "123") + consumer = _consumer(MagicMock()) + + await consumer._publish_event( + "codecrow:analysis:events:job-ttl", + {"type": "status"}, + ) + + assert consumer._redis.events == [ + ("codecrow:analysis:events:job-ttl", {"type": "status"}), + ] + assert consumer._redis.expiries == [ + ("codecrow:analysis:events:job-ttl", 123), + ] + + @pytest.mark.asyncio(loop_scope="function") async def test_empty_summarize_result_is_published_as_error_without_final(): command_service = MagicMock() @@ -125,6 +205,7 @@ async def test_start_uses_blocking_read_safe_redis_timeouts(): command_service = MagicMock() redis_client = MagicMock() redis_client.aclose = AsyncMock() + redis_client.set = AsyncMock() consumer = CommandQueueConsumer(command_service) consumer._consume_loop = AsyncMock() @@ -142,6 +223,26 @@ async def test_start_uses_blocking_read_safe_redis_timeouts(): "socket_timeout": 30, "health_check_interval": 30, } + redis_client.set.assert_awaited_with( + "codecrow:commands:consumer:heartbeat", + "alive", + ex=consumer.consumer_heartbeat_ttl_seconds, + ) + + +@pytest.mark.asyncio(loop_scope="function") +async def test_command_consumer_heartbeat_uses_the_java_supervision_contract(): + consumer = CommandQueueConsumer(MagicMock()) + consumer._redis = MagicMock() + consumer._redis.set = AsyncMock() + + await consumer._publish_consumer_heartbeat() + + consumer._redis.set.assert_awaited_once_with( + "codecrow:commands:consumer:heartbeat", + "alive", + ex=consumer.consumer_heartbeat_ttl_seconds, + ) @pytest.mark.asyncio(loop_scope="function") @@ -166,3 +267,85 @@ async def stop_after_backoff(_seconds): await consumer._consume_loop() warning.assert_called_once() + + +@pytest.mark.asyncio(loop_scope="function") +async def test_worker_capacity_is_reserved_before_command_is_dequeued(): + consumer = CommandQueueConsumer(MagicMock()) + consumer._job_semaphore = asyncio.Semaphore(1) + consumer._redis = MagicMock() + + async def stop_after_dequeue(*_args, **_kwargs): + consumer.is_running = False + return None + + consumer._redis.brpop = AsyncMock(side_effect=stop_after_dequeue) + consumer.is_running = True + await consumer._job_semaphore.acquire() + + consume_task = asyncio.create_task(consumer._consume_loop()) + await asyncio.sleep(0) + await asyncio.sleep(0) + consumer._redis.brpop.assert_not_awaited() + + consumer._job_semaphore.release() + await consume_task + + consumer._redis.brpop.assert_awaited_once_with( + [consumer.job_queue_key], + timeout=1, + ) + assert not consumer._job_semaphore.locked() + + +@pytest.mark.asyncio(loop_scope="function") +async def test_stop_waits_for_admitted_command_before_closing_redis(): + consumer = CommandQueueConsumer(MagicMock()) + consumer._redis = MagicMock() + consumer._redis.aclose = AsyncMock() + started = asyncio.Event() + release = asyncio.Event() + + async def handle_until_released(_payload): + started.set() + await release.wait() + + consumer._handle_job = AsyncMock(side_effect=handle_until_released) + await consumer._job_semaphore.acquire() + job_task = asyncio.create_task(consumer._handle_admitted_job("payload")) + consumer._job_tasks.add(job_task) + job_task.add_done_callback(consumer._job_tasks.discard) + await started.wait() + + stop_task = asyncio.create_task(consumer.stop()) + await asyncio.sleep(0) + + assert not stop_task.done() + consumer._redis.aclose.assert_not_awaited() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + + assert job_task.done() + consumer._redis.aclose.assert_awaited_once_with() + + +def test_redis_outage_diagnostic_is_bounded_until_recovery(): + consumer = CommandQueueConsumer(MagicMock()) + + with ( + patch("server.command_queue_consumer.logger.warning") as warning, + patch("server.command_queue_consumer.logger.debug") as debug, + patch("server.command_queue_consumer.logger.info") as info, + ): + consumer._record_redis_failure("command event publication", RuntimeError("down")) + consumer._record_redis_failure("command event publication", RuntimeError("still down")) + consumer._record_redis_success("command queue read") + consumer._record_redis_success("command event publication") + + warning.assert_called_once() + debug.assert_called_once() + info.assert_called_once_with( + "Redis connectivity restored during %s", + "command event publication", + ) diff --git a/python-ecosystem/inference-orchestrator/tests/test_command_service.py b/python-ecosystem/inference-orchestrator/tests/test_command_service.py index aa8ed4d5..f28aa2b8 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_command_service.py +++ b/python-ecosystem/inference-orchestrator/tests/test_command_service.py @@ -7,6 +7,7 @@ """ import pytest import json +import logging from unittest.mock import AsyncMock, MagicMock, patch from service.command.command_service import CommandService @@ -77,6 +78,42 @@ def test_returns_dict(self, service): assert isinstance(result, dict) +@pytest.mark.asyncio(loop_scope="function") +async def test_summarize_rag_failure_has_one_owner_diagnostic_and_event( + service, + caplog, +): + service.rag_client.get_pr_context = AsyncMock(return_value={ + "status": "error", + "status_code": 503, + "error": "RAG service unavailable", + }) + request = MagicMock( + projectWorkspace="ws", + projectNamespace="project", + pullRequestId=42, + ) + request.get_rag_branch.return_value = "main" + request.get_rag_base_branch.return_value = None + events = [] + + with caplog.at_level( + logging.WARNING, + logger="service.command.command_service", + ): + result = await service._fetch_rag_context_for_summarize( + request, + events.append, + ) + + assert result is None + assert sum( + "Optional RAG context unavailable for summarize" in record.getMessage() + for record in caplog.records + ) == 1 + assert any(event.get("state") == "rag_skipped" for event in events) + + # ── _build_platform_jvm_props ──────────────────────────────────── class TestBuildPlatformJvmProps: diff --git a/python-ecosystem/inference-orchestrator/tests/test_llm_factory.py b/python-ecosystem/inference-orchestrator/tests/test_llm_factory.py index d12c15e4..19618748 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_llm_factory.py +++ b/python-ecosystem/inference-orchestrator/tests/test_llm_factory.py @@ -3,12 +3,12 @@ Covers: LLMFactory._normalize_provider, get_supported_providers, _check_unsupported_gemini_model, create_llm (all providers), - QaDocumentationService._create_llm, _create_rag_client + QaDocumentationService._create_llm and QA orchestration wiring """ import asyncio import pytest -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from langchain_google_genai import ChatGoogleGenerativeAI @@ -488,19 +488,35 @@ def test_create_llm(self): "QA_DOC_AI_API_KEY": "test", "RAG_PIPELINE_URL": "http://rag:8020", }) - def test_create_rag_client(self): + def test_does_not_configure_rag_mutation_client(self): svc = QaDocumentationService() - client = svc._create_rag_client() - # Should not be None when URL is configured - assert client is not None - - @patch.dict("os.environ", { - "QA_DOC_AI_PROVIDER": "openai", - "QA_DOC_AI_MODEL": "gpt-4o", - "QA_DOC_AI_API_KEY": "test", - "RAG_PIPELINE_URL": "", - }) - def test_create_rag_client_no_url(self): + assert not hasattr(svc, "_rag_pipeline_url") + assert not hasattr(svc, "_create_rag_client") + + @pytest.mark.asyncio(loop_scope="function") + @patch("service.qa_documentation.qa_doc_service.QaDocOrchestrator") + async def test_generate_never_wires_a_rag_mutation_client( + self, + orchestrator_type, + ): svc = QaDocumentationService() - client = svc._create_rag_client() - assert client is None + llm = MagicMock() + svc._create_llm = MagicMock(return_value=llm) + orchestrator_type.return_value.run = AsyncMock(return_value={ + "documentation_needed": False, + "documentation": None, + }) + + await svc.generate( + project_id=1, + project_name="project", + pr_number=17, + issues_found=0, + files_analyzed=1, + pr_metadata={}, + template_mode="BASE", + custom_template=None, + task_context=None, + ) + + orchestrator_type.assert_called_once_with(llm=llm) diff --git a/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py index 5e0bf4e7..353af6a0 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py @@ -5,8 +5,10 @@ _deduplicate_previous_issues, _ensure_all_files_planned, _count_files, _convert_cross_file_issues """ +import logging +from unittest.mock import AsyncMock, MagicMock + import pytest -from unittest.mock import MagicMock from service.review.orchestrator.orchestrator import ( MultiStageReviewOrchestrator, @@ -65,6 +67,43 @@ def test_task_evidence_key_falls_back_to_server_built_history(): assert _task_evidence_key(request) == "SHOP-42" +@pytest.mark.asyncio +async def test_pr_cleanup_does_not_report_success_when_delete_returns_false( + caplog, +): + rag_client = MagicMock() + rag_client.delete_pr_files = AsyncMock(return_value=False) + orchestrator = MultiStageReviewOrchestrator( + llm=MagicMock(), + mcp_client=None, + rag_client=rag_client, + ) + orchestrator._pr_number = 42 + orchestrator._pr_indexed = True + request = MagicMock( + projectWorkspace="workspace", + projectNamespace="project", + ragCollectionTarget="cc_workspace_project_main_generation", + ) + + with caplog.at_level( + logging.INFO, + logger="service.review.orchestrator.orchestrator", + ): + await orchestrator._cleanup_pr_files(request) + + rag_client.delete_pr_files.assert_awaited_once_with( + workspace="workspace", + project="project", + pr_number=42, + collection_target="cc_workspace_project_main_generation", + ) + assert "Cleaned up PR #42 indexed data" not in caplog.text + assert "PR #42 indexed-data cleanup did not complete" in caplog.text + assert orchestrator._pr_number is None + assert orchestrator._pr_indexed is False + + # ── _filter_diff_for_files ────────────────────────────────────── class TestFilterDiffForFiles: diff --git a/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py b/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py index b834b212..42c91f30 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py +++ b/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py @@ -992,6 +992,69 @@ async def test_project_disabled_rag_skips_pr_overlay(): assert rag.index_requests == [] +@pytest.mark.asyncio +async def test_project_disabled_rag_clears_bindings_and_never_queries_client(): + request = _request().model_copy(update={ + "ragEnabled": False, + "ragPrGenerationFingerprint": "sha256:" + "a" * 64, + "ragPrOverlayGenerationManifestSha256": "b" * 64, + "ragBasePluginFingerprint": "sha256:" + "c" * 64, + "ragBasePluginDescriptorFingerprint": "sha256:" + "d" * 64, + "ragBasePluginImplementationFingerprint": "sha256:" + "e" * 64, + "ragBaseIndexRepresentationFingerprint": "sha256:" + "f" * 64, + # Force Stage 2 even for the small neutral fixture. + "taskContext": {"task_key": "SHOP-42"}, + }) + rag = DeterministicRagSpy() + rag.index_pr_files = AsyncMock( + side_effect=AssertionError("disabled RAG must not index") + ) + rag.get_deterministic_context = AsyncMock( + side_effect=AssertionError("disabled RAG must not retrieve") + ) + rag.get_pr_context = AsyncMock( + side_effect=AssertionError("disabled RAG must not retrieve") + ) + rag.search_for_duplicates = AsyncMock( + side_effect=AssertionError("disabled RAG must not query duplicates") + ) + session = PromptCaptureSession(request=request) + orchestrator = MultiStageReviewOrchestrator( + llm=PromptCaptureLLM(session), + mcp_client=None, + rag_client=rag, + ) + + result = await orchestrator.orchestrate_review( + request, + rag_context={"relevant_code": ["DISABLED_GLOBAL_RAG_SENTINEL"]}, + processed_diff=DiffProcessor().process(request.rawDiff), + ) + + assert result["issues"] == [] + rag.index_pr_files.assert_not_awaited() + rag.get_deterministic_context.assert_not_awaited() + rag.get_pr_context.assert_not_awaited() + rag.search_for_duplicates.assert_not_awaited() + assert all( + getattr(request, field_name) is None + for field_name in ( + "ragCollectionTarget", + "ragBaseGenerationManifestSha256", + "ragPrGenerationFingerprint", + "ragPrOverlayGenerationManifestSha256", + "ragBasePluginFingerprint", + "ragBasePluginDescriptorFingerprint", + "ragBasePluginImplementationFingerprint", + "ragBaseIndexRepresentationFingerprint", + ) + ) + assert all( + "DISABLED_GLOBAL_RAG_SENTINEL" not in prompt["renderedPrompt"] + for prompt in session.prompts + ) + + @pytest.mark.asyncio async def test_pr_overlay_receives_one_exact_snapshot_identity(): rag = DeterministicRagSpy() @@ -1044,6 +1107,42 @@ async def reject_overlay(**kwargs): assert rag.index_requests assert orchestrator._pr_indexed is False + assert request.ragCollectionTarget == "cc_workspace_project_main_generation" + assert request.ragBaseGenerationManifestSha256 == "3" * 64 + assert request.ragPrGenerationFingerprint is None + assert request.ragPrOverlayGenerationManifestSha256 is None + + +@pytest.mark.asyncio +async def test_incomplete_pr_overlay_receipt_degrades_to_exact_base_binding(): + rag = DeterministicRagSpy() + complete_index = rag.index_pr_files + + async def omit_overlay_manifest(**kwargs): + result = await complete_index(**kwargs) + result.pop("overlay_generation_manifest_sha256") + result["review_groups"] = [["src/file_0.py"]] + return result + + rag.index_pr_files = omit_overlay_manifest + request = _request() + orchestrator = MultiStageReviewOrchestrator( + llm=object(), + mcp_client=None, + rag_client=rag, + ) + + await orchestrator._index_pr_files( + request, + DiffProcessor().process(request.rawDiff), + ) + + assert orchestrator._pr_indexed is False + assert orchestrator._repository_review_groups == () + assert request.ragCollectionTarget == "cc_workspace_project_main_generation" + assert request.ragBaseGenerationManifestSha256 == "3" * 64 + assert request.ragPrGenerationFingerprint is None + assert request.ragPrOverlayGenerationManifestSha256 is None @pytest.mark.asyncio diff --git a/python-ecosystem/inference-orchestrator/tests/test_qa_documentation.py b/python-ecosystem/inference-orchestrator/tests/test_qa_documentation.py index 553d6d6d..19a454e8 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_qa_documentation.py +++ b/python-ecosystem/inference-orchestrator/tests/test_qa_documentation.py @@ -7,10 +7,11 @@ BaseOrchestrator._simple_batch, filter_diff_for_files, get_file_content_from_enrichment, build_enrichment_lookup """ -import pytest import json from unittest.mock import AsyncMock, MagicMock +import pytest + from service.qa_documentation.qa_doc_orchestrator import QaDocOrchestrator from service.qa_documentation.base_orchestrator import ( BaseOrchestrator, @@ -86,6 +87,13 @@ def test_items_have_file_info_and_priority(self): assert item["file_info"].path == "a.py" assert item["priority"] == "MEDIUM" + def test_qa_orchestrator_has_no_rag_mutation_lifecycle(self): + orchestrator = QaDocOrchestrator(llm=MagicMock()) + + assert not hasattr(orchestrator, "rag_client") + assert not hasattr(orchestrator, "index_pr_files") + assert not hasattr(orchestrator, "cleanup_pr_files") + # ── BaseOrchestrator.filter_diff_for_files ─────────────────────── diff --git a/python-ecosystem/inference-orchestrator/tests/test_queue_consumer.py b/python-ecosystem/inference-orchestrator/tests/test_queue_consumer.py index 027e737f..746dd613 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/tests/test_queue_consumer.py @@ -112,6 +112,39 @@ async def process(_request, _callback): assert events[-1]["type"] == "final" +@pytest.mark.asyncio(loop_scope="function") +async def test_service_error_is_the_only_terminal_review_event(): + review_service = MagicMock() + + async def process(_request, callback): + callback({"type": "error", "message": "provider failed"}) + callback({"type": "status", "state": "late_diagnostic"}) + return { + "result": { + "status": "error", + "message": "provider failed", + }, + } + + review_service.process_review_request = AsyncMock(side_effect=process) + consumer = RedisQueueConsumer(review_service) + consumer._redis = FakeRedis() + + with patch("server.queue_consumer.ReviewRequestDto", return_value=MagicMock()): + await consumer._handle_job(_payload()) + + events = [event for _, event in consumer._redis.events] + assert events == [ + { + "type": "status", + "state": "acknowledged", + "message": "Orchestrator picked up job from queue", + }, + {"type": "error", "message": "provider failed"}, + ] + assert sum(event["type"] in {"error", "final"} for event in events) == 1 + + @pytest.mark.asyncio(loop_scope="function") async def test_neutral_mixed_language_dry_run_traverses_queue_handler( monkeypatch, @@ -291,3 +324,56 @@ async def stop_after_dequeue(*_args, **_kwargs): timeout=1, ) assert not consumer._job_semaphore.locked() + + +@pytest.mark.asyncio(loop_scope="function") +async def test_stop_waits_for_admitted_review_before_closing_redis(): + consumer = RedisQueueConsumer(MagicMock()) + consumer._redis = MagicMock() + consumer._redis.aclose = AsyncMock() + started = asyncio.Event() + release = asyncio.Event() + + async def handle_until_released(_payload): + started.set() + await release.wait() + + consumer._handle_job = AsyncMock(side_effect=handle_until_released) + await consumer._job_semaphore.acquire() + job_task = asyncio.create_task(consumer._handle_admitted_job("payload")) + consumer._job_tasks.add(job_task) + job_task.add_done_callback(consumer._job_tasks.discard) + await started.wait() + + stop_task = asyncio.create_task(consumer.stop()) + await asyncio.sleep(0) + + assert not stop_task.done() + consumer._redis.aclose.assert_not_awaited() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + + assert job_task.done() + consumer._redis.aclose.assert_awaited_once_with() + + +def test_redis_outage_diagnostic_is_bounded_until_recovery(): + consumer = RedisQueueConsumer(MagicMock()) + + with ( + patch("server.queue_consumer.logger.warning") as warning, + patch("server.queue_consumer.logger.debug") as debug, + patch("server.queue_consumer.logger.info") as info, + ): + consumer._record_redis_failure("review event publication", RuntimeError("down")) + consumer._record_redis_failure("review consumer heartbeat", RuntimeError("still down")) + consumer._record_redis_success("review queue read") + consumer._record_redis_success("review consumer heartbeat") + + warning.assert_called_once() + debug.assert_called_once() + info.assert_called_once_with( + "Redis connectivity restored during %s", + "review consumer heartbeat", + ) diff --git a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py index 510cac40..ec264777 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py +++ b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py @@ -1,6 +1,8 @@ """ Unit tests for service.rag.rag_client — RagClient (all async methods). """ +import logging + import pytest import httpx import respx @@ -185,6 +187,19 @@ async def test_delete_pr_files_uses_exact_generation_target(self): ] == "cc_w1_p2_branch_generation" await c.close() + @pytest.mark.asyncio(loop_scope="function") + @respx.mock + async def test_delete_pr_files_treats_missing_collection_as_idempotent(self): + respx.delete("http://rag:8001/index/pr-files/ws/proj/1").mock( + return_value=httpx.Response( + 200, + json={"status": "skipped", "message": "Collection does not exist"}, + ) + ) + c = RagClient(base_url="http://rag:8001", enabled=True) + assert await c.delete_pr_files("ws", "proj", 1) is True + await c.close() + # ── Error handling ─────────────────────────────────────────── @@ -204,7 +219,7 @@ async def test_get_pr_context_http_error(self): @pytest.mark.asyncio(loop_scope="function") @respx.mock - async def test_get_pr_context_preserves_reindex_409_detail(self): + async def test_get_pr_context_preserves_reindex_409_detail(self, caplog): respx.post("http://rag:8001/query/pr-context").mock( return_value=httpx.Response( 409, @@ -214,12 +229,18 @@ async def test_get_pr_context_preserves_reindex_409_detail(self): ) ) c = RagClient(base_url="http://rag:8001", enabled=True) - r = await c.get_pr_context("ws", "proj", "main", ["a.py"]) + with caplog.at_level(logging.DEBUG, logger="service.rag.rag_client"): + r = await c.get_pr_context("ws", "proj", "main", ["a.py"]) assert r == { "status": "error", "status_code": 409, "error": "branch 'main' requires a full reindex", } + assert not any( + record.levelno >= logging.WARNING + for record in caplog.records + if record.name == "service.rag.rag_client" + ) await c.close() @pytest.mark.asyncio(loop_scope="function") @@ -256,7 +277,40 @@ async def test_deterministic_context_error(self): @pytest.mark.asyncio(loop_scope="function") @respx.mock - async def test_index_pr_files_error(self): + async def test_deterministic_context_409_is_returned_without_client_warning( + self, + caplog, + ): + respx.post("http://rag:8001/query/deterministic").mock( + return_value=httpx.Response( + 409, + json={"detail": "requested PR overlay generation is unavailable"}, + ) + ) + c = RagClient(base_url="http://rag:8001", enabled=True) + with caplog.at_level(logging.DEBUG, logger="service.rag.rag_client"): + r = await c.get_deterministic_context( + "ws", + "proj", + ["main"], + ["a.py"], + ) + + assert r == { + "status": "error", + "status_code": 409, + "error": "requested PR overlay generation is unavailable", + } + assert not any( + record.levelno >= logging.WARNING + for record in caplog.records + if record.name == "service.rag.rag_client" + ) + await c.close() + + @pytest.mark.asyncio(loop_scope="function") + @respx.mock + async def test_index_pr_files_error(self, caplog): respx.post("http://rag:8001/index/pr-files").mock( return_value=httpx.Response( 409, @@ -264,20 +318,58 @@ async def test_index_pr_files_error(self): ) ) c = RagClient(base_url="http://rag:8001", enabled=True) - r = await c.index_pr_files("ws", "proj", 1, "main", [{"path": "a.py", "content": "x", "change_type": "M"}]) + with caplog.at_level(logging.DEBUG, logger="service.rag.rag_client"): + r = await c.index_pr_files("ws", "proj", 1, "main", [{"path": "a.py", "content": "x", "change_type": "M"}]) assert r["status"] == "error" assert r["status_code"] == 409 assert r["error"] == "target branch is missing plugin snapshots" + assert not any( + record.levelno >= logging.WARNING + for record in caplog.records + if record.name == "service.rag.rag_client" + ) await c.close() @pytest.mark.asyncio(loop_scope="function") @respx.mock - async def test_delete_pr_files_error(self): + async def test_delete_pr_files_error(self, caplog): respx.delete("http://rag:8001/index/pr-files/ws/proj/1").mock( - return_value=httpx.Response(500) + return_value=httpx.Response( + 500, + json={"detail": "cleanup backend timed out"}, + ) ) c = RagClient(base_url="http://rag:8001", enabled=True) - assert await c.delete_pr_files("ws", "proj", 1) is False + with caplog.at_level(logging.WARNING, logger="service.rag.rag_client"): + assert await c.delete_pr_files("ws", "proj", 1) is False + assert "status=500 detail=cleanup backend timed out" in caplog.text + await c.close() + + @pytest.mark.asyncio(loop_scope="function") + @respx.mock + async def test_delete_pr_failure_logs_only_outage_transitions(self, caplog): + route = respx.delete( + "http://rag:8001/index/pr-files/ws/proj/1" + ).mock(side_effect=[ + httpx.Response(500, json={"detail": "timed out"}), + httpx.Response(500, json={"detail": "timed out"}), + httpx.Response(200, json={"status": "deleted"}), + ]) + c = RagClient(base_url="http://rag:8001", enabled=True) + + with caplog.at_level(logging.DEBUG, logger="service.rag.rag_client"): + assert await c.delete_pr_files("ws", "proj", 1) is False + assert await c.delete_pr_files("ws", "proj", 1) is False + assert await c.delete_pr_files("ws", "proj", 1) is True + + assert route.call_count == 3 + warnings = [ + record for record in caplog.records + if record.levelno == logging.WARNING + and "RAG PR cleanup is degraded" in record.getMessage() + ] + assert len(warnings) == 1 + assert "RAG PR cleanup recovered" in caplog.text await c.close() diff --git a/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py index 405c33c6..ec007212 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py @@ -5,7 +5,7 @@ _create_mcp_client """ import pytest -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from service.review.review_service import ( ReviewService, @@ -166,6 +166,38 @@ def test_non_pr_request_can_use_global_fallback(self): assert allow_unbound_global_rag_fallback(request) is True + @pytest.mark.asyncio(loop_scope="function") + async def test_failed_global_fallback_emits_reduced_context_status( + self, + service, + ): + service.rag_client = MagicMock(enabled=True) + service.rag_client.get_pr_context = AsyncMock(return_value={ + "status": "error", + "status_code": 503, + "error": "RAG service unavailable", + }) + service.rag_cache.get.return_value = None + request = MagicMock( + ragEnabled=True, + projectId=1, + pullRequestId=None, + projectWorkspace="ws", + projectNamespace="project", + changedFiles=["src/a.py"], + diffSnippets=[], + prTitle="Change A", + prDescription=None, + ) + request.get_rag_branch.return_value = "main" + request.get_rag_base_branch.return_value = None + events = [] + + result = await service._fetch_rag_context(request, events.append) + + assert result is None + assert any(event.get("state") == "rag_skipped" for event in events) + # ── _create_llm ────────────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py b/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py index 461da71f..270dac9f 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py @@ -8,6 +8,7 @@ """ import pytest import asyncio +import logging import time from unittest.mock import MagicMock, patch, AsyncMock @@ -39,6 +40,7 @@ _build_duplication_queries_from_diff, _scope_deterministic_to_diff, _extract_calibrated_issues, + _invoke_stage_1_batch_llm, create_smart_batches_wrapper, ) from model.multi_stage import ( @@ -54,6 +56,27 @@ # ── chunk_files ────────────────────────────────────────────────── +@pytest.mark.asyncio(loop_scope="function") +async def test_batch_llm_attempt_details_do_not_duplicate_owner_warning(caplog): + class FailingLlm: + def with_structured_output(self, _schema): + return self + + async def ainvoke(self, _prompt): + raise RuntimeError("provider unavailable") + + with caplog.at_level(logging.DEBUG): + result = await _invoke_stage_1_batch_llm( + FailingLlm(), "prompt", ["src/a.py"], "capped" + ) + + assert result is None + assert not [ + record for record in caplog.records + if record.levelno >= logging.WARNING + ] + + class TestChunkFiles: def _make_groups(self, paths_per_group): groups = [] @@ -1151,6 +1174,7 @@ def _exact_request(self): request.currentCommitHash = "a" * 40 request.commitHash = "a" * 40 request.baseCommitHash = "b" * 40 + request.ragCollectionTarget = "cc_ws_proj_main_generation" request.ragBaseGenerationManifestSha256 = "c" * 64 request.ragPrGenerationFingerprint = "d" * 64 request.ragPrOverlayGenerationManifestSha256 = "e" * 64 @@ -1161,6 +1185,35 @@ def _exact_request(self): request.previousCodeAnalysisIssues = [] return request + @staticmethod + def _batch(path="src/a.py"): + return [{ + "file": ReviewFile( + path=path, + focus_areas=["general"], + risk_level="MEDIUM", + ), + "priority": "MEDIUM", + }] + + class _FallbackAwaitProbe: + def __init__(self): + self.awaited = False + + def __await__(self): + self.awaited = True + + async def resolve(): + return { + "relevant_code": [{ + "text": "STALE_FALLBACK_SENTINEL", + "metadata": {"path": "src/a.py"}, + "score": 1.0, + }] + } + + return resolve().__await__() + @pytest.mark.asyncio(loop_scope="function") async def test_missing_target_branch_does_not_query_an_invented_branch(self): request = self._request() @@ -1182,6 +1235,7 @@ async def test_missing_target_branch_does_not_query_an_invented_branch(self): assert result is None assert state.deterministic_retrieval_states == ["failed"] + assert state.context_disabled is True rag.get_deterministic_context.assert_not_awaited() rag.get_pr_context.assert_not_awaited() rag.search_for_duplicates.assert_not_awaited() @@ -1373,49 +1427,126 @@ async def search_for_duplicates(self, **kwargs): assert rag.semantic_calls == 1 @pytest.mark.asyncio(loop_scope="function") - async def test_deterministic_failure_is_recorded_and_semantic_filler_survives(self): + async def test_concurrent_semantic_failures_emit_one_owner_warning( + self, + caplog, + ): + release = asyncio.Event() + class Rag: + def __init__(self): + self.semantic_calls = 0 + async def get_deterministic_context(self, **kwargs): + return {"context": {"chunks": [], "related_definitions": {}}} + + async def get_pr_context(self, **kwargs): + self.semantic_calls += 1 + await release.wait() return { "status": "error", - "status_code": 500, - "error": "exact retrieval failed", + "status_code": 503, + "error": "RAG service unavailable", } - async def get_pr_context(self, **kwargs): + async def search_for_duplicates(self, **kwargs): + return [] + + rag = Rag() + state = Stage1RagState() + with caplog.at_level( + logging.WARNING, + logger="service.review.orchestrator.stage_1_file_review", + ): + tasks = [ + asyncio.create_task(fetch_batch_rag_context( + rag, + self._request(), + [file_path], + ["changed line"], + batch_priority="MEDIUM", + rag_state=state, + )) + for file_path in ("src/a.py", "src/b.py") + ] + while rag.semantic_calls < 2: + await asyncio.sleep(0) + release.set() + assert await asyncio.gather(*tasks) == [None, None] + + assert state.semantic_disabled is True + assert state.semantic_failures == 1 + owner_warnings = [ + record for record in caplog.records + if record.levelno == logging.WARNING + and "Semantic RAG filler failed" in record.getMessage() + ] + assert len(owner_warnings) == 1 + + @pytest.mark.asyncio(loop_scope="function") + async def test_deterministic_failure_opens_one_review_context_circuit( + self, + caplog, + ): + class Rag: + def __init__(self): + self.deterministic_calls = 0 + self.semantic_calls = 0 + + async def get_deterministic_context(self, **kwargs): + self.deterministic_calls += 1 return { - "context": { - "relevant_code": [ - { - "text": "class Client {}", - "metadata": {"path": "src/client.py"}, - } - ] - } + "status": "error", + "status_code": 500, + "error": "RAG service unavailable", } + async def get_pr_context(self, **kwargs): + self.semantic_calls += 1 + raise AssertionError("semantic retrieval must not run") + async def search_for_duplicates(self, **kwargs): return [] + rag = Rag() state = Stage1RagState() - result = await fetch_batch_rag_context( - Rag(), - self._request(), - ["src/a.py"], - ["changed line"], - batch_priority="MEDIUM", - rag_state=state, - ) + with caplog.at_level( + logging.WARNING, + logger="service.review.orchestrator.stage_1_file_review", + ): + first = await fetch_batch_rag_context( + rag, + self._request(), + ["src/a.py"], + ["changed line"], + batch_priority="MEDIUM", + rag_state=state, + ) + second = await fetch_batch_rag_context( + rag, + self._request(), + ["src/b.py"], + ["changed line"], + batch_priority="MEDIUM", + rag_state=state, + ) - assert result is not None - assert [chunk["text"] for chunk in result["relevant_code"]] == [ - "class Client {}" - ] + assert first is None + assert second is None + assert rag.deterministic_calls == 1 + assert rag.semantic_calls == 0 assert state.deterministic_retrieval_states == ["failed"] - assert state.semantic_disabled is False + assert state.context_disabled is True + assert state.semantic_disabled is True + owner_warnings = [ + record for record in caplog.records + if record.levelno == logging.WARNING + and "Optional RAG context is unavailable" in record.getMessage() + ] + assert len(owner_warnings) == 1 @pytest.mark.asyncio(loop_scope="function") - async def test_exact_deterministic_failure_prevents_review_model_call(self): + async def test_exact_deterministic_failure_fails_open_to_review_model(self): class Rag: async def get_deterministic_context(self, **kwargs): return { @@ -1430,72 +1561,129 @@ async def get_pr_context(self, **kwargs): async def search_for_duplicates(self, **kwargs): return [] - batch = [{ - "file": ReviewFile( - path="src/a.py", - focus_areas=["general"], - risk_level="MEDIUM", - ), - "priority": "MEDIUM", - }] + fallback = self._FallbackAwaitProbe() with patch( "service.review.orchestrator.stage_1_file_review." "_invoke_stage_1_batch_llm", new_callable=AsyncMock, ) as invoke: - with pytest.raises( - RuntimeError, - match="revision-bound deterministic RAG retrieval failed", - ): - await review_file_batch( - MagicMock(), - self._exact_request(), - batch, - rag_client=Rag(), - prepared_context=Stage1PreparedContext(), - pr_indexed=True, - rag_state=Stage1RagState(), - ) + invoke.return_value = [] + state = Stage1RagState() + result = await review_file_batch( + MagicMock(), + self._exact_request(), + self._batch(), + rag_client=Rag(), + prepared_context=Stage1PreparedContext(), + fallback_rag_context=fallback, + pr_indexed=True, + rag_state=state, + ) - invoke.assert_not_awaited() + assert result == [] + assert state.context_disabled is True + assert fallback.awaited is False + assert "STALE_FALLBACK_SENTINEL" not in invoke.await_args.args[1] + invoke.assert_awaited_once() @pytest.mark.asyncio(loop_scope="function") - async def test_exact_missing_rag_client_prevents_review_model_call(self): - batch = [{ - "file": ReviewFile( - path="src/a.py", - focus_areas=["general"], - risk_level="MEDIUM", - ), - "priority": "MEDIUM", - }] + async def test_exact_missing_rag_client_fails_open_to_review_model(self): + fallback = self._FallbackAwaitProbe() with patch( "service.review.orchestrator.stage_1_file_review." "_invoke_stage_1_batch_llm", new_callable=AsyncMock, ) as invoke: - with pytest.raises( - RuntimeError, - match="requires a RAG client", - ): - await review_file_batch( - MagicMock(), - self._exact_request(), - batch, - rag_client=None, - prepared_context=Stage1PreparedContext(), - pr_indexed=True, - rag_state=Stage1RagState(), - ) + invoke.return_value = [] + state = Stage1RagState() + result = await review_file_batch( + MagicMock(), + self._exact_request(), + self._batch(), + rag_client=None, + prepared_context=Stage1PreparedContext(), + fallback_rag_context=fallback, + pr_indexed=True, + rag_state=state, + ) + + assert result == [] + assert state.context_disabled is True + assert fallback.awaited is False + assert "STALE_FALLBACK_SENTINEL" not in invoke.await_args.args[1] + invoke.assert_awaited_once() + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_base_binding_never_consumes_unbound_fallback(self): + request = self._exact_request() + request.ragPrGenerationFingerprint = None + request.ragPrOverlayGenerationManifestSha256 = None + fallback = self._FallbackAwaitProbe() + + with patch( + "service.review.orchestrator.stage_1_file_review." + "_invoke_stage_1_batch_llm", + new_callable=AsyncMock, + return_value=[], + ) as invoke: + state = Stage1RagState() + result = await review_file_batch( + MagicMock(), + request, + self._batch(), + rag_client=None, + prepared_context=Stage1PreparedContext(), + fallback_rag_context=fallback, + pr_indexed=False, + rag_state=state, + ) + + assert result == [] + assert state.context_disabled is True + assert fallback.awaited is False + assert "STALE_FALLBACK_SENTINEL" not in invoke.await_args.args[1] + + @pytest.mark.asyncio(loop_scope="function") + async def test_legacy_unbound_review_still_uses_scoped_fallback(self): + request = self._request() + request.rawDiff = "" + request.deltaDiff = None + request.taskContext = None + request.projectRules = [] + request.previousCodeAnalysisIssues = [] + fallback = self._FallbackAwaitProbe() + + with patch( + "service.review.orchestrator.stage_1_file_review." + "_invoke_stage_1_batch_llm", + new_callable=AsyncMock, + return_value=[], + ) as invoke: + result = await review_file_batch( + MagicMock(), + request, + self._batch(), + rag_client=None, + prepared_context=Stage1PreparedContext(), + fallback_rag_context=fallback, + pr_indexed=False, + rag_state=Stage1RagState(), + ) - invoke.assert_not_awaited() + assert result == [] + assert fallback.awaited is True + assert "STALE_FALLBACK_SENTINEL" in invoke.await_args.args[1] @pytest.mark.asyncio(loop_scope="function") - async def test_exact_partial_deterministic_state_fails_closed(self): + async def test_exact_partial_deterministic_state_fails_open_once(self): class Rag: + def __init__(self): + self.deterministic_calls = 0 + async def get_deterministic_context(self, **kwargs): + self.deterministic_calls += 1 return { "context": { "chunks": [{"text": "partial context"}], @@ -1506,21 +1694,33 @@ async def get_deterministic_context(self, **kwargs): async def search_for_duplicates(self, **kwargs): return [] - with pytest.raises( - RuntimeError, - match="deterministic RAG retrieval is not complete: partial", - ): - await fetch_batch_rag_context( - Rag(), - self._exact_request(), - ["src/a.py"], - ["changed line"], - pr_indexed=True, - rag_state=Stage1RagState(), - ) + rag = Rag() + state = Stage1RagState() + first = await fetch_batch_rag_context( + rag, + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=state, + ) + second = await fetch_batch_rag_context( + rag, + self._exact_request(), + ["src/b.py"], + ["changed line"], + pr_indexed=True, + rag_state=state, + ) + + assert first is None + assert second is None + assert rag.deterministic_calls == 1 + assert state.context_disabled is True + assert state.deterministic_retrieval_states == ["partial"] @pytest.mark.asyncio(loop_scope="function") - async def test_exact_semantic_transport_failure_fails_closed(self): + async def test_exact_semantic_transport_failure_keeps_review_available(self): class Rag: async def get_deterministic_context(self, **kwargs): return { @@ -1540,15 +1740,105 @@ async def get_pr_context(self, **kwargs): async def search_for_duplicates(self, **kwargs): return [] - with pytest.raises(RuntimeError, match="semantic backend unavailable"): - await fetch_batch_rag_context( - Rag(), - self._exact_request(), - ["src/a.py"], - ["changed line"], - pr_indexed=True, - rag_state=Stage1RagState(), - ) + state = Stage1RagState() + result = await fetch_batch_rag_context( + Rag(), + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=state, + ) + + assert result is None + assert state.context_disabled is False + assert state.semantic_disabled is True + assert state.semantic_disable_reason == "semantic backend unavailable" + + @pytest.mark.asyncio(loop_scope="function") + async def test_degraded_overlay_uses_only_exact_base_binding( + self, + monkeypatch, + ): + import service.review.orchestrator.stage_1_file_review as stage1 + + monkeypatch.setattr(stage1, "SEMANTIC_RAG_FILLER_ENABLED", True) + + class Rag: + def __init__(self): + self.deterministic_request = None + self.semantic_request = None + + async def get_deterministic_context(self, **kwargs): + self.deterministic_request = kwargs + return { + "context": { + "chunks": [], + "_metadata": {"retrieval_state": "complete"}, + } + } + + async def get_pr_context(self, **kwargs): + self.semantic_request = kwargs + return {"context": {"relevant_code": []}} + + async def search_for_duplicates(self, **kwargs): + return [] + + request = self._exact_request() + request.ragPrGenerationFingerprint = None + request.ragPrOverlayGenerationManifestSha256 = None + rag = Rag() + + result = await fetch_batch_rag_context( + rag, + request, + ["src/a.py"], + ["changed line"], + pr_indexed=False, + rag_state=Stage1RagState(), + ) + + assert result == {"relevant_code": []} + assert rag.deterministic_request["branches"] == ["feature"] + assert rag.deterministic_request["base_revision"] == "b" * 40 + assert ( + rag.deterministic_request["base_generation_manifest_sha256"] + == "c" * 64 + ) + assert ( + rag.deterministic_request["collection_target"] + == "cc_ws_proj_main_generation" + ) + assert rag.deterministic_request["pr_number"] is None + assert rag.deterministic_request["pr_changed_files"] is None + assert rag.deterministic_request["source_revision"] is None + assert rag.deterministic_request["pr_generation_fingerprint"] is None + assert ( + rag.deterministic_request[ + "pr_overlay_generation_manifest_sha256" + ] + is None + ) + assert rag.semantic_request["base_branch"] is None + assert rag.semantic_request["pr_number"] is None + assert rag.semantic_request["all_pr_changed_files"] is None + assert rag.semantic_request["deleted_files"] is None + assert rag.semantic_request["source_revision"] is None + assert rag.semantic_request["base_revision"] == "b" * 40 + assert ( + rag.semantic_request["base_generation_manifest_sha256"] + == "c" * 64 + ) + assert ( + rag.semantic_request["collection_target"] + == "cc_ws_proj_main_generation" + ) + assert rag.semantic_request["pr_generation_fingerprint"] is None + assert ( + rag.semantic_request["pr_overlay_generation_manifest_sha256"] + is None + ) @pytest.mark.asyncio(loop_scope="function") async def test_exact_bound_success_allows_intentional_semantic_disable( diff --git a/python-ecosystem/inference-orchestrator/tests/test_streaming_router_lifecycle.py b/python-ecosystem/inference-orchestrator/tests/test_streaming_router_lifecycle.py new file mode 100644 index 00000000..30275822 --- /dev/null +++ b/python-ecosystem/inference-orchestrator/tests/test_streaming_router_lifecycle.py @@ -0,0 +1,135 @@ +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from api.routers.commands import ask_endpoint, summarize_endpoint +from api.routers.review import review_endpoint + + +def _streaming_request(service_name, service): + state = SimpleNamespace(**{service_name: service}) + return SimpleNamespace( + headers={"accept": "application/x-ndjson"}, + app=SimpleNamespace(state=state), + ) + + +async def _assert_disconnect_cancels_runner(response, cancelled): + stream = response.body_iterator + queued = await anext(stream) + assert '"state": "queued"' in queued + + progress = await anext(stream) + assert '"state": "working"' in progress + + await stream.aclose() + await asyncio.wait_for(cancelled.wait(), timeout=1) + + +async def _collect_stream_events(response): + return [json.loads(line) async for line in response.body_iterator] + + +@pytest.mark.asyncio(loop_scope="function") +async def test_review_stream_disconnect_cancels_service_runner(): + cancelled = asyncio.Event() + service = MagicMock() + + async def process(_request, event_callback): + event_callback({"type": "status", "state": "working"}) + try: + await asyncio.Future() + finally: + cancelled.set() + + service.process_review_request = AsyncMock(side_effect=process) + response = await review_endpoint( + MagicMock(), + _streaming_request("review_service", service), + ) + + await _assert_disconnect_cancels_runner(response, cancelled) + + +@pytest.mark.asyncio(loop_scope="function") +async def test_review_stream_emits_one_terminal_service_error(): + service = MagicMock() + + async def process(_request, event_callback): + event_callback({"type": "error", "message": "provider failed"}) + event_callback({"type": "status", "state": "late_diagnostic"}) + return {"result": {"status": "error", "message": "provider failed"}} + + service.process_review_request = AsyncMock(side_effect=process) + response = await review_endpoint( + MagicMock(), + _streaming_request("review_service", service), + ) + + events = await _collect_stream_events(response) + assert events[-1] == {"type": "error", "message": "provider failed"} + assert sum(event["type"] in {"error", "final"} for event in events) == 1 + + +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.parametrize( + ("endpoint", "method_name"), + ( + (summarize_endpoint, "process_summarize"), + (ask_endpoint, "process_ask"), + ), +) +async def test_command_stream_disconnect_cancels_service_runner( + endpoint, + method_name, +): + cancelled = asyncio.Event() + service = MagicMock() + + async def process(_request, event_callback): + event_callback({"type": "status", "state": "working"}) + try: + await asyncio.Future() + finally: + cancelled.set() + + setattr(service, method_name, AsyncMock(side_effect=process)) + response = await endpoint( + MagicMock(), + _streaming_request("command_service", service), + ) + + await _assert_disconnect_cancels_runner(response, cancelled) + + +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.parametrize( + ("endpoint", "method_name"), + ( + (summarize_endpoint, "process_summarize"), + (ask_endpoint, "process_ask"), + ), +) +async def test_command_stream_emits_one_terminal_service_error( + endpoint, + method_name, +): + service = MagicMock() + + async def process(_request, event_callback): + event_callback({"type": "error", "message": "provider failed"}) + event_callback({"type": "status", "state": "late_diagnostic"}) + return {"error": "provider failed"} + + setattr(service, method_name, AsyncMock(side_effect=process)) + response = await endpoint( + MagicMock(), + _streaming_request("command_service", service), + ) + + events = await _collect_stream_events(response) + assert events[-1] == {"type": "error", "message": "provider failed"} + assert sum(event["type"] in {"error", "final"} for event in events) == 1 diff --git a/python-ecosystem/rag-pipeline/integration/test_pr_endpoints.py b/python-ecosystem/rag-pipeline/integration/test_pr_endpoints.py index 3a7a6f3e..2ef1d126 100644 --- a/python-ecosystem/rag-pipeline/integration/test_pr_endpoints.py +++ b/python-ecosystem/rag-pipeline/integration/test_pr_endpoints.py @@ -263,7 +263,10 @@ async def test_delete_pr_files_success(self, client, auth_headers, rag_app): import rag_pipeline.api.api as api_module im = api_module.index_manager im._get_project_collection_name.return_value = "col_ws_proj" - im._collection_manager.collection_exists.return_value = True + im._collection_manager.resolve_collection_target.side_effect = None + im._collection_manager.resolve_collection_target.return_value = ( + "col_ws_proj" + ) im.qdrant_client.delete.return_value = None resp = await client.delete( @@ -281,7 +284,8 @@ async def test_delete_pr_files_collection_not_found(self, client, auth_headers, import rag_pipeline.api.api as api_module im = api_module.index_manager im._get_project_collection_name.return_value = "col_ws_proj" - im._collection_manager.collection_exists.return_value = False + im._collection_manager.resolve_collection_target.side_effect = None + im._collection_manager.resolve_collection_target.return_value = None resp = await client.delete( "/index/pr-files/ws/proj/99", @@ -291,12 +295,22 @@ async def test_delete_pr_files_collection_not_found(self, client, auth_headers, body = resp.json() assert body["status"] == "skipped" + # The application singleton is session-scoped in this integration + # suite. Restore a normal resolver so this absence case cannot leak + # into the following Qdrant-error scenario. + im._collection_manager.resolve_collection_target.side_effect = ( + lambda collection_name: collection_name + ) + async def test_delete_pr_files_qdrant_error(self, client, auth_headers, rag_app): """Qdrant error → 500.""" import rag_pipeline.api.api as api_module im = api_module.index_manager im._get_project_collection_name.return_value = "col_ws_proj" - im._collection_manager.collection_exists.return_value = True + im._collection_manager.resolve_collection_target.side_effect = None + im._collection_manager.resolve_collection_target.return_value = ( + "col_ws_proj" + ) im.qdrant_client.delete.side_effect = RuntimeError("qdrant down") resp = await client.delete( diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/api.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/api.py index fcaccc42..a7a3c12c 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/api.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/api.py @@ -47,18 +47,34 @@ def _pending_janitor_interval_seconds() -> int: async def _pending_collection_janitor(manager: RAGIndexManager) -> None: """Periodically remove only expired, unowned pending collections.""" interval = _pending_janitor_interval_seconds() + unavailable = False while True: try: cleaned = await asyncio.to_thread( manager.cleanup_expired_pending_collections ) + if unavailable: + logger.info("Pending collection janitor recovered") + unavailable = False if cleaned: logger.info("Pending collection janitor removed %s collections", cleaned) except asyncio.CancelledError: raise - except Exception: + except Exception as exception: # Cleanup is auxiliary: retain uncertain collections and keep serving. - logger.exception("Pending collection janitor failed") + if not unavailable: + logger.warning( + "Pending collection janitor unavailable; retaining " + "uncertain collections: %s", + exception, + exc_info=True, + ) + unavailable = True + else: + logger.debug( + "Pending collection janitor still unavailable: %s", + exception, + ) await asyncio.sleep(interval) @@ -77,6 +93,17 @@ async def lifespan(app: FastAPI): config, plugin_catalog=index_manager.plugin_catalog, ) + from .routers.index import ( + cleanup_orphaned_index_repository_stream_workspaces, + ) + cleaned_stream_workspaces = ( + cleanup_orphaned_index_repository_stream_workspaces() + ) + if cleaned_stream_workspaces: + logger.info( + "Removed %s orphaned RAG HTTP index workspaces", + cleaned_stream_workspaces, + ) # Initialize and start the Redis Queue Consumer from ..server.rag_queue_consumer import RAGQueueConsumer @@ -98,10 +125,18 @@ async def lifespan(app: FastAPI): pass if hasattr(app.state, 'rag_queue_consumer'): await app.state.rag_queue_consumer.stop() + # HTTP streaming requests run synchronous indexing in dedicated workers. + # A disconnected response task can be gone before that call returns, so + # drain the independently tracked workers before closing shared embedding + # and Qdrant clients. + from .routers.index import drain_index_repository_stream_workers + await drain_index_repository_stream_workers() if hasattr(index_manager, 'embed_model') and hasattr(index_manager.embed_model, 'close'): index_manager.embed_model.close() if hasattr(query_service, 'embed_model') and hasattr(query_service.embed_model, 'close'): query_service.embed_model.close() + if query_service is not None: + query_service.close() if index_manager is not None: index_manager.close() logger.info("RAG Pipeline API shutdown complete") diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py index 21922f54..7236354a 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py @@ -48,6 +48,7 @@ class IndexRequest(BaseModel): publish_legacy_project_alias: bool = False preserve_other_branches: bool = False cleanup_repo_path: bool = False + transfer_repo_ownership: bool = False include_patterns: Optional[List[str]] = None exclude_patterns: Optional[List[str]] = None @@ -142,6 +143,10 @@ class GenerationAliasPublicationRequest(BaseModel): branch: str commit: str collection_target: str = Field(min_length=1) + generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) publish_branch_alias: bool = True publish_legacy_project_alias: bool = False diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py index aeb8f473..dd298266 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py @@ -1,10 +1,16 @@ """Index and branch management endpoints.""" import asyncio +import fcntl import json import logging -from queue import Empty, Queue -from threading import Thread -from typing import List +import os +import shutil +import time +from queue import Empty, Full, Queue +from pathlib import Path +from threading import Lock, Thread, current_thread +from typing import BinaryIO, Callable, List +from uuid import uuid4 from fastapi import APIRouter, HTTPException, BackgroundTasks, Query from fastapi.responses import StreamingResponse @@ -26,6 +32,223 @@ router = APIRouter(tags=["index"]) +class _IndexStreamWorkerRegistry: + """Track synchronous HTTP-stream indexing beyond request cancellation.""" + + def __init__(self) -> None: + self._workers: set[Thread] = set() + self._lock = Lock() + + def start(self, target: Callable[[], None]) -> Thread: + """Admit and start one worker before exposing its response stream.""" + + def run_tracked() -> None: + try: + target() + finally: + with self._lock: + self._workers.discard(current_thread()) + + worker = Thread( + target=run_tracked, + name="rag-index-progress", + # Graceful shutdown drains these workers explicitly. Keeping them + # non-daemon also prevents interpreter teardown from closing + # shared clients while an admitted index operation is still using + # them. + daemon=False, + ) + with self._lock: + self._workers.add(worker) + try: + worker.start() + except BaseException: + with self._lock: + self._workers.discard(worker) + raise + return worker + + async def wait_for(self, worker: Thread) -> None: + """Wait for a worker, deferring cancellation until it has returned.""" + cancellation: asyncio.CancelledError | None = None + while worker.is_alive(): + try: + await asyncio.sleep(0.05) + except asyncio.CancelledError as exception: + # A disconnected streaming client must not unwind its server + # handler while the admitted worker can still read the + # caller-owned repository snapshot. + cancellation = cancellation or exception + worker.join() + if cancellation is not None: + raise cancellation + + async def drain(self) -> None: + """Wait for every currently admitted stream worker.""" + announced = False + while True: + with self._lock: + active_workers = tuple(self._workers) + if not active_workers: + return + if not announced: + logger.info( + "Waiting for %s HTTP RAG indexing workers before shutdown", + len(active_workers), + ) + announced = True + for worker in active_workers: + await self.wait_for(worker) + + @property + def active_count(self) -> int: + with self._lock: + return len(self._workers) + +_index_stream_workers = _IndexStreamWorkerRegistry() +_OWNED_STREAM_DIRECTORY_PREFIX = "codecrow-rag-owned-stream-" +_DEFAULT_OWNED_STREAM_ORPHAN_AGE_SECONDS = 24 * 60 * 60 + + +async def drain_index_repository_stream_workers() -> None: + """Drain admitted HTTP index operations before shared clients close.""" + await _index_stream_workers.drain() + + +def cleanup_orphaned_index_repository_stream_workspaces( + max_age_seconds: int = _DEFAULT_OWNED_STREAM_ORPHAN_AGE_SECONDS, +) -> int: + """Remove only old RAG-owned workspaces left by an earlier process.""" + allowed_root = Path(os.environ.get("ALLOWED_REPO_ROOT", "/tmp")).resolve() + if not allowed_root.is_dir(): + return 0 + cutoff = time.time() - max(0, max_age_seconds) + cleaned = 0 + for candidate in allowed_root.iterdir(): + if ( + not candidate.name.startswith(_OWNED_STREAM_DIRECTORY_PREFIX) + or candidate.is_symlink() + or not candidate.is_dir() + ): + continue + try: + if candidate.stat().st_mtime > cutoff: + continue + lock_path = allowed_root / f".{candidate.name}.lock" + lock_descriptor = os.open( + lock_path, + os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, + 0o600, + ) + lock_file = os.fdopen(lock_descriptor, "a+b") + try: + try: + fcntl.flock( + lock_file.fileno(), + fcntl.LOCK_EX | fcntl.LOCK_NB, + ) + except BlockingIOError: + # Another RAG process still owns this workspace. Its age + # alone is never authority to disrupt an active worker. + continue + shutil.rmtree(candidate) + cleaned += 1 + finally: + lock_file.close() + if not candidate.exists(): + lock_path.unlink(missing_ok=True) + except FileNotFoundError: + continue + except Exception: + logger.exception( + "Failed to remove orphaned RAG HTTP index workspace: %s", + candidate, + ) + return cleaned + + +def _coalesce_stream_progress( + events: Queue[dict], + event: dict, +) -> None: + """Retain only the latest undelivered optional progress event.""" + while True: + try: + events.put_nowait(event) + return + except Full: + try: + events.get_nowait() + except Empty: + continue + + +def _take_stream_repository_ownership( + repo_path: str, +) -> tuple[Path, BinaryIO]: + """Atomically move one Java-owned snapshot into the RAG namespace.""" + allowed_root = Path(os.environ.get("ALLOWED_REPO_ROOT", "/tmp")).resolve() + source = Path(repo_path).resolve() + if ( + source.parent != allowed_root + or not source.name.startswith("codecrow-rag-branch-generation-") + or not source.is_dir() + ): + raise ValueError( + "stream repository ownership transfer requires an existing " + "codecrow-rag-branch-generation-* directory directly under " + f"{allowed_root}" + ) + owned = allowed_root / f"{_OWNED_STREAM_DIRECTORY_PREFIX}{uuid4().hex}" + lock_path = allowed_root / f".{owned.name}.lock" + lock_file = None + try: + lock_descriptor = os.open( + lock_path, + os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, + 0o600, + ) + lock_file = os.fdopen(lock_descriptor, "a+b") + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BaseException: + if lock_file is not None: + lock_file.close() + lock_path.unlink(missing_ok=True) + raise + # Both services mount the same temporary volume. A rename in that volume + # is atomic: after this point Java may safely clean the now-absent source + # path even if the admission SSE is lost, while RAG alone owns `owned`. + try: + source.rename(owned) + except BaseException: + lock_file.close() + lock_path.unlink(missing_ok=True) + raise + return owned, lock_file + + +def _remove_owned_stream_repository( + repo_path: Path, + lock_file: BinaryIO, +) -> None: + removed = False + try: + shutil.rmtree(repo_path) + removed = True + except FileNotFoundError: + removed = True + except Exception: + logger.exception( + "Failed to remove RAG-owned HTTP index workspace: %s", + repo_path, + ) + finally: + lock_path = repo_path.parent / f".{repo_path.name}.lock" + lock_file.close() + if removed: + lock_path.unlink(missing_ok=True) + + def _get_singletons(): """Get lifecycle-managed singletons from the api module.""" from ..api import config, index_manager @@ -139,73 +362,149 @@ def index_repository_stream(request: IndexRequest): delivery a prerequisite for a successful snapshot. """ _, index_manager = _get_singletons() + progress_events: Queue[dict] = Queue(maxsize=1) + terminal_events: Queue[tuple[str, object]] = Queue(maxsize=1) + repository_ownership_transferred = ( + getattr(request, "transfer_repo_ownership", False) is True + ) + index_repo_path = Path(request.repo_path) + repository_ownership_lock: BinaryIO | None = None + if repository_ownership_transferred: + try: + ( + index_repo_path, + repository_ownership_lock, + ) = _take_stream_repository_ownership(request.repo_path) + except ValueError as exception: + raise HTTPException(status_code=400, detail=str(exception)) + except OSError as exception: + raise HTTPException( + status_code=503, + detail=( + "RAG could not take repository snapshot ownership: " + f"{exception}" + ), + ) - async def event_stream(): - events: Queue[tuple[str, object]] = Queue() - - def progress(event: dict) -> None: - events.put(("progress", event)) + def progress(event: dict) -> None: + _coalesce_stream_progress(progress_events, event) - def run_index() -> None: - try: - optional_generation_args = {} - if request.source_tree_sha256: - optional_generation_args["source_tree_sha256"] = ( - request.source_tree_sha256 - ) - if request.collection_target: - optional_generation_args["collection_target"] = ( - request.collection_target - ) - if getattr(request, "publish_branch_alias", False) is True: - optional_generation_args["publish_branch_alias"] = True - if getattr(request, "publish_legacy_project_alias", False) is True: - optional_generation_args["publish_legacy_project_alias"] = True - stats = index_manager.index_repository( - repo_path=request.repo_path, - workspace=request.workspace, - project=request.project, - branch=request.branch, - commit=request.commit, - preserve_other_branches=request.preserve_other_branches, - include_patterns=request.include_patterns, - exclude_patterns=request.exclude_patterns, - progress_callback=progress, - **optional_generation_args, + def run_index() -> None: + try: + optional_generation_args = {} + if request.source_tree_sha256: + optional_generation_args["source_tree_sha256"] = ( + request.source_tree_sha256 + ) + if request.collection_target: + optional_generation_args["collection_target"] = ( + request.collection_target + ) + if getattr(request, "publish_branch_alias", False) is True: + optional_generation_args["publish_branch_alias"] = True + if getattr(request, "publish_legacy_project_alias", False) is True: + optional_generation_args["publish_legacy_project_alias"] = True + stats = index_manager.index_repository( + repo_path=str(index_repo_path), + workspace=request.workspace, + project=request.project, + branch=request.branch, + commit=request.commit, + preserve_other_branches=request.preserve_other_branches, + include_patterns=request.include_patterns, + exclude_patterns=request.exclude_patterns, + progress_callback=progress, + **optional_generation_args, + ) + terminal_events.put( + ("complete", stats.model_dump(mode="json")) + ) + except Exception as exception: + # The terminal event gives the Java job owner complete context and + # that owner emits the rate-bounded diagnostic. Avoid logging the + # same failure again at this transport layer. + logger.debug( + "RAG repository stream worker failed: %s", exception, + exc_info=True, + ) + terminal_events.put(("error", {"message": str(exception)})) + finally: + if repository_ownership_transferred: + _remove_owned_stream_repository( + index_repo_path, + repository_ownership_lock, ) - events.put(("complete", stats.model_dump(mode="json"))) - except Exception as exception: - logger.error("Error indexing repository with progress: %s", exception) - events.put(("error", {"message": str(exception)})) - worker = Thread( - target=run_index, - name="rag-index-progress", - daemon=True, - ) - worker.start() - while True: + try: + # Admission happens before successful response headers. Ownership is + # already represented by an atomic rename, so a lost admission event + # cannot make the Java caller delete the path this worker is reading. + worker = _index_stream_workers.start(run_index) + except BaseException: + if repository_ownership_transferred: + _remove_owned_stream_repository( + index_repo_path, + repository_ownership_lock, + ) + raise + + async def event_stream(): + try: + if repository_ownership_transferred: + admitted = { + "type": "admitted", + "repositoryOwnershipTransferred": True, + } + yield f"data: {json.dumps(admitted)}\n\n" + while True: + try: + payload = progress_events.get_nowait() + event_type = "progress" + except Empty: + try: + event_type, payload = terminal_events.get_nowait() + except Empty: + if not worker.is_alive(): + # run_index publishes a terminal event before it + # returns. This guards an unexpected BaseException + # in the worker without leaving the stream open. + event_type = "error" + payload = { + "message": ( + "RAG indexing worker stopped without a " + "terminal result" + ) + } + else: + # Polling thread-safe queues avoids nesting a + # blocking consumer inside Starlette's thread pool. + await asyncio.sleep(0.05) + continue + if event_type == "progress": + event = {"type": "progress", **payload} + elif event_type == "complete": + event = {"type": "complete", "result": payload} + else: + event = {"type": "error", **payload} + yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n" + if event_type in {"complete", "error"}: + break + finally: + # StreamingResponse closes this async generator when its client + # disconnects. Defer handler teardown until the independently + # admitted synchronous operation has returned. + worker_drain = asyncio.create_task( + _index_stream_workers.wait_for(worker) + ) try: - event_type, payload = events.get_nowait() - except Empty: - # Polling a thread-safe queue avoids nesting a blocking queue - # consumer inside Starlette's thread pool. The short wait keeps - # the event loop responsive and progress delivery prompt. - await asyncio.sleep(0.05) - continue - if event_type == "progress": - event = {"type": "progress", **payload} - elif event_type == "complete": - event = {"type": "complete", "result": payload} - else: - event = {"type": "error", **payload} - yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n" - if event_type in {"complete", "error"}: - # The terminal event is scheduled just before the producer - # returns. Join that final unwind so a short-lived consumer - # cannot finish while its producer thread is still active. - worker.join(timeout=1.0) - break + await asyncio.shield(worker_drain) + except asyncio.CancelledError: + # Cancellation can be delivered before an awaited coroutine + # executes its own cancellation handler. Shield admission + # draining as a separate task, then re-raise only after it has + # completed. + await worker_drain + raise return StreamingResponse(event_stream(), media_type="text/event-stream") @@ -348,6 +647,7 @@ def publish_generation_aliases(request: GenerationAliasPublicationRequest): branch=request.branch, commit=request.commit, collection_target=request.collection_target, + generation_manifest_sha256=request.generation_manifest_sha256, publish_branch_alias=request.publish_branch_alias, publish_legacy_project_alias=request.publish_legacy_project_alias, ) @@ -359,7 +659,10 @@ def publish_generation_aliases(request: GenerationAliasPublicationRequest): except MutationCoordinationUnavailable as e: raise HTTPException(status_code=503, detail=str(e)) except Exception as e: - logger.error("Error publishing readable generation aliases: %s", e) + # Alias repair is optional and retried by the registry owner, which + # emits the contextual transition alert. Avoid a fixed-delay ERROR + # stream here while preserving the HTTP failure for that caller. + logger.info("Readable generation alias publication failed: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -387,12 +690,22 @@ def delete_branch( project: str, branch: str, collection_target: str | None = Query(default=None), + generation_revision: str | None = Query(default=None, min_length=1), + generation_manifest_sha256: str | None = Query( + default=None, + pattern=r"^[0-9a-f]{64}$", + ), ): """Delete all points for a specific branch from the project collection.""" _, index_manager = _get_singletons() try: success = index_manager.delete_branch( - workspace, project, branch, collection_target=collection_target + workspace, + project, + branch, + collection_target=collection_target, + generation_revision=generation_revision, + generation_manifest_sha256=generation_manifest_sha256, ) if success: return { diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py index b86625e4..32dbd678 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py @@ -800,10 +800,10 @@ def index_pr_files(request: PRIndexRequest): except HTTPException: raise except IncrementalIndexPreconditionError as e: - logger.warning("Rejected PR indexing against invalid repository state: %s", e) + logger.info("Rejected PR indexing against invalid repository state: %s", e) raise HTTPException(status_code=409, detail=str(e)) except ValueError as e: - logger.warning(f"Invalid request for PR indexing: {e}") + logger.info(f"Invalid request for PR indexing: {e}") raise HTTPException(status_code=400, detail=str(e)) except MutationLeaseUnavailable as e: raise HTTPException(status_code=409, detail=str(e)) @@ -826,6 +826,8 @@ def delete_pr_files( ): """Delete all indexed points for a specific PR.""" index_manager = _get_index_manager() + if not isinstance(collection_target, str) or not collection_target.strip(): + collection_target = None try: with index_manager.pr_overlay_mutation( workspace, @@ -837,13 +839,25 @@ def delete_pr_files( collection_target or index_manager._get_project_collection_name(workspace, project) ) - - if not index_manager._collection_manager.collection_exists(collection_name): + physical_collection = ( + index_manager._collection_manager.resolve_collection_target( + collection_name + ) + ) + if physical_collection is None: return {"status": "skipped", "message": "Collection does not exist"} + # Existing exact generations may predate the PR filter indexes. + # Repair them before the acknowledged filter delete. Keeping + # wait=True is correctness-critical: releasing the same-PR lease + # before Qdrant applies the delete could erase a subsequent rerun. + index_manager._collection_manager.ensure_payload_indexes( + physical_collection + ) + lease.assert_owned() index_manager.qdrant_client.delete( - collection_name=collection_name, + collection_name=physical_collection, points_selector=Filter( must=[ FieldCondition(key="workspace", match=MatchValue(value=workspace)), @@ -851,15 +865,20 @@ def delete_pr_files( FieldCondition(key="pr", match=MatchValue(value=True)), FieldCondition(key="pr_number", match=MatchValue(value=pr_number)), ] - ) + ), + wait=True, ) - logger.info(f"Deleted PR #{pr_number} points from {collection_name}") + logger.info( + "Deleted PR #%s points from %s", + pr_number, + physical_collection, + ) return { "status": "deleted", "pr_number": pr_number, - "collection": collection_name + "collection": physical_collection } except MutationLeaseUnavailable as e: @@ -867,5 +886,5 @@ def delete_pr_files( except MutationCoordinationUnavailable as e: raise HTTPException(status_code=503, detail=str(e)) except Exception as e: - logger.error(f"Error deleting PR files: {e}") + logger.debug("PR file deletion failed: %s", e, exc_info=True) raise HTTPException(status_code=500, detail=str(e)) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py index 562d5b9e..ce72bd8c 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py @@ -287,12 +287,11 @@ def acquire( def is_operation_active(self, token: str) -> bool: if not self.enabled or self._client is None: return False - try: - return bool(self._client.exists(f"codecrow:rag:operation:{token}")) - except Exception: - # Cleanup is auxiliary and must fail open by retaining collections. - logger.warning("Could not verify pending-collection operation %s", token) - return True + # Cleanup is auxiliary and must retain uncertain collections. Let an + # unavailable ownership store escape to the lifecycle janitor instead + # of logging once per candidate; that owner stops the pass and emits + # one transition-bounded outage/recovery diagnostic. + return bool(self._client.exists(f"codecrow:rag:operation:{token}")) def close(self) -> None: if self._client is not None: @@ -322,6 +321,7 @@ def __init__( health_check_interval=30, ) self._disabled_until = 0.0 + self._distributed_unavailable = False self._state_lock = threading.Lock() self._key = "codecrow:rag:openrouter:index:permits" @@ -337,8 +337,8 @@ def permit(self) -> Iterator[None]: if distributed: try: self._client.zrem(self._key, token) - except Exception: - logger.warning("Could not release distributed OpenRouter permit") + except Exception as exception: + self._record_distributed_failure(exception) self._local.release() def _acquire_distributed(self, token: str) -> bool: @@ -346,6 +346,7 @@ def _acquire_distributed(self, token: str) -> bool: if time.monotonic() < self._disabled_until: return False deadline = time.monotonic() + self.acquire_timeout_seconds + wait_reported = False while True: now = time.time() try: @@ -360,23 +361,40 @@ def _acquire_distributed(self, token: str) -> bool: self.permit_seconds, ) if acquired: + self._record_distributed_recovery() return True except Exception as exception: with self._state_lock: self._disabled_until = time.monotonic() + 60 - logger.warning( - "Distributed OpenRouter capacity limit unavailable; " - "using process-local cap for 60s: %s", - exception, - ) + self._record_distributed_failure(exception) return False if time.monotonic() >= deadline: - logger.warning( - "Still waiting for distributed OpenRouter capacity after %.1fs", + log = logger.info if not wait_reported else logger.debug + log( + "Waiting for distributed OpenRouter capacity after %.1fs", self.acquire_timeout_seconds, ) + wait_reported = True deadline = time.monotonic() + self.acquire_timeout_seconds time.sleep(0.05) + def _record_distributed_failure(self, exception: BaseException) -> None: + with self._state_lock: + first_failure = not self._distributed_unavailable + self._distributed_unavailable = True + log = logger.warning if first_failure else logger.debug + log( + "Distributed OpenRouter capacity limit unavailable; using " + "process-local cap: %s", + exception, + ) + + def _record_distributed_recovery(self) -> None: + with self._state_lock: + recovered = self._distributed_unavailable + self._distributed_unavailable = False + if recovered: + logger.info("Distributed OpenRouter capacity limit recovered") + def close(self) -> None: self._client.close() diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py index 75c5977b..5bc2c805 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py @@ -6,6 +6,7 @@ import json import math import re +import uuid from collections.abc import Iterable, Mapping, Sequence from typing import Any @@ -377,3 +378,13 @@ def build_generation_manifest_node( def is_sha256_hex(value: object) -> bool: """Return whether ``value`` is one canonical lower-case SHA-256 digest.""" return isinstance(value, str) and _SHA256_RE.fullmatch(value) is not None + + +def generation_manifest_point_id( + workspace: str, + project: str, + branch: str, +) -> str: + """Return the deterministic storage ID of a repository generation seal.""" + key = f"{workspace}:{project}:{branch}:{GENERATION_MANIFEST_PATH}:0" + return str(uuid.uuid5(uuid.NAMESPACE_DNS, key)) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py index f2a43d84..6c893b5e 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py @@ -7,12 +7,16 @@ import logging import os import re +import threading import time import uuid from typing import Callable, Mapping, Optional, List from qdrant_client import QdrantClient -from qdrant_client.http.exceptions import UnexpectedResponse +from qdrant_client.http.exceptions import ( + ResponseHandlingException, + UnexpectedResponse, +) from qdrant_client.models import ( Distance, VectorParams, CreateAlias, DeleteAlias, CreateAliasOperation, DeleteAliasOperation, @@ -29,6 +33,9 @@ def __init__(self, client: QdrantClient, embedding_dim: int): self.client = client self.embedding_dim = embedding_dim self.vectors_on_disk = os.environ.get("QDRANT_VECTORS_ON_DISK", "true").lower() == "true" + self._payload_indexes_ensured: set[str] = set() + self._payload_indexes_in_progress: set[str] = set() + self._payload_index_condition = threading.Condition() def ensure_collection_exists(self, collection_name: str) -> None: """Ensure Qdrant collection exists with proper configuration. @@ -37,6 +44,9 @@ def ensure_collection_exists(self, collection_name: str) -> None: """ if self.alias_exists(collection_name): logger.info(f"Collection name {collection_name} is an alias, using existing aliased collection") + physical = self.resolve_collection_target(collection_name) + if physical is not None: + self.ensure_payload_indexes(physical) return collections = self.client.get_collections().collections @@ -53,9 +63,10 @@ def ensure_collection_exists(self, collection_name: str) -> None: "Collection %s was created concurrently; using it", collection_name, ) - self._ensure_payload_indexes(collection_name) + self.ensure_payload_indexes(collection_name) else: logger.info(f"Collection {collection_name} already exists") + self.ensure_payload_indexes(collection_name) def create_pending_collection( self, @@ -79,6 +90,8 @@ def create_pending_collection( ) logger.info(f"Creating pending collection: {pending_name}") if self._create_collection(pending_name): + # Pending names are deliberately unique and short lived; do + # not retain every generation in the process-wide cache. self._ensure_payload_indexes(pending_name) return pending_name logger.warning( @@ -115,36 +128,136 @@ def _physical_collection_exists(self, collection_name: str) -> bool: collections = self.client.get_collections().collections return any(collection.name == collection_name for collection in collections) - def _ensure_payload_indexes(self, collection_name: str) -> None: - """Create payload indexes for efficient filtering on common fields.""" - fields = ( - "path", - "branch", - "architecture_paths", - "architecture_group", - "snapshot_plugin", - "snapshot_kind", + @staticmethod + def _payload_index_specs(): + """Fields used by bounded tenant, revision, and PR filters.""" + return ( + ("path", PayloadSchemaType.KEYWORD), + ("branch", PayloadSchemaType.KEYWORD), + ("workspace", PayloadSchemaType.KEYWORD), + ("project", PayloadSchemaType.KEYWORD), + ("commit", PayloadSchemaType.KEYWORD), + ("architecture_paths", PayloadSchemaType.KEYWORD), + ("architecture_group", PayloadSchemaType.KEYWORD), + ("snapshot_plugin", PayloadSchemaType.KEYWORD), + ("snapshot_kind", PayloadSchemaType.KEYWORD), + ("pr", PayloadSchemaType.BOOL), + ("pr_number", PayloadSchemaType.INTEGER), + ("repository_generation_manifest", PayloadSchemaType.BOOL), + ("generation_manifest_sha256", PayloadSchemaType.KEYWORD), ) - for field_name in fields: + + def ensure_payload_indexes(self, collection_name: str) -> None: + """Repair required indexes once per physical collection and process. + + Collections created before a field was introduced are repaired on + first use after restart. Failures remain fail-open for the current + operation and retry on a later request instead of being cached. + """ + with self._payload_index_condition: + while collection_name in self._payload_indexes_in_progress: + self._payload_index_condition.wait() + if collection_name in self._payload_indexes_ensured: + return + self._payload_indexes_in_progress.add(collection_name) + + successful = False + try: + successful = self._ensure_payload_indexes(collection_name) + finally: + with self._payload_index_condition: + self._payload_indexes_in_progress.discard(collection_name) + if successful: + self._payload_indexes_ensured.add(collection_name) + self._payload_index_condition.notify_all() + + def _existing_payload_index_types( + self, + collection_name: str, + ) -> Optional[dict]: + """Read schemas once so existing fields need no write request.""" + try: + payload_schema = getattr( + self.client.get_collection(collection_name), + "payload_schema", + {}, + ) + except Exception as exception: + logger.warning( + "Deferring payload index repair on %s because its schema " + "could not be inspected: %s", + collection_name, + exception, + ) + return None + if not isinstance(payload_schema, Mapping): + return {} + return { + field_name: getattr(index_info, "data_type", index_info) + for field_name, index_info in payload_schema.items() + } + + def _ensure_payload_indexes(self, collection_name: str) -> bool: + """Create payload indexes for efficient filtering on common fields.""" + successful = True + failures = [] + existing_types = self._existing_payload_index_types(collection_name) + if existing_types is None: + return False + for field_name, field_schema in self._payload_index_specs(): + if existing_types.get(field_name) == field_schema: + continue try: self.client.create_payload_index( collection_name=collection_name, field_name=field_name, - field_schema=PayloadSchemaType.KEYWORD, + field_schema=field_schema, + wait=True, ) + except ResponseHandlingException as exception: + successful = False + failures.append((field_name, exception)) + break + except UnexpectedResponse as exception: + successful = False + failures.append((field_name, exception)) + if exception.status_code in {401, 403, 404, 408, 429} or ( + exception.status_code >= 500 + ): + break except Exception as exception: - logger.warning( - "Failed to create payload index %s on %s: %s", - field_name, - collection_name, - exception, - ) - logger.info(f"Payload indexes ensured for {collection_name}") + successful = False + failures.append((field_name, exception)) + if successful: + logger.info("Payload indexes ensured for %s", collection_name) + else: + first_field, first_exception = failures[0] + logger.warning( + "Payload index repair failed for %s field(s) on %s; first " + "failure was %s: %s", + len(failures), + collection_name, + first_field, + first_exception, + ) + logger.info( + "Payload index repair remains incomplete for %s; it will be " + "retried on a later use", + collection_name, + ) + return successful def delete_collection(self, collection_name: str) -> bool: """Delete a collection.""" try: self.client.delete_collection(collection_name) + # A direct/legacy name can be recreated in the same process. Wait + # for a concurrent repair to finish, then invalidate its receipt + # so the replacement collection receives every required index. + with self._payload_index_condition: + while collection_name in self._payload_indexes_in_progress: + self._payload_index_condition.wait() + self._payload_indexes_ensured.discard(collection_name) logger.info(f"Deleted collection: {collection_name}") return True except Exception as e: @@ -338,13 +451,14 @@ def cleanup_expired_pending_collections( int(os.getenv("RAG_PENDING_COLLECTION_MAX_AGE_SECONDS", "21600")), ) now = int(time.time()) - try: - aliased_targets = { - alias.collection_name for alias in self.client.get_aliases().aliases - } - except Exception: - logger.warning("Pending collection janitor could not read aliases") - return 0 + # Alias membership is a safety precondition: an unavailable Qdrant + # response must escape to the lifecycle janitor, which owns the + # transition-bounded outage/recovery diagnostic. Returning zero here + # would make an outage look like a healthy empty cleanup and emit one + # warning on every scheduled pass. + aliased_targets = { + alias.collection_name for alias in self.client.get_aliases().aliases + } pattern = re.compile( r"_pending_(\d{10})_([a-fA-F0-9]{8,32})_[a-fA-F0-9]{8}$" diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py index da808628..7bb58dc7 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py @@ -1751,21 +1751,30 @@ def _apply_change_set( commit=revision, ) - if repository_analysis_plugins: - documents_by_path = { - document.metadata["path"]: document for document in documents - } - missing = sorted( - path for path in active_updated_paths - if path not in documents_by_path + documents_by_path = { + document.metadata["path"]: document for document in documents + } + missing = sorted( + path for path in active_updated_paths + if path not in documents_by_path + ) + if missing: + logger.info( + "Quarantining %s changed repository files that could not " + "be loaded during incremental indexing: %s", + len(missing), + ", ".join(missing[:10]), ) - if missing: - logger.warning( - "Quarantining %s changed repository files that could not " - "be loaded during incremental indexing: %s", - len(missing), - ", ".join(missing[:10]), - ) + + if repository_analysis_plugins: + # A loader omission is not a deletion. Lock files, binary files, + # oversized files, and locally excluded files can all be present + # in the authoritative checkout while intentionally producing no + # Document. Feeding those paths to repository plugins as + # tombstones corrupts their restored state; dereferencing the + # absent document also used to raise a raw KeyError (for example, + # ``composer.lock``). Only successfully loaded updates and + # provider-declared deletions participate in the plugin overlay. artifacts = tuple(sorted(( *( FileArtifact( @@ -1773,10 +1782,11 @@ def _apply_change_set( content=documents_by_path[path].text, ) for path in active_updated_paths + if path in documents_by_path ), *( FileArtifact(path=path, content="", deleted=True) - for path in (*active_deleted_paths, *missing) + for path in active_deleted_paths ), ), key=lambda artifact: artifact.path)) handle = self.plugin_runtime.start_repository_analysis( @@ -1821,8 +1831,21 @@ def _apply_change_set( for node in semantic_nodes: node.metadata.update(identity_metadata) + # Preserve the last usable semantic and repository-graph + # representations for updates which the loader quarantined. In + # copy-on-write generations this also keeps the already copied points + # intact. Real deletions and loaded updates still replace their prior + # representations normally. Repository facts intentionally continue + # to use the authoritative provider change set above. + replacement_paths = sorted({ + *active_deleted_paths, + *( + path for path in active_updated_paths + if path in documents_by_path + ), + }) old_path_records = self._records_for_values( - collection_name, branch, "path", paths + collection_name, branch, "path", replacement_paths ) old_semantic_records = [ record for record in old_path_records @@ -1838,7 +1861,10 @@ def _apply_change_set( if analysis is not None: impacted_old_nodes = self._records_for_values( - collection_name, branch, "architecture_paths", paths + collection_name, + branch, + "architecture_paths", + replacement_paths, ) impacted_old_nodes = [ record for record in impacted_old_nodes @@ -1848,7 +1874,10 @@ def _apply_change_set( architecture_group_from_payload(record.payload or {}) for record in impacted_old_nodes } - groups = old_groups | affected_architecture_groups(analysis, paths) + groups = old_groups | affected_architecture_groups( + analysis, + replacement_paths, + ) group_ids = {architecture_group_id(group) for group in groups} old_group_nodes = self._records_for_values( collection_name, branch, "architecture_group", group_ids @@ -1866,7 +1895,7 @@ def _apply_change_set( self.representation_fingerprint, groups=groups, ) - related_paths = set(paths) + related_paths = set(replacement_paths) for record in old_group_nodes: related_paths.update( (record.payload or {}).get("architecture_paths") or () diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py index ffce9bdc..3a909d0f 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py @@ -63,6 +63,13 @@ def read_repository_revision_preflight(*args, **kwargs): ) +def read_repository_generation_manifest_receipt(*args, **kwargs): + """Patchable boundary around bounded readable-alias verification.""" + return revision_preflight.read_repository_generation_manifest_receipt( + *args, **kwargs + ) + + def _config_int(config, name: str, default: int) -> int: value = getattr(config, name, default) if isinstance(value, bool) or not isinstance(value, (int, str)): @@ -162,6 +169,7 @@ def __init__(self, config: RAGConfig): self.qdrant_client = QdrantClient( url=config.qdrant_url, api_key=config.qdrant_api_key or None, + timeout=_config_int(config, "qdrant_timeout_seconds", 30), ) logger.info(f"Connected to Qdrant at {config.qdrant_url}") @@ -455,6 +463,7 @@ def publish_generation_aliases( branch: str, commit: str, collection_target: str, + generation_manifest_sha256: Optional[str] = None, publish_branch_alias: bool = True, publish_legacy_project_alias: bool = False, ) -> List[str]: @@ -486,11 +495,14 @@ def publish_generation_aliases( raise IncrementalIndexPreconditionError( "repository generation is unavailable for alias publication" ) - receipt = read_repository_revision_preflight( + receipt = read_repository_generation_manifest_receipt( self.qdrant_client, physical, + workspace, + project, branch, commit, + generation_manifest_sha256, ) if receipt is None or any(receipt.get(key) != value for key, value in ( ("workspace", workspace), @@ -907,11 +919,18 @@ def delete_branch( project: str, branch: str, collection_target: Optional[str] = None, + generation_revision: Optional[str] = None, + generation_manifest_sha256: Optional[str] = None, ) -> bool: """Delete all points for a specific branch from the project collection.""" if collection_target: return self.delete_collection_target( - workspace, project, branch, collection_target + workspace, + project, + branch, + collection_target, + generation_revision, + generation_manifest_sha256, ) with self._mutation_coordinator.acquire( workspace, @@ -921,7 +940,10 @@ def delete_branch( collection_name = self._get_project_collection_name(workspace, project) if not self._collection_manager.collection_exists(collection_name): if not self._collection_manager.alias_exists(collection_name): - logger.warning(f"Collection {collection_name} does not exist") + logger.info( + "Branch cleanup is already complete; collection %s does not exist", + collection_name, + ) return False lease.assert_owned() @@ -933,8 +955,14 @@ def delete_collection_target( project: str, branch: str, collection_target: str, + generation_revision: Optional[str], + generation_manifest_sha256: Optional[str], ) -> bool: - """Delete one exact generation after proving its tenant ownership.""" + """Delete one registry-selected generation after O(1) seal proof.""" + if not generation_revision or not generation_manifest_sha256: + raise IncrementalIndexPreconditionError( + "exact generation deletion requires its revision and manifest receipt" + ) with self._mutation_coordinator.acquire( workspace, project, "delete-generation" ) as lease: @@ -943,32 +971,18 @@ def delete_collection_target( ) if physical is None: return False - offset = None - point_count = 0 - while True: - points, offset = self.qdrant_client.scroll( - collection_name=physical, - limit=256, - offset=offset, - with_payload=["workspace", "project", "branch"], - with_vectors=False, - ) - point_count += len(points) - for point in points: - payload = point.payload or {} - if any(payload.get(key) != value for key, value in ( - ("workspace", workspace), - ("project", project), - ("branch", branch), - )): - raise IncrementalIndexPreconditionError( - "collection target does not belong to the requested tenant branch" - ) - if offset is None: - break - if point_count == 0: + receipt = read_repository_generation_manifest_receipt( + self.qdrant_client, + physical, + workspace, + project, + branch, + generation_revision, + generation_manifest_sha256, + ) + if receipt is None: raise IncrementalIndexPreconditionError( - "empty collection target cannot be ownership-verified" + "collection target does not match the registry generation receipt" ) lease.assert_owned() if self._collection_manager.alias_exists(collection_target): @@ -1064,7 +1078,10 @@ def pr_overlay_mutation( ) def close(self) -> None: - self._mutation_coordinator.close() + try: + self._mutation_coordinator.close() + finally: + self.qdrant_client.close() # Statistics diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py index 96825c99..672c4746 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py @@ -16,6 +16,7 @@ canonical_index_selection_policy, compute_index_selection_policy_sha256, compute_generation_members_digest, + generation_manifest_point_id, generation_manifest_content, is_sha256_hex, verified_generation_member, @@ -305,6 +306,119 @@ def _validate_generation_manifest( } +def read_repository_generation_manifest_receipt( + client, + collection_name: str, + workspace: str, + project: str, + branch: str, + commit: str, + generation_manifest_sha256: str | None = None, +): + """Validate one registry-selected immutable seal without scanning members. + + Readable aliases are non-authoritative operator conveniences. The Java + registry has already accepted the digest returned by the full generation + build, so alias repair only needs to prove that the selected physical + target still contains that exact coordinate-bound manifest. Exact review + preflight continues to verify every member and vector separately. + """ + if ( + generation_manifest_sha256 is not None + and not is_sha256_hex(generation_manifest_sha256) + ): + raise IncrementalIndexPreconditionError( + "repository generation manifest receipt is invalid" + ) + records = client.retrieve( + collection_name=collection_name, + ids=[generation_manifest_point_id(workspace, project, branch)], + with_payload=True, + with_vectors=False, + ) + if len(records) != 1: + return None + payload = records[0].payload or {} + stored_manifest_sha256 = payload.get("generation_manifest_sha256") + if any(payload.get(key) != value for key, value in ( + ("workspace", workspace), + ("project", project), + ("branch", branch), + ("commit", commit), + )): + return None + if ( + payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is not True + or payload.get("generation_schema") != GENERATION_SCHEMA + or payload.get("path") != GENERATION_MANIFEST_PATH + or not is_sha256_hex(stored_manifest_sha256) + or ( + generation_manifest_sha256 is not None + and stored_manifest_sha256 != generation_manifest_sha256 + ) + ): + return None + + # Recompute the manifest digest from its complete coordinate/membership + # metadata. This remains O(1): member contents are intentionally not read + # on the optional alias-repair path. + expected_count = payload.get("generation_member_count") + members_sha256 = payload.get("generation_members_sha256") + source_tree_sha256 = payload.get("source_tree_sha256") + include_patterns = payload.get("index_include_patterns") + exclude_patterns = payload.get("index_exclude_patterns") + selection_digest = payload.get("index_selection_policy_sha256") + if ( + type(expected_count) is not int + or expected_count < 1 + or not is_sha256_hex(members_sha256) + or not is_sha256_hex(source_tree_sha256) + or not isinstance(include_patterns, list) + or not isinstance(exclude_patterns, list) + or not is_sha256_hex(selection_digest) + ): + return None + try: + policy = canonical_index_selection_policy( + include_patterns, + exclude_patterns, + ) + except GenerationManifestError: + return None + if ( + include_patterns != policy["includePatterns"] + or exclude_patterns != policy["excludePatterns"] + or selection_digest != compute_index_selection_policy_sha256( + include_patterns, + exclude_patterns, + ) + ): + return None + content = generation_manifest_content( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=expected_count, + members_sha256=members_sha256, + source_tree_sha256=source_tree_sha256, + index_include_patterns=include_patterns, + index_exclude_patterns=exclude_patterns, + index_selection_policy_sha256=selection_digest, + ) + if hashlib.sha256(content.encode("utf-8")).hexdigest() != stored_manifest_sha256: + return None + if payload.get("text", payload.get("_node_content")) != content: + return None + return { + "workspace": workspace, + "project": project, + "branch": branch, + "commit": commit, + "generation_manifest_sha256": stored_manifest_sha256, + } + + def _require_unmixed_branch_revision( client, collection_name: str, diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py index 163b39aa..7b886e29 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py @@ -63,6 +63,10 @@ class RAGConfig(BaseModel): qdrant_url: str = Field(default_factory=lambda: os.getenv("QDRANT_URL", "http://qdrant:6333")) qdrant_api_key: str = Field(default_factory=lambda: os.getenv("QDRANT_API_KEY", "")) qdrant_collection_prefix: str = Field(default_factory=lambda: os.getenv("QDRANT_COLLECTION_PREFIX", "codecrow")) + qdrant_timeout_seconds: int = Field( + default_factory=lambda: int(os.getenv("QDRANT_TIMEOUT_SECONDS", "30")), + ge=1, + ) # Embedding provider selection: "ollama" (local) or "openrouter" (cloud) embedding_provider: str = Field(default_factory=lambda: os.getenv("EMBEDDING_PROVIDER", "ollama")) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py index 9926393a..9f23775e 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py @@ -1,8 +1,10 @@ import asyncio +from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor import json import logging import os import shutil +import threading import time from pathlib import Path from typing import Dict, Any, Optional @@ -29,6 +31,11 @@ def __init__(self, index_manager: RAGIndexManager): self.is_running = False self._redis: Optional[redis.Redis] = None self._task: Optional[asyncio.Task] = None + self._job_tasks: set[asyncio.Task] = set() + self._index_workers: set[ConcurrentFuture] = set() + self._index_workers_lock = threading.Lock() + self._queue_read_unavailable = False + self._event_delivery_unavailable = False max_concurrent = int(os.environ.get("MAX_CONCURRENT_RAG_JOBS", "2")) self._job_semaphore = asyncio.Semaphore(max_concurrent) self.heartbeat_seconds = max( @@ -63,6 +70,24 @@ async def stop(self): await self._task except asyncio.CancelledError: pass + + # A dequeued job is already admitted durable work. Let every admitted + # index call finish before API shutdown closes its embedding and + # Qdrant clients. Deployment supervisors may still enforce their + # outer termination deadline, but this service must not manufacture a + # client-use-after-close race itself. + active_jobs = tuple(self._job_tasks) + if active_jobs: + logger.info( + "Waiting for %s admitted RAG indexing jobs before shutdown", + len(active_jobs), + ) + await asyncio.gather(*active_jobs, return_exceptions=True) + + # A job task can be canceled independently while its synchronous + # indexing call is still running. Wait for that admitted durable work + # before the application closes shared embedding and Qdrant clients. + await self._wait_for_index_workers() if self._redis: await self._redis.aclose() @@ -83,21 +108,26 @@ async def _consume_loop(self): break result = await self._redis.brpop([self.job_queue_key], timeout=1) + self._record_queue_read_recovery() if not result: continue queue_name, payload_str = result logger.debug(f"Received raw RAG job payload from {queue_name}") - asyncio.create_task(self._handle_admitted_job(payload_str)) + job_task = asyncio.create_task( + self._handle_admitted_job(payload_str) + ) + self._job_tasks.add(job_task) + job_task.add_done_callback(self._job_tasks.discard) permit_acquired = False except asyncio.CancelledError: break except RedisTimeoutError as error: - logger.warning("Redis RAG queue read timed out; retrying: %s", error) + self._record_queue_read_failure(error) await asyncio.sleep(1) except Exception as e: - logger.error(f"Error in RAG Queue consume loop: {e}", exc_info=True) + self._record_queue_read_failure(e) await asyncio.sleep(2) finally: if permit_acquired: @@ -119,7 +149,11 @@ async def _handle_job(self, payload_str: str): """Process a single RAG job popped from the queue.""" job_id = "UNKNOWN" event_queue_key = None - indexing_future = None + index_worker_future = None + index_executor = None + progress_queue = None + progress_publisher_task = None + progress_delivery_available = False try: payload = json.loads(payload_str) @@ -158,63 +192,115 @@ async def _handle_job(self, payload_str: str): # index_manager.index_repository is synchronous, so we run it in an executor loop = asyncio.get_running_loop() progress_delivery_available = True - progress_futures = [] + # Progress is observability, so cap producer pressure and retain + # only the latest pending status when Redis publication is slower + # than indexing. + progress_queue = asyncio.Queue(maxsize=1) + + async def publish_progress_events() -> None: + while True: + event = await progress_queue.get() + try: + if event is None: + return + await self._publish_event(event_queue_key, event) + finally: + progress_queue.task_done() + + progress_publisher_task = asyncio.create_task( + publish_progress_events() + ) def publish_progress(event: Dict[str, Any]) -> None: - nonlocal progress_delivery_available if not progress_delivery_available: return payload = {"type": "status", **event} - future = asyncio.run_coroutine_threadsafe( - self._publish_event(event_queue_key, payload), - loop, - ) - # Never block the indexing executor waiting for its own event - # loop. Drain these publications before the terminal event so - # ordering is retained and delivery remains fail-open. - progress_futures.append(future) + try: + # Transfer plain data to the event-loop thread. Creating + # cross-loop coroutine futures here used to retain the + # default executor thread after a job completed and could + # hang process/test-loop shutdown. + loop.call_soon_threadsafe( + self._coalesce_progress_event, + progress_queue, + payload, + ) + except RuntimeError: + # The service is shutting down. Progress is auxiliary and + # must never keep the indexing worker alive. + return - indexing_future = loop.run_in_executor( - None, - lambda: self.index_manager.index_repository( - repo_path=request_dto.repo_path, - workspace=request_dto.workspace, - project=request_dto.project, - branch=request_dto.branch, - commit=request_dto.commit, - source_tree_sha256=request_dto.source_tree_sha256, - preserve_other_branches=request_dto.preserve_other_branches, - include_patterns=request_dto.include_patterns, - exclude_patterns=request_dto.exclude_patterns, - collection_target=request_dto.collection_target, - progress_callback=publish_progress, - ) + # Keep long-lived indexing work out of asyncio's process-wide + # default executor. A per-job executor has an explicit lifecycle, + # so completed progress jobs cannot leave idle threads attached to + # an event loop and block graceful shutdown. + index_executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="rag-index", ) - while True: - done, _ = await asyncio.wait( - {indexing_future}, - timeout=self.heartbeat_seconds, - return_when=asyncio.FIRST_COMPLETED, - ) - if done: - break - await self._publish_event(event_queue_key, { - "type": "status", - "state": "processing", - "message": "RAG indexing is still processing", - }) - result_obj = await indexing_future - for progress_future in progress_futures: + def run_index_job(): try: - await asyncio.wrap_future(progress_future) - except Exception as exception: - progress_delivery_available = False - logger.warning( - "Could not publish RAG progress for job %s: %s", - job_id, - exception, + return self.index_manager.index_repository( + repo_path=request_dto.repo_path, + workspace=request_dto.workspace, + project=request_dto.project, + branch=request_dto.branch, + commit=request_dto.commit, + source_tree_sha256=request_dto.source_tree_sha256, + preserve_other_branches=request_dto.preserve_other_branches, + include_patterns=request_dto.include_patterns, + exclude_patterns=request_dto.exclude_patterns, + collection_target=request_dto.collection_target, + progress_callback=publish_progress, ) + finally: + # Cleanup belongs to the synchronous worker lifecycle, so + # cancellation can never delete a checkout while that same + # worker is still indexing it. + if request_dto.cleanup_repo_path: + self._cleanup_owned_repository_path( + request_dto.repo_path + ) + + index_worker_future = index_executor.submit(run_index_job) + self._track_index_worker(index_worker_future) + next_heartbeat = loop.time() + self.heartbeat_seconds + while not index_worker_future.done(): + # Poll the thread-safe concurrent future rather than relying + # on a worker-thread callback to wake the asyncio selector. + # The latter can be retained until a long heartbeat timeout + # on some Python/event-loop combinations during shutdown. + await asyncio.sleep( + min(0.1, max(0.0, next_heartbeat - loop.time())) + ) + if ( + not index_worker_future.done() + and loop.time() >= next_heartbeat + ): + await self._publish_event(event_queue_key, { + "type": "status", + "state": "processing", + "message": "RAG indexing is still processing", + }) + next_heartbeat = loop.time() + self.heartbeat_seconds + + try: + result_obj = index_worker_future.result() + finally: + # The worker has returned, so all progress callbacks have + # already been enqueued ahead of this sentinel. Drain them + # before the final/error event to preserve event ordering. + progress_delivery_available = False + # A callback queued from the worker can be behind this task's + # timer wakeup in the loop-ready queue. Yield once so every + # already-scheduled transfer reaches the bounded queue before + # it is joined and closed. + await asyncio.sleep(0) + await progress_queue.join() + progress_queue.put_nowait(None) + await progress_publisher_task + progress_publisher_task = None # Serialize the IndexStats result to a dictionary result = result_obj.dict() if hasattr(result_obj, "dict") else result_obj.model_dump() @@ -237,28 +323,140 @@ def publish_progress(event: Dict[str, Any]) -> None: "message": f"Internal RAG pipeline error: {str(e)}" }) finally: - if "request_dto" in locals() and request_dto.cleanup_repo_path: - if indexing_future is None or indexing_future.done(): - await asyncio.to_thread( - self._cleanup_owned_repository_path, - request_dto.repo_path, - ) + progress_delivery_available = False + if progress_publisher_task is not None: + if index_worker_future is not None and not index_worker_future.done(): + progress_publisher_task.cancel() + try: + await progress_publisher_task + except asyncio.CancelledError: + pass else: - logger.warning( - "Preserving RAG job workspace because indexing is still active: %s", - request_dto.repo_path, - ) + await asyncio.sleep(0) + await progress_queue.join() + progress_queue.put_nowait(None) + await progress_publisher_task + worker_finished = ( + index_worker_future is None or index_worker_future.done() + ) + if ( + not worker_finished + and "request_dto" in locals() + and request_dto.cleanup_repo_path + ): + logger.info( + "Deferring owned RAG workspace cleanup until active " + "indexing returns: %s", + request_dto.repo_path, + ) + if index_executor is not None: + # wait=False still marks the executor closed; a live + # synchronous call exits its worker as soon as it returns. + # Completed calls are joined before the job task returns. + index_executor.shutdown( + wait=worker_finished, + cancel_futures=False, + ) + + def _track_index_worker(self, worker_future: ConcurrentFuture) -> None: + """Track synchronous indexing independently of its asyncio job task.""" + with self._index_workers_lock: + self._index_workers.add(worker_future) + worker_future.add_done_callback(self._retire_index_worker) + + def _retire_index_worker(self, worker_future: ConcurrentFuture) -> None: + with self._index_workers_lock: + self._index_workers.discard(worker_future) + + async def _wait_for_index_workers(self) -> None: + announced = False + while True: + with self._index_workers_lock: + active_count = len(self._index_workers) + if active_count == 0: + return + if not announced: + logger.info( + "Waiting for %s active RAG indexing workers before shutdown", + active_count, + ) + announced = True + await asyncio.sleep(0.01) + + @staticmethod + def _coalesce_progress_event( + progress_queue: asyncio.Queue, + event: Optional[Dict[str, Any]], + ) -> None: + """Keep at most the latest pending progress event or sentinel.""" + if progress_queue.full(): + try: + pending = progress_queue.get_nowait() + progress_queue.task_done() + if pending is None: + # Once the publisher is closing, late auxiliary progress + # must not replace its sentinel and strand the task. + progress_queue.put_nowait(None) + return + except asyncio.QueueEmpty: + pass + progress_queue.put_nowait(event) - async def _publish_event(self, key: str, event: Dict[str, Any]): + async def _publish_event(self, key: str, event: Dict[str, Any]) -> bool: """Publish an event back to the job's specific event list. LPUSH (Java uses rightPop).""" + if not self._redis: + return False try: - if not self._redis: - return event_json = json.dumps(event) - await self._redis.lpush(key, event_json) - await self._redis.expire(key, self.event_ttl_seconds) + # The event and its retention policy are one Redis transaction. + # A process/network failure cannot commit LPUSH while losing the + # EXPIRE and leave an orphaned event list indefinitely. + async with self._redis.pipeline(transaction=True) as pipeline: + pipeline.lpush(key, event_json) + pipeline.expire(key, self.event_ttl_seconds) + await pipeline.execute() + if self._event_delivery_unavailable: + logger.info("Redis RAG event delivery recovered") + self._event_delivery_unavailable = False + return True except Exception as e: - logger.error(f"Failed to publish event to {key}: {e}") + event_type = event.get("type", "unknown") + if not self._event_delivery_unavailable: + logger.warning( + "Redis RAG event delivery unavailable; events remain " + "fail-open (key=%s type=%s): %s", + key, + event_type, + e, + ) + self._event_delivery_unavailable = True + else: + logger.debug( + "Redis RAG event delivery still unavailable " + "(key=%s type=%s): %s", + key, + event_type, + e, + ) + return False + + def _record_queue_read_failure(self, error: BaseException) -> None: + if not self._queue_read_unavailable: + logger.warning( + "Redis RAG queue read unavailable; retrying: %s", + error, + ) + self._queue_read_unavailable = True + else: + logger.debug( + "Redis RAG queue read still unavailable; retrying: %s", + error, + ) + + def _record_queue_read_recovery(self) -> None: + if self._queue_read_unavailable: + logger.info("Redis RAG queue read recovered") + self._queue_read_unavailable = False @staticmethod def _cleanup_owned_repository_path(repo_path: str) -> None: diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py index a2c8b192..bc4b6a33 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py @@ -41,9 +41,15 @@ def __init__(self, config: RAGConfig, plugin_catalog=None): index_representation_fingerprint(config) ) self._observed_branch_cache: set[tuple[str, str]] = set() + qdrant_timeout = getattr(config, "qdrant_timeout_seconds", 30) + if isinstance(qdrant_timeout, bool) or not isinstance( + qdrant_timeout, int + ): + qdrant_timeout = 30 self.qdrant_client = QdrantClient( url=config.qdrant_url, api_key=config.qdrant_api_key or None, + timeout=qdrant_timeout, ) embed_info = get_embedding_model_info(config) @@ -82,19 +88,30 @@ def _accept_stored_points(self, points: List[Any]) -> List[Any]: def _collection_or_alias_exists(self, name: str) -> bool: """Check if a collection or alias with the given name exists.""" try: - collections = [c.name for c in self.qdrant_client.get_collections().collections] + collections = [ + c.name for c in self.qdrant_client.get_collections().collections + ] if name in collections: return True aliases = self.qdrant_client.get_aliases() - if any(a.alias_name == name for a in aliases.aliases): - return True - - return False - except Exception as e: - logger.warning(f"Error checking collection/alias existence: {e}") + return any(a.alias_name == name for a in aliases.aliases) + except Exception as exception: + # Exact callers convert absence to their revision-bound 409; legacy + # optional callers return empty context. The HTTP owner records the + # one contextual degraded diagnostic, so this shared probe stays + # below warning level. + logger.debug( + "Collection/alias probe unavailable for %s: %s", + name, + exception, + ) return False + def close(self) -> None: + """Release the query-side Qdrant transport.""" + self.qdrant_client.close() + def _get_project_collection_name(self, workspace: str, project: str) -> str: """Generate the legacy shared collection name for a project.""" namespace = make_project_namespace(workspace, project) diff --git a/python-ecosystem/rag-pipeline/tests/test_api_app.py b/python-ecosystem/rag-pipeline/tests/test_api_app.py index 9c51a015..0aa54a7e 100644 --- a/python-ecosystem/rag-pipeline/tests/test_api_app.py +++ b/python-ecosystem/rag-pipeline/tests/test_api_app.py @@ -1,9 +1,11 @@ """ Tests for rag_pipeline.api.api — App creation, middleware, lifespan. """ +import asyncio import logging import os import pytest +from types import SimpleNamespace from unittest.mock import patch, MagicMock, AsyncMock @@ -108,6 +110,62 @@ def test_app_exists(self): assert app is not None assert app.title == "CodeCrow RAG API" + @pytest.mark.asyncio + async def test_shutdown_drains_http_index_workers_before_clients_close(self): + import rag_pipeline.api.api as api_module + + order = [] + manager = MagicMock() + manager.embed_model.close.side_effect = lambda: order.append( + "manager-embed-close" + ) + manager.close.side_effect = lambda: order.append("manager-close") + query_service = MagicMock() + query_service.embed_model.close.side_effect = lambda: order.append( + "query-embed-close" + ) + query_service.close.side_effect = lambda: order.append("query-close") + queue_consumer = MagicMock() + queue_consumer.start = AsyncMock() + queue_consumer.stop = AsyncMock() + drain_workers = AsyncMock(side_effect=lambda: order.append("drain")) + test_app = SimpleNamespace(state=SimpleNamespace()) + + with ( + patch.object(api_module, "RAGConfig", return_value=MagicMock()), + patch.object( + api_module, + "RAGIndexManager", + return_value=manager, + ), + patch.object( + api_module, + "RAGQueryService", + return_value=query_service, + ), + patch( + "rag_pipeline.server.rag_queue_consumer.RAGQueueConsumer", + return_value=queue_consumer, + ), + patch( + "rag_pipeline.api.routers.index." + "drain_index_repository_stream_workers", + drain_workers, + ), + ): + async with api_module.lifespan(test_app): + pass + + queue_consumer.stop.assert_awaited_once() + drain_workers.assert_awaited_once() + assert order == [ + "drain", + "manager-embed-close", + "query-embed-close", + "query-close", + "manager-close", + ] + class TestPendingCollectionJanitor: @@ -137,3 +195,42 @@ def test_malformed_interval_falls_back_to_default(self, caplog): assert _pending_janitor_interval_seconds() == 3600 assert "Invalid RAG_PENDING_JANITOR_INTERVAL_SECONDS" in caplog.text + + @pytest.mark.asyncio + async def test_outage_logs_once_and_reports_recovery(self, caplog): + from rag_pipeline.api.api import _pending_collection_janitor + + cleanup_attempt = AsyncMock(side_effect=[ + RuntimeError("qdrant unavailable"), + RuntimeError("qdrant unavailable"), + 0, + asyncio.CancelledError(), + ]) + with ( + patch( + "rag_pipeline.api.api.asyncio.to_thread", + cleanup_attempt, + ), + patch( + "rag_pipeline.api.api.asyncio.sleep", + AsyncMock(return_value=None), + ), + caplog.at_level(logging.DEBUG), + ): + with pytest.raises(asyncio.CancelledError): + await _pending_collection_janitor(MagicMock()) + + janitor_records = [ + record + for record in caplog.records + if "Pending collection janitor" in record.getMessage() + ] + assert sum( + record.levelno == logging.WARNING for record in janitor_records + ) == 1 + assert not any( + record.levelno >= logging.ERROR for record in janitor_records + ) + assert any( + "recovered" in record.getMessage() for record in janitor_records + ) diff --git a/python-ecosystem/rag-pipeline/tests/test_api_models.py b/python-ecosystem/rag-pipeline/tests/test_api_models.py index 330ecdcf..bddc31d5 100644 --- a/python-ecosystem/rag-pipeline/tests/test_api_models.py +++ b/python-ecosystem/rag-pipeline/tests/test_api_models.py @@ -22,6 +22,7 @@ ApplyChangesRequest, UpdateFilesRequest, CleanupStaleBranchesRequest, + GenerationAliasPublicationRequest, VectorGraphRequest, VectorNodeRequest, ) @@ -41,6 +42,7 @@ def test_valid_path(self): assert req.workspace == "ws" assert req.preserve_other_branches is False assert req.cleanup_repo_path is False + assert req.transfer_repo_ownership is False @patch.dict(os.environ, {"ALLOWED_REPO_ROOT": "/tmp"}) def test_other_branch_preservation_requires_explicit_opt_in(self): @@ -66,6 +68,18 @@ def test_queue_consumer_cleanup_requires_explicit_opt_in(self): ) assert req.cleanup_repo_path is True + @patch.dict(os.environ, {"ALLOWED_REPO_ROOT": "/tmp"}) + def test_stream_repository_ownership_requires_explicit_opt_in(self): + req = IndexRequest( + repo_path="/tmp/codecrow-rag-branch-generation-owned", + workspace="ws", + project="proj", + branch="main", + commit="abc123", + transfer_repo_ownership=True, + ) + assert req.transfer_repo_ownership is True + @patch.dict(os.environ, {"ALLOWED_REPO_ROOT": "/tmp"}) def test_path_traversal_rejected(self): with pytest.raises(ValueError, match="Path must be under"): @@ -294,6 +308,39 @@ def test_construction(self): assert req.branch == "feature/old" +class TestGenerationAliasPublicationRequest: + + def test_accepts_legacy_caller_without_registry_manifest_receipt(self): + legacy_request = GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="target", + ) + assert legacy_request.generation_manifest_sha256 is None + + request = GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="target", + generation_manifest_sha256="b" * 64, + ) + assert request.generation_manifest_sha256 == "b" * 64 + + with pytest.raises(ValueError): + GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="target", + generation_manifest_sha256="invalid", + ) + + class TestCleanupStaleBranches: def test_requires_authoritative_branches(self): diff --git a/python-ecosystem/rag-pipeline/tests/test_config.py b/python-ecosystem/rag-pipeline/tests/test_config.py index 0164d8a0..b3d4be90 100644 --- a/python-ecosystem/rag-pipeline/tests/test_config.py +++ b/python-ecosystem/rag-pipeline/tests/test_config.py @@ -64,6 +64,11 @@ def test_default_values(self): assert config.retrieval_top_k == 10 assert config.similarity_threshold == 0.7 assert config.max_file_size_bytes == 1024 * 1024 + assert config.qdrant_timeout_seconds == 30 + + def test_qdrant_timeout_is_configurable(self): + with patch.dict(os.environ, {"QDRANT_TIMEOUT_SECONDS": "45"}): + assert RAGConfig().qdrant_timeout_seconds == 45 def test_auto_detect_embedding_dim_ollama(self): config = RAGConfig(embedding_provider="ollama", ollama_model="all-minilm", embedding_dim=0) diff --git a/python-ecosystem/rag-pipeline/tests/test_coordination.py b/python-ecosystem/rag-pipeline/tests/test_coordination.py index 2d560346..816b460d 100644 --- a/python-ecosystem/rag-pipeline/tests/test_coordination.py +++ b/python-ecosystem/rag-pipeline/tests/test_coordination.py @@ -1,4 +1,5 @@ from types import SimpleNamespace +import logging from unittest.mock import MagicMock, patch import pytest @@ -137,6 +138,40 @@ def test_openrouter_capacity_limiter_falls_back_locally_when_redis_is_unavailabl pool._local.release() +def test_openrouter_capacity_outage_logs_only_transitions(caplog): + pool = RedisPermitPool( + "redis://unused", + 2, + permit_seconds=60, + acquire_timeout_seconds=0.1, + ) + pool._client = MagicMock() + pool._client.eval.side_effect = [ + RuntimeError("redis unavailable"), + RuntimeError("redis unavailable"), + True, + ] + + with caplog.at_level(logging.DEBUG): + pool._disabled_until = 0 + with pool.permit(): + pass + pool._disabled_until = 0 + with pool.permit(): + pass + pool._disabled_until = 0 + with pool.permit(): + pass + + warnings = [ + record for record in caplog.records + if record.levelno == logging.WARNING + and "capacity limit unavailable" in record.getMessage() + ] + assert len(warnings) == 1 + assert "capacity limit recovered" in caplog.text + + def test_pending_janitor_keeps_live_and_aliased_collections_and_deletes_expired(): client = MagicMock() client.get_aliases.return_value.aliases = [ @@ -166,3 +201,26 @@ def test_pending_janitor_keeps_live_and_aliased_collections_and_deletes_expired( client.delete_collection.assert_called_once_with( "base_pending_1000000000_eeeeeeee_ffffffff" ) + + +def test_pending_janitor_propagates_alias_read_failure_to_lifecycle_owner(): + client = MagicMock() + client.get_aliases.side_effect = RuntimeError("qdrant unavailable") + manager = CollectionManager(client, 3) + + with pytest.raises(RuntimeError, match="qdrant unavailable"): + manager.cleanup_expired_pending_collections( + is_operation_active=lambda _token: False, + min_age_seconds=300, + ) + + client.get_collections.assert_not_called() + client.delete_collection.assert_not_called() + + +def test_pending_janitor_operation_check_propagates_redis_failure(): + coordinator = _coordinator() + coordinator._client.exists.side_effect = RuntimeError("redis unavailable") + + with pytest.raises(RuntimeError, match="redis unavailable"): + coordinator.is_operation_active("aaaaaaaa") diff --git a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py index f7c22c9e..f57d08f9 100644 --- a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py +++ b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py @@ -17,6 +17,7 @@ build_generation_manifest_node, collect_generation_members, compute_generation_members_digest, + generation_manifest_point_id, ) from rag_pipeline.core.index_manager.collection_manager import CollectionManager from rag_pipeline.core.index_manager.manager import RAGIndexManager @@ -123,6 +124,16 @@ def _sealed_source(client, point_ops, collection): ) == (1, 0) +def _manifest_digest(client, collection): + records = client.retrieve( + collection_name=collection, + ids=[generation_manifest_point_id("ws", "project", "develop")], + with_payload=True, + with_vectors=False, + ) + return records[0].payload["generation_manifest_sha256"] + + def test_copy_on_write_advance_keeps_source_and_seals_target_revision(tmp_path): client = QdrantClient(":memory:") source_physical = "develop_source_physical" @@ -275,10 +286,57 @@ def test_exact_generation_delete_verifies_tenant_coordinates(): with pytest.raises( IncrementalIndexPreconditionError, - match="does not belong", + match="does not match the registry generation receipt", ): manager.delete_collection_target( - "ws", "project", "develop", "foreign_generation" + "ws", + "project", + "develop", + "foreign_generation", + SOURCE_COMMIT, + "1" * 64, ) assert manager._collection_manager.collection_exists("foreign_generation") + + +def test_exact_generation_delete_allows_tenant_owned_pr_overlay_points(): + client = QdrantClient(":memory:") + client.create_collection( + collection_name="review_generation", + vectors_config=VectorParams(size=4, distance=Distance.COSINE), + ) + point_ops = PointOperations(client, _Embedding(), embedding_dim=4) + _sealed_source(client, point_ops, "review_generation") + point_ops.process_and_upsert_chunks( + [TextNode(text="pull request change", metadata={ + "workspace": "ws", + "project": "project", + "branch": "feature/review", + "path": "src/Changed.java", + "pr": True, + "pr_number": 42, + })], + "review_generation", + "ws", + "project", + "feature/review", + ) + manifest_digest = _manifest_digest(client, "review_generation") + client.scroll = MagicMock( + side_effect=AssertionError("exact generation deletion must remain O(1)") + ) + manager = object.__new__(RAGIndexManager) + manager.qdrant_client = client + manager._collection_manager = CollectionManager(client, 4) + manager._mutation_coordinator = _Coordinator() + + assert manager.delete_collection_target( + "ws", + "project", + "develop", + "review_generation", + SOURCE_COMMIT, + manifest_digest, + ) is True + assert not manager._collection_manager.collection_exists("review_generation") diff --git a/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py b/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py index 7473cdff..7ff852d8 100644 --- a/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py +++ b/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py @@ -7,7 +7,15 @@ from qdrant_client import QdrantClient from qdrant_client.models import Distance, VectorParams -from codecrow_plugins import FileArtifact, PluginRuntime, ProjectSelector, build_repository_facts +from codecrow_plugins import ( + FileArtifact, + FileDisposition, + PluginRuntime, + ProjectSelector, + RepositoryAnalysis, + RepositorySnapshot, + build_repository_facts, +) from codecrow_plugins.bootstrap import discover_builtin_plugins from rag_pipeline.core.index_manager.collection_manager import CollectionManager from rag_pipeline.core.index_manager.indexer import FileOperations, RepositoryIndexer @@ -686,6 +694,100 @@ def test_incremental_update_rejects_missing_repository_analysis_snapshots( point_ops.prepare_chunks_for_embedding.assert_not_called() +def test_excluded_composer_lock_update_is_quarantined_not_deleted( + tmp_path, + monkeypatch, + caplog, +): + """A present excluded lockfile must not become a plugin tombstone.""" + _write_repository(tmp_path, "Acme\\Checkout\\Model\\Cart") + lock_path = tmp_path / "composer.lock" + lock_path.write_text('{"packages": []}', encoding="utf-8") + catalog = discover_builtin_plugins() + selector = ProjectSelector(catalog.registry) + facts = _repository_facts(tmp_path, "base", catalog) + capabilities = selector.select(facts) + implementation_fingerprint = catalog.implementation_fingerprint( + capabilities.repository_plugins + ) + + from rag_pipeline.core import repository_overlay + + monkeypatch.setattr( + repository_overlay, + "load_repository_facts", + lambda *_args: ( + facts, + capabilities.repository_plugins, + capabilities.fingerprint, + capabilities.descriptor_fingerprint, + implementation_fingerprint, + ), + ) + monkeypatch.setattr( + repository_overlay, + "load_repository_snapshots", + lambda *_args: ( + (RepositorySnapshot("magento", "state", "{}"),), + capabilities.repository_plugins, + capabilities.fingerprint, + capabilities.descriptor_fingerprint, + implementation_fingerprint, + ), + ) + monkeypatch.setattr( + "rag_pipeline.core.index_manager.indexer.observe_branch_representation", + lambda *_args, **_kwargs: None, + ) + + handle = MagicMock() + handle.finish.return_value = (RepositoryAnalysis(), ()) + runtime = MagicMock() + runtime.repository_analysis_plugins.return_value = ("magento",) + runtime.file_disposition.return_value = FileDisposition.FULL + runtime.start_repository_analysis.return_value = handle + client = MagicMock() + client.scroll.return_value = ([], None) + operations = FileOperations( + client, + MagicMock(), + MagicMock(), + MagicMock(), + MagicMock(), + DocumentLoader(SimpleNamespace( + excluded_patterns=("*.lock",), + max_file_size_bytes=1_000_000, + )), + plugin_catalog=catalog, + plugin_runtime=runtime, + plugin_selector=selector, + ) + operations._records_for_values = MagicMock(return_value=[]) + operations._replace_points = MagicMock(return_value=1) + + operations.apply_changes( + ["composer.lock"], + [], + str(tmp_path), + "ws", + "project", + "main", + "changed", + "repository", + ) + + handle.ingest.assert_called_once_with(()) + assert all( + "composer.lock" not in set(call.args[3]) + for call in operations._records_for_values.call_args_list + ) + operations._replace_points.assert_called_once() + assert not any( + record.levelno >= 30 and "composer.lock" in record.getMessage() + for record in caplog.records + ) + + def test_generic_change_set_accepts_older_build_and_updates_one_generation( tmp_path, ): diff --git a/python-ecosystem/rag-pipeline/tests/test_index_manager.py b/python-ecosystem/rag-pipeline/tests/test_index_manager.py index aba24d6d..832f3085 100644 --- a/python-ecosystem/rag-pipeline/tests/test_index_manager.py +++ b/python-ecosystem/rag-pipeline/tests/test_index_manager.py @@ -5,7 +5,10 @@ import pytest import uuid from httpx import Headers -from qdrant_client.http.exceptions import UnexpectedResponse +from qdrant_client.http.exceptions import ( + ResponseHandlingException, + UnexpectedResponse, +) from unittest.mock import patch, MagicMock, PropertyMock from datetime import datetime @@ -126,20 +129,103 @@ def test_atomic_assign_aliases_replaces_all_requested_aliases_in_one_call(self): ] assert len(operations) == 3 - def test_payload_index_failure_does_not_skip_remaining_indexes(self): + def test_payload_index_failure_does_not_skip_remaining_indexes( + self, + caplog, + ): cm = self._make() cm.client.create_payload_index.side_effect = [ RuntimeError("path index already exists"), - True, - True, - True, - True, - True, + *([True] * 12), ] cm._ensure_payload_indexes("test_coll") - assert cm.client.create_payload_index.call_count == 6 + assert cm.client.create_payload_index.call_count == 13 + warnings = [ + record for record in caplog.records + if record.levelname == "WARNING" + ] + assert len(warnings) == 1 + assert "failed for 1 field(s)" in warnings[0].getMessage() + + def test_payload_index_failures_emit_one_aggregate_warning(self, caplog): + cm = self._make() + cm.client.create_payload_index.side_effect = RuntimeError("unsupported") + + assert cm._ensure_payload_indexes("test_coll") is False + + assert cm.client.create_payload_index.call_count == 13 + warnings = [ + record for record in caplog.records + if record.levelname == "WARNING" + ] + assert len(warnings) == 1 + assert "failed for 13 field(s)" in warnings[0].getMessage() + + def test_payload_index_repair_defers_when_schema_inspection_times_out(self): + cm = self._make() + cm.client.get_collection.side_effect = ResponseHandlingException( + TimeoutError("timed out") + ) + + assert cm._ensure_payload_indexes("test_coll") is False + cm.client.create_payload_index.assert_not_called() + + def test_payload_index_repair_stops_after_transport_failure(self): + cm = self._make() + cm.client.create_payload_index.side_effect = ( + ResponseHandlingException(TimeoutError("timed out")) + ) + + assert cm._ensure_payload_indexes("test_coll") is False + assert cm.client.create_payload_index.call_count == 1 + + def test_existing_collection_repairs_payload_indexes_once(self): + cm = self._make() + collection = MagicMock(name="test_coll") + collection.name = "test_coll" + cm.client.get_collections.return_value.collections = [collection] + cm.alias_exists = MagicMock(return_value=False) + + cm.ensure_collection_exists("test_coll") + cm.ensure_collection_exists("test_coll") + + assert cm.client.create_payload_index.call_count == 13 + + def test_payload_index_repair_only_creates_missing_schemas(self): + from qdrant_client.models import PayloadSchemaType + + cm = self._make() + existing = MagicMock() + existing.data_type = PayloadSchemaType.KEYWORD + cm.client.get_collection.return_value.payload_schema = { + "workspace": existing, + } + + cm._ensure_payload_indexes("test_coll") + + fields = { + call.kwargs["field_name"] + for call in cm.client.create_payload_index.call_args_list + } + assert "workspace" not in fields + assert "project" in fields + + def test_pr_payload_indexes_use_filter_compatible_schemas(self): + from qdrant_client.models import PayloadSchemaType + + cm = self._make() + cm._ensure_payload_indexes("test_coll") + schemas = { + call.kwargs["field_name"]: call.kwargs["field_schema"] + for call in cm.client.create_payload_index.call_args_list + } + + assert schemas["workspace"] is PayloadSchemaType.KEYWORD + assert schemas["project"] is PayloadSchemaType.KEYWORD + assert schemas["pr"] is PayloadSchemaType.BOOL + assert schemas["pr_number"] is PayloadSchemaType.INTEGER def test_delete_collection(self): cm = self._make() @@ -147,6 +233,20 @@ def test_delete_collection(self): assert result is True cm.client.delete_collection.assert_called_once_with("test_coll") + def test_delete_then_recreate_repairs_payload_indexes_again(self): + cm = self._make() + cm._payload_indexes_ensured.add("test_coll") + + assert cm.delete_collection("test_coll") is True + + collection = MagicMock() + collection.name = "test_coll" + cm.client.get_collections.return_value.collections = [collection] + cm.alias_exists = MagicMock(return_value=False) + cm.ensure_collection_exists("test_coll") + + assert cm.client.create_payload_index.call_count == 13 + def test_delete_collection_failure(self): cm = self._make() cm.client.delete_collection.side_effect = Exception("fail") @@ -399,6 +499,23 @@ def test_init(self, MockQdrant, mock_info, mock_create): mgr = RAGIndexManager(self._mock_config()) assert mgr.qdrant_client is not None assert mgr.embed_model is mock_embed + MockQdrant.assert_called_once_with( + url="http://localhost:6333", + api_key=None, + timeout=30, + ) + + def test_close_releases_coordinator_and_qdrant_client(self): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + manager._mutation_coordinator = MagicMock() + manager.qdrant_client = MagicMock() + + manager.close() + + manager._mutation_coordinator.close.assert_called_once_with() + manager.qdrant_client.close.assert_called_once_with() @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") diff --git a/python-ecosystem/rag-pipeline/tests/test_rag_queue_consumer.py b/python-ecosystem/rag-pipeline/tests/test_rag_queue_consumer.py index c4152ac2..6469cee4 100644 --- a/python-ecosystem/rag-pipeline/tests/test_rag_queue_consumer.py +++ b/python-ecosystem/rag-pipeline/tests/test_rag_queue_consumer.py @@ -1,6 +1,8 @@ import asyncio import json -from unittest.mock import AsyncMock, Mock, patch +import logging +import threading +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -12,16 +14,37 @@ def model_dump(self): return {"document_count": 1, "chunk_count": 1} +def _redis_with_transactional_pipeline(): + redis_client = AsyncMock() + pipeline = MagicMock() + pipeline.lpush.return_value = pipeline + pipeline.expire.return_value = pipeline + pipeline.execute = AsyncMock(return_value=[1, True]) + pipeline.__aenter__ = AsyncMock(return_value=pipeline) + pipeline.__aexit__ = AsyncMock(return_value=None) + redis_client.pipeline = Mock(return_value=pipeline) + return redis_client, pipeline + + @pytest.mark.asyncio async def test_active_indexing_emits_heartbeats_and_refreshes_event_ttl(tmp_path): owned_repo = tmp_path / "codecrow-rag-owned" owned_repo.mkdir() manager = Mock() + indexing_started = threading.Event() + finish_indexing = threading.Event() + + def index_until_released(**_kwargs): + indexing_started.set() + finish_indexing.wait(timeout=2) + return _Stats() + + manager.index_repository.side_effect = index_until_released consumer = RAGQueueConsumer(manager) consumer.heartbeat_seconds = 0.01 consumer.event_ttl_seconds = 123 - consumer._redis = AsyncMock() + consumer._redis, event_pipeline = _redis_with_transactional_pipeline() payload = json.dumps({ "job_id": "job-1", @@ -35,22 +58,21 @@ async def test_active_indexing_emits_heartbeats_and_refreshes_event_ttl(tmp_path }, }) - loop = asyncio.get_running_loop() - indexing_future = loop.create_future() - with patch.object(loop, "run_in_executor", return_value=indexing_future): - task = asyncio.create_task(consumer._handle_job(payload)) - await asyncio.sleep(0.04) - indexing_future.set_result(_Stats()) - await task + task = asyncio.create_task(consumer._handle_job(payload)) + while not indexing_started.is_set(): + await asyncio.sleep(0) + await asyncio.sleep(0.04) + finish_indexing.set() + await task events = [ json.loads(call.args[1]) - for call in consumer._redis.lpush.await_args_list + for call in event_pipeline.lpush.call_args_list ] assert any(event.get("state") == "processing" for event in events) assert events[-1]["type"] == "final" - assert consumer._redis.expire.await_count == len(events) - consumer._redis.expire.assert_awaited_with( + assert event_pipeline.expire.call_count == len(events) + event_pipeline.expire.assert_called_with( "codecrow:analysis:events:job-1", 123 ) @@ -73,7 +95,7 @@ def index_with_progress(**kwargs): manager.index_repository.side_effect = index_with_progress consumer = RAGQueueConsumer(manager) - consumer._redis = AsyncMock() + consumer._redis, event_pipeline = _redis_with_transactional_pipeline() payload = json.dumps({ "job_id": "job-progress", "request": { @@ -88,9 +110,14 @@ def index_with_progress(**kwargs): await consumer._handle_job(payload) + assert not any( + thread.is_alive() and thread.name.startswith("rag-index") + for thread in threading.enumerate() + ) + events = [ json.loads(call.args[1]) - for call in consumer._redis.lpush.await_args_list + for call in event_pipeline.lpush.call_args_list ] assert any( event.get("stage") == "indexing" and event.get("progress") == 40 @@ -108,7 +135,7 @@ async def test_consumer_removes_only_explicitly_owned_workspace(tmp_path): manager = Mock() manager.index_repository.return_value = _Stats() consumer = RAGQueueConsumer(manager) - consumer._redis = AsyncMock() + consumer._redis, _ = _redis_with_transactional_pipeline() payload = json.dumps({ "job_id": "job-2", @@ -122,20 +149,7 @@ async def test_consumer_removes_only_explicitly_owned_workspace(tmp_path): }, }) - loop = asyncio.get_running_loop() - - def run_inline(_executor, function): - result = loop.create_future() - try: - result.set_result(function()) - except Exception as error: - result.set_exception(error) - return result - - with ( - patch.object(loop, "run_in_executor", side_effect=run_inline), - patch.dict("os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)}), - ): + with patch.dict("os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)}): await consumer._handle_job(payload) assert not owned_repo.exists() @@ -151,6 +165,199 @@ def test_cleanup_refuses_paths_outside_owned_temp_namespace(tmp_path): assert unrelated.exists() +@pytest.mark.asyncio +async def test_cancelled_job_retires_executor_after_worker_returns(tmp_path): + owned_repo = tmp_path / "codecrow-rag-owned" + owned_repo.mkdir() + (owned_repo / "source.py").write_text("value = 1", encoding="utf-8") + started = threading.Event() + release = threading.Event() + + manager = Mock() + + def index_until_released(**_kwargs): + started.set() + release.wait(timeout=2) + return _Stats() + + manager.index_repository.side_effect = index_until_released + consumer = RAGQueueConsumer(manager) + consumer._redis, _ = _redis_with_transactional_pipeline() + payload = json.dumps({ + "job_id": "job-cancelled", + "request": { + "repo_path": str(owned_repo), + "workspace": "ws", + "project": "project", + "branch": "main", + "commit": "abc123", + "cleanup_repo_path": True, + }, + }) + + with patch.dict("os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)}): + task = asyncio.create_task(consumer._handle_job(payload)) + while not started.is_set(): + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert owned_repo.exists() + + stop_task = asyncio.create_task(consumer.stop()) + await asyncio.sleep(0) + assert not stop_task.done() + consumer._redis.aclose.assert_not_awaited() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + + assert not any( + thread.is_alive() and thread.name.startswith("rag-index") + for thread in threading.enumerate() + ) + assert not owned_repo.exists() + + +@pytest.mark.asyncio +async def test_stop_waits_for_admitted_job_before_closing_redis(tmp_path): + owned_repo = tmp_path / "codecrow-rag-owned" + owned_repo.mkdir() + started = threading.Event() + release = threading.Event() + + manager = Mock() + + def index_until_released(**_kwargs): + started.set() + release.wait(timeout=2) + return _Stats() + + manager.index_repository.side_effect = index_until_released + consumer = RAGQueueConsumer(manager) + consumer._redis, _ = _redis_with_transactional_pipeline() + payload = json.dumps({ + "job_id": "job-shutdown", + "request": { + "repo_path": str(owned_repo), + "workspace": "ws", + "project": "project", + "branch": "main", + "commit": "abc123", + "cleanup_repo_path": False, + }, + }) + + job_task = asyncio.create_task(consumer._handle_job(payload)) + consumer._job_tasks.add(job_task) + job_task.add_done_callback(consumer._job_tasks.discard) + while not started.is_set(): + await asyncio.sleep(0) + + stop_task = asyncio.create_task(consumer.stop()) + await asyncio.sleep(0) + assert not stop_task.done() + consumer._redis.aclose.assert_not_awaited() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + + assert job_task.done() + consumer._redis.aclose.assert_awaited_once_with() + assert not any( + thread.is_alive() and thread.name.startswith("rag-index") + for thread in threading.enumerate() + ) + + +@pytest.mark.asyncio +async def test_progress_buffer_coalesces_to_latest_event(): + queue = asyncio.Queue(maxsize=1) + + RAGQueueConsumer._coalesce_progress_event(queue, {"progress": 10}) + RAGQueueConsumer._coalesce_progress_event(queue, {"progress": 20}) + + assert queue.qsize() == 1 + assert await queue.get() == {"progress": 20} + queue.task_done() + await queue.join() + + +@pytest.mark.asyncio +async def test_late_progress_does_not_replace_publisher_sentinel(): + queue = asyncio.Queue(maxsize=1) + queue.put_nowait(None) + + RAGQueueConsumer._coalesce_progress_event(queue, {"progress": 100}) + + assert await queue.get() is None + queue.task_done() + await queue.join() + + +@pytest.mark.asyncio +async def test_event_delivery_outage_logs_one_warning_until_recovery(caplog): + consumer = RAGQueueConsumer(Mock()) + consumer._redis, event_pipeline = _redis_with_transactional_pipeline() + event_pipeline.execute.side_effect = [ + RuntimeError("redis unavailable"), + RuntimeError("redis unavailable"), + [1, True], + ] + + with caplog.at_level(logging.DEBUG): + assert not await consumer._publish_event("events", {"type": "status"}) + assert not await consumer._publish_event("events", {"type": "status"}) + assert await consumer._publish_event("events", {"type": "final"}) + + outage_records = [ + record + for record in caplog.records + if "Redis RAG event delivery" in record.getMessage() + ] + assert sum(record.levelno == logging.WARNING for record in outage_records) == 1 + assert not any(record.levelno >= logging.ERROR for record in outage_records) + assert any("recovered" in record.getMessage() for record in outage_records) + + +@pytest.mark.asyncio +async def test_event_and_ttl_are_enqueued_in_one_redis_transaction(): + consumer = RAGQueueConsumer(Mock()) + consumer.event_ttl_seconds = 321 + consumer._redis, event_pipeline = _redis_with_transactional_pipeline() + + assert await consumer._publish_event( + "events", {"type": "final", "result": {"ok": True}} + ) + + consumer._redis.pipeline.assert_called_once_with(transaction=True) + event_pipeline.lpush.assert_called_once() + assert json.loads(event_pipeline.lpush.call_args.args[1]) == { + "type": "final", + "result": {"ok": True}, + } + event_pipeline.expire.assert_called_once_with("events", 321) + event_pipeline.execute.assert_awaited_once_with() + + +def test_queue_read_outage_logs_one_warning_until_recovery(caplog): + consumer = RAGQueueConsumer(Mock()) + + with caplog.at_level(logging.DEBUG): + consumer._record_queue_read_failure(RuntimeError("redis unavailable")) + consumer._record_queue_read_failure(RuntimeError("redis unavailable")) + consumer._record_queue_read_recovery() + + outage_records = [ + record + for record in caplog.records + if "Redis RAG queue read" in record.getMessage() + ] + assert sum(record.levelno == logging.WARNING for record in outage_records) == 1 + assert not any(record.levelno >= logging.ERROR for record in outage_records) + assert any("recovered" in record.getMessage() for record in outage_records) + + @pytest.mark.asyncio async def test_worker_capacity_is_reserved_before_rag_job_is_dequeued(): consumer = RAGQueueConsumer(Mock()) diff --git a/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py b/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py index 2283e1fd..8fbbc569 100644 --- a/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py +++ b/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py @@ -13,6 +13,7 @@ ) from rag_pipeline.core.generation_manifest import ( + GENERATION_MANIFEST_PATH, GENERATION_SCHEMA, GenerationManifestError, build_generation_manifest_node, @@ -20,14 +21,17 @@ collect_generation_members, compute_generation_member_digest, compute_generation_members_digest, + generation_manifest_point_id, seal_generation_members, ) from rag_pipeline.core.index_representation import ( INDEX_REPRESENTATION_PAYLOAD_KEY, ) from rag_pipeline.core.index_manager.manager import RAGIndexManager +from rag_pipeline.core.index_manager.point_operations import PointOperations from rag_pipeline.core.repository_overlay import IncrementalIndexPreconditionError from rag_pipeline.core.revision_preflight import ( + read_repository_generation_manifest_receipt, read_repository_revision_preflight, ) @@ -205,6 +209,82 @@ def test_exact_revision_preflight_returns_verified_state_identity(): ) is None +def test_alias_receipt_reads_only_deterministic_manifest_point(): + manifest = _complete_revision_payloads()[-1] + client = MagicMock() + client.retrieve.return_value = [SimpleNamespace(payload=manifest)] + digest = manifest["generation_manifest_sha256"] + + result = read_repository_generation_manifest_receipt( + client, + "physical-generation", + "workspace", + "project", + "main", + COMMIT, + digest, + ) + + assert result["generation_manifest_sha256"] == digest + client.retrieve.assert_called_once_with( + collection_name="physical-generation", + ids=[generation_manifest_point_id("workspace", "project", "main")], + with_payload=True, + with_vectors=False, + ) + client.scroll.assert_not_called() + + +def test_manifest_receipt_id_matches_point_storage_contract(): + assert generation_manifest_point_id( + "workspace", + "project", + "main", + ) == PointOperations.generate_point_id( + "workspace", + "project", + "main", + GENERATION_MANIFEST_PATH, + 0, + ) + + +def test_alias_receipt_rejects_registry_digest_mismatch(): + manifest = _complete_revision_payloads()[-1] + client = MagicMock() + client.retrieve.return_value = [SimpleNamespace(payload=manifest)] + + assert read_repository_generation_manifest_receipt( + client, + "physical-generation", + "workspace", + "project", + "main", + COMMIT, + "f" * 64, + ) is None + + +def test_alias_receipt_accepts_legacy_caller_without_registry_digest(): + manifest = _complete_revision_payloads()[-1] + client = MagicMock() + client.retrieve.return_value = [SimpleNamespace(payload=manifest)] + + receipt = read_repository_generation_manifest_receipt( + client, + "physical-generation", + "workspace", + "project", + "main", + COMMIT, + ) + + assert receipt["generation_manifest_sha256"] == manifest[ + "generation_manifest_sha256" + ] + client.scroll.assert_not_called() + + def test_exact_revision_preflight_rejects_incomplete_repository_state(): content = _facts_content() digest = hashlib.sha256(content.encode("utf-8")).hexdigest() @@ -643,3 +723,63 @@ def test_index_manager_retains_stale_representation_as_provenance(mock_read): ) assert result["index_representation_fingerprint"] == "sha256:stale" + + +@patch( + "rag_pipeline.core.index_manager.manager." + "read_repository_generation_manifest_receipt" +) +@patch( + "rag_pipeline.core.index_manager.manager." + "read_repository_revision_preflight" +) +def test_alias_publication_uses_bounded_registry_receipt( + mock_full_preflight, + mock_manifest_receipt, +): + manager = object.__new__(RAGIndexManager) + manager.config = SimpleNamespace(qdrant_collection_prefix="code") + manager.qdrant_client = MagicMock() + manager._collection_manager = MagicMock() + manager._collection_manager.resolve_collection_target.return_value = ( + "physical-generation" + ) + manager._mutation_coordinator = MagicMock() + lease = SimpleNamespace(assert_owned=MagicMock()) + manager._mutation_coordinator.acquire.return_value.__enter__.return_value = ( + lease + ) + mock_manifest_receipt.return_value = { + "workspace": "workspace", + "project": "project", + "branch": "main", + "commit": COMMIT, + "generation_manifest_sha256": "e" * 64, + } + + aliases = manager.publish_generation_aliases( + "workspace", + "project", + "main", + COMMIT, + "generation-target", + "e" * 64, + publish_branch_alias=True, + publish_legacy_project_alias=False, + ) + + assert aliases == ["code_workspace__project__main"] + mock_manifest_receipt.assert_called_once_with( + manager.qdrant_client, + "physical-generation", + "workspace", + "project", + "main", + COMMIT, + "e" * 64, + ) + mock_full_preflight.assert_not_called() + lease.assert_owned.assert_called_once() + manager._collection_manager.atomic_assign_aliases.assert_called_once_with({ + "code_workspace__project__main": "physical-generation", + }) diff --git a/python-ecosystem/rag-pipeline/tests/test_router_index.py b/python-ecosystem/rag-pipeline/tests/test_router_index.py index 6bd01f88..43639611 100644 --- a/python-ecosystem/rag-pipeline/tests/test_router_index.py +++ b/python-ecosystem/rag-pipeline/tests/test_router_index.py @@ -13,6 +13,12 @@ """ import asyncio import json +import logging +import os +import shutil +import threading +from pathlib import Path +from queue import Queue import pytest from unittest.mock import patch, MagicMock from fastapi import HTTPException @@ -241,6 +247,282 @@ async def consume(): assert events[1]["type"] == "complete" assert events[1]["result"]["chunk_count"] == 50 + def test_stream_progress_is_bounded_and_keeps_latest_event(self): + from rag_pipeline.api.routers.index import _coalesce_stream_progress + + events = Queue(maxsize=1) + for batch in range(100): + _coalesce_stream_progress(events, {"completedBatches": batch}) + + assert events.qsize() == 1 + assert events.get_nowait() == {"completedBatches": 99} + + def test_orphan_cleanup_removes_only_old_owned_stream_directories( + self, + tmp_path, + ): + from rag_pipeline.api.routers.index import ( + _remove_owned_stream_repository, + _take_stream_repository_ownership, + cleanup_orphaned_index_repository_stream_workspaces, + ) + + old_owned = tmp_path / "codecrow-rag-owned-stream-old" + old_owned.mkdir() + (old_owned / "source.py").write_text("old", encoding="utf-8") + active_source = ( + tmp_path / "codecrow-rag-branch-generation-active" + ) + active_source.mkdir() + unrelated = tmp_path / "codecrow-rag-branch-generation-old" + unrelated.mkdir() + old_mtime = 1_000_000 + os.utime(old_owned, (old_mtime, old_mtime)) + os.utime(unrelated, (old_mtime, old_mtime)) + + with patch.dict( + "os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)} + ): + active_owned, active_lock = _take_stream_repository_ownership( + str(active_source) + ) + os.utime(active_owned, (old_mtime, old_mtime)) + try: + cleaned = ( + cleanup_orphaned_index_repository_stream_workspaces( + max_age_seconds=3600 + ) + ) + active_survived_cleanup = active_owned.exists() + finally: + _remove_owned_stream_repository(active_owned, active_lock) + + assert cleaned == 1 + assert not old_owned.exists() + assert active_survived_cleanup + assert not active_owned.exists() + assert unrelated.exists() + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_stream_cancellation_waits_for_admitted_index_worker( + self, + mock_get, + ): + _, im = _mock_singletons() + started = threading.Event() + release = threading.Event() + finished = threading.Event() + stats = IndexStats( + namespace="ns", document_count=10, chunk_count=50, + last_updated="2024-01-01", workspace="ws", project="proj", + branch="main", + ) + + def blocking_index(**_kwargs): + started.set() + if not release.wait(timeout=5): + raise RuntimeError("test did not release indexing worker") + finished.set() + return stats + + im.index_repository.side_effect = blocking_index + mock_get.return_value = (_, im) + from rag_pipeline.api.routers.index import ( + _index_stream_workers, + index_repository_stream, + ) + + req = MagicMock() + req.repo_path = "/tmp/repo" + req.workspace = "ws" + req.project = "proj" + req.branch = "main" + req.commit = "abc" + req.preserve_other_branches = False + req.include_patterns = None + req.exclude_patterns = None + req.source_tree_sha256 = None + req.collection_target = "target" + response = index_repository_stream(req) + + async def scenario(): + consumer_started = asyncio.Event() + + async def consume(): + consumer_started.set() + async for _item in response.body_iterator: + pass + + consumer = asyncio.create_task(consume()) + await consumer_started.wait() + for _ in range(100): + if started.is_set(): + break + await asyncio.sleep(0.01) + assert started.is_set() + assert _index_stream_workers.active_count == 1 + + consumer.cancel() + await asyncio.sleep(0.1) + assert not consumer.done() + + release.set() + with pytest.raises(asyncio.CancelledError): + await consumer + + assert finished.is_set() + assert _index_stream_workers.active_count == 0 + + try: + asyncio.run(scenario()) + finally: + release.set() + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_disconnect_cannot_delete_transferred_snapshot_under_active_worker( + self, + mock_get, + tmp_path, + ): + _, im = _mock_singletons() + source = tmp_path / "codecrow-rag-branch-generation-disconnect" + source.mkdir() + (source / "source.py").write_text("value = 1", encoding="utf-8") + started = threading.Event() + release = threading.Event() + worker_path: list[Path] = [] + stats = IndexStats( + namespace="ns", document_count=1, chunk_count=1, + last_updated="2024-01-01", workspace="ws", project="proj", + branch="main", + ) + + def blocking_index(**kwargs): + owned_path = Path(kwargs["repo_path"]) + worker_path.append(owned_path) + assert owned_path != source + assert owned_path.exists() + started.set() + if not release.wait(timeout=5): + raise RuntimeError("test did not release indexing worker") + # Java may clean its original path after the stream disconnects, + # but the atomically moved RAG-owned path remains valid. + assert owned_path.exists() + return stats + + im.index_repository.side_effect = blocking_index + mock_get.return_value = (_, im) + from rag_pipeline.api.models import IndexRequest + from rag_pipeline.api.routers.index import ( + _index_stream_workers, + index_repository_stream, + ) + + request = IndexRequest( + repo_path=str(source), + workspace="ws", + project="proj", + branch="main", + commit="abc", + collection_target="target", + transfer_repo_ownership=True, + ) + + async def scenario(): + response = index_repository_stream(request) + admitted = asyncio.Event() + + async def consume(): + async for item in response.body_iterator: + event = json.loads( + (item.decode() if isinstance(item, bytes) else item) + .removeprefix("data: ").strip() + ) + if event["type"] == "admitted": + admitted.set() + + consumer = asyncio.create_task(consume()) + await asyncio.wait_for(admitted.wait(), timeout=1) + for _ in range(100): + if started.is_set(): + break + await asyncio.sleep(0.01) + assert started.is_set() + assert not source.exists() + + consumer.cancel() + # Simulate the Java pre-admission fallback after a lost stream. + # It targets only the old path, which no longer backs the worker. + shutil.rmtree(source, ignore_errors=True) + await asyncio.sleep(0.1) + assert not consumer.done() + assert worker_path[0].exists() + + release.set() + with pytest.raises(asyncio.CancelledError): + await consumer + + assert not worker_path[0].exists() + assert _index_stream_workers.active_count == 0 + + try: + with patch.dict( + "os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)} + ): + asyncio.run(scenario()) + finally: + release.set() + for path in worker_path: + shutil.rmtree(path, ignore_errors=True) + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_stream_worker_failure_is_terminal_without_duplicate_error_log( + self, + mock_get, + caplog, + ): + _, im = _mock_singletons() + im.index_repository.side_effect = RuntimeError("qdrant unavailable") + mock_get.return_value = (_, im) + from rag_pipeline.api.routers.index import index_repository_stream + + req = MagicMock() + req.repo_path = "/tmp/repo" + req.workspace = "ws" + req.project = "proj" + req.branch = "main" + req.commit = "abc" + req.preserve_other_branches = False + req.include_patterns = None + req.exclude_patterns = None + req.source_tree_sha256 = None + req.collection_target = "target" + req.transfer_repo_ownership = False + + response = index_repository_stream(req) + + async def consume(): + return [item async for item in response.body_iterator] + + with caplog.at_level(logging.DEBUG): + events = [ + json.loads( + (item.decode() if isinstance(item, bytes) else item) + .removeprefix("data: ").strip() + ) + for item in asyncio.run(consume()) + ] + + assert events[-1] == { + "type": "error", + "message": "qdrant unavailable", + } + assert not any( + record.levelno >= logging.ERROR + and "qdrant unavailable" in record.getMessage() + for record in caplog.records + ) + @patch("rag_pipeline.api.routers.index._get_singletons") def test_exact_index_forwards_readable_alias_publication(self, mock_get): _, im = _mock_singletons() @@ -296,6 +578,65 @@ def test_forwards_readable_alias_publication(self, mock_get): assert im.advance_generation.call_args.kwargs["publish_branch_alias"] is True +class TestGenerationAliasPublication: + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_forwards_registry_manifest_receipt(self, mock_get): + _, im = _mock_singletons() + im.publish_generation_aliases.return_value = ["readable-main"] + mock_get.return_value = (_, im) + + from rag_pipeline.api.models import GenerationAliasPublicationRequest + from rag_pipeline.api.routers.index import publish_generation_aliases + + request = GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="exact-target", + generation_manifest_sha256="b" * 64, + ) + + result = publish_generation_aliases(request) + + assert result["status"] == "published" + assert im.publish_generation_aliases.call_args.kwargs[ + "generation_manifest_sha256" + ] == "b" * 64 + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_retryable_failure_does_not_emit_server_error_log( + self, + mock_get, + caplog, + ): + _, im = _mock_singletons() + im.publish_generation_aliases.side_effect = RuntimeError("timed out") + mock_get.return_value = (_, im) + + from rag_pipeline.api.models import GenerationAliasPublicationRequest + from rag_pipeline.api.routers.index import publish_generation_aliases + + request = GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="exact-target", + ) + + with pytest.raises(HTTPException) as exc_info: + publish_generation_aliases(request) + + assert exc_info.value.status_code == 500 + assert not any( + record.levelname == "ERROR" + and "alias" in record.getMessage().lower() + for record in caplog.records + ) + + # ───────────────────────────────────────────────────────────── # update_files / delete_files # ───────────────────────────────────────────────────────────── @@ -494,6 +835,28 @@ def test_delete_branch_not_found(self, mock_get): result = delete_branch("ws", "proj", "feat") assert result["status"] == "not_found" + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_delete_exact_generation_forwards_registry_receipt(self, mock_get): + _, im = _mock_singletons() + im.delete_branch.return_value = True + mock_get.return_value = (_, im) + + from rag_pipeline.api.routers.index import delete_branch + digest = "a" * 64 + result = delete_branch( + "ws", "proj", "develop", "target", "revision", digest + ) + + assert result["status"] == "success" + im.delete_branch.assert_called_once_with( + "ws", + "proj", + "develop", + collection_target="target", + generation_revision="revision", + generation_manifest_sha256=digest, + ) + @patch("rag_pipeline.api.routers.index._get_singletons") def test_list_branches(self, mock_get): _, im = _mock_singletons() diff --git a/python-ecosystem/rag-pipeline/tests/test_router_pr.py b/python-ecosystem/rag-pipeline/tests/test_router_pr.py index 7a1276c2..30bec98b 100644 --- a/python-ecosystem/rag-pipeline/tests/test_router_pr.py +++ b/python-ecosystem/rag-pipeline/tests/test_router_pr.py @@ -38,6 +38,9 @@ def _make_index_manager(): ) im._get_project_collection_name.return_value = "rag_ws__proj" im._collection_manager.collection_exists.return_value = True + im._collection_manager.resolve_collection_target.side_effect = ( + lambda collection_name: collection_name + ) im.splitter.split_documents.return_value = [] im.splitter.split_documents_resilient.side_effect = ( lambda documents, capabilities=None: ( @@ -829,6 +832,10 @@ def test_success(self, mock_get): "pr": True, "pr_number": 42, } + assert im.qdrant_client.delete.call_args.kwargs["wait"] is True + im._collection_manager.ensure_payload_indexes.assert_called_once_with( + "rag_ws__proj" + ) @patch("rag_pipeline.api.routers.pr._get_index_manager") def test_explicit_collection_target_still_uses_tenant_filter(self, mock_get): @@ -852,7 +859,8 @@ def test_explicit_collection_target_still_uses_tenant_filter(self, mock_get): @patch("rag_pipeline.api.routers.pr._get_index_manager") def test_collection_not_found(self, mock_get): im = _make_index_manager() - im._collection_manager.collection_exists.return_value = False + im._collection_manager.resolve_collection_target.return_value = None + im._collection_manager.resolve_collection_target.side_effect = None mock_get.return_value = im from rag_pipeline.api.routers.pr import delete_pr_files diff --git a/python-ecosystem/rag-pipeline/tests/test_services.py b/python-ecosystem/rag-pipeline/tests/test_services.py index ef48a436..3bdec6b1 100644 --- a/python-ecosystem/rag-pipeline/tests/test_services.py +++ b/python-ecosystem/rag-pipeline/tests/test_services.py @@ -42,10 +42,17 @@ def test_init(self, MockQdrant, mock_info, mock_create): base = RAGQueryBase(config) assert base.config is config - MockQdrant.assert_called_once_with(url="http://localhost:6333", api_key=None) + MockQdrant.assert_called_once_with( + url="http://localhost:6333", + api_key=None, + timeout=30, + ) assert base.qdrant_client is not None assert base.embed_model is not None + base.close() + MockQdrant.return_value.close.assert_called_once_with() + @patch("rag_pipeline.services.base.create_embedding_model") @patch("rag_pipeline.services.base.get_embedding_model_info") @patch("rag_pipeline.services.base.QdrantClient") diff --git a/tools/review_quality/neutral_prompt_context_gate.py b/tools/review_quality/neutral_prompt_context_gate.py index fe805346..935fda69 100644 --- a/tools/review_quality/neutral_prompt_context_gate.py +++ b/tools/review_quality/neutral_prompt_context_gate.py @@ -139,6 +139,15 @@ def _request( currentCommitHash=_revision("head", digest), commitHash=_revision("head", digest), baseCommitHash=_revision("base", digest), + # The deterministic adapter models a PR overlay on an immutable base + # generation. Supply the same complete base binding production sends; + # the dry-run facade then returns the matching PR-generation receipts. + # Without these coordinates Stage 1 correctly treats the request as an + # unbound legacy review and must not query PR-scoped vectors. + ragCollectionTarget=f"neutral_{digest}_main_generation", + ragBaseGenerationManifestSha256=hashlib.sha256( + f"base-generation\0{digest}".encode("utf-8") + ).hexdigest(), changedFiles=list(changed), rawDiff=_raw_diff(definition), prTitle=f"Provider-free neutral prompt gate: {definition.case_id}",