diff --git a/deployment/config/inference-orchestrator/.env.sample b/deployment/config/inference-orchestrator/.env.sample index 95f6ef62..d8e365e2 100644 --- a/deployment/config/inference-orchestrator/.env.sample +++ b/deployment/config/inference-orchestrator/.env.sample @@ -16,6 +16,7 @@ SERVICE_SECRET=change-me-to-a-random-secret # ANALYSIS_CONSUMER_HEARTBEAT_SECONDS=5 # MAX_CONCURRENT_COMMANDS=10 # COMMAND_TIMEOUT_SECONDS=600 +# COMMAND_EVENT_TTL_SECONDS=3600 # REVIEW_TIMEOUT_SECONDS=1500 # REVIEW_GLOBAL_RAG_QUERY_TIMEOUT_SECONDS=5 diff --git a/deployment/config/java-shared/application.properties.sample b/deployment/config/java-shared/application.properties.sample index c788fb31..245f789c 100644 --- a/deployment/config/java-shared/application.properties.sample +++ b/deployment/config/java-shared/application.properties.sample @@ -192,6 +192,19 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF # Repair interval for readable Qdrant current-branch aliases of active generations. #codecrow.rag.operator-alias.reconcile-interval-ms=300000 #codecrow.rag.operator-alias.reconcile-initial-delay-ms=15000 +# Exact-generation operation recovery. A live build heartbeats its persisted +# operation; a published operation also repairs an interrupted job/status handoff. +#codecrow.rag.generation.stale-after-minutes=30 +#codecrow.rag.generation.recovery-interval-ms=300000 +#codecrow.rag.generation.recovery-initial-delay-ms=60000 +# Legacy unsealed incremental updates use a separate database job lease because +# they do not own an immutable generation operation. +#codecrow.rag.legacy-job.lease-seconds=120 +#codecrow.rag.legacy-job.heartbeat-interval-seconds=15 +#codecrow.rag.legacy-job.heartbeat-threads=4 +#codecrow.rag.legacy-job.recovery-interval-ms=30000 +#codecrow.rag.legacy-job.recovery-initial-delay-ms=30000 +#codecrow.rag.legacy-job.recovery-batch-size=100 # Shared VCS acquisition threshold for incremental RAG and reconciliation # fallback. The legacy codecrow.rag.incremental.archive-file-threshold property # remains a fallback when this property is not set. @@ -200,6 +213,10 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF # Analysis Lock Configuration # Lock timeout - maximum time for a single analysis to hold a lock (in minutes) #analysis.lock.timeout.minutes=30 +# Independent lease-renewal scheduler capacity (default 4, minimum 2 threads) +#analysis.lock.heartbeat.threads=4 +# Independent lease-renewal interval (seconds); bounded to one-third of the lease +#analysis.lock.heartbeat.interval.seconds=60 # RAG indexing lock timeout - longer timeout for RAG indexing operations (in minutes, default 6 hours) #analysis.lock.rag.timeout.minutes=360 # Lock wait timeout - maximum time to wait for a lock to be released (in minutes) diff --git a/deployment/config/rag-pipeline/.env.sample b/deployment/config/rag-pipeline/.env.sample index 8d678290..936b514f 100644 --- a/deployment/config/rag-pipeline/.env.sample +++ b/deployment/config/rag-pipeline/.env.sample @@ -49,6 +49,7 @@ SERVICE_SECRET=change-me-to-a-random-secret # QDRANT_URL=http://qdrant:6333 # QDRANT_API_KEY= # QDRANT_COLLECTION_PREFIX=codecrow +# QDRANT_TIMEOUT_SECONDS=30 # QDRANT_VECTORS_ON_DISK=true # QDRANT_UPSERT_BATCH_SIZE=128 diff --git a/frontend b/frontend index cc7c44a3..4d314b5b 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit cc7c44a3c0623f3e9259e5f388a27bdea0527100 +Subproject commit 4d314b5b70e0bea43feefe8597f9728db57098d1 diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java index 58642f6d..5b91670f 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java @@ -12,11 +12,13 @@ import org.rostilos.codecrow.queue.RedisQueueService; import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import java.io.IOException; import java.security.GeneralSecurityException; +import java.time.OffsetDateTime; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -40,6 +42,9 @@ public class AiAnalysisClient { @Autowired(required = false) private RagBranchIndexGenerationRepository branchGenerationRepository; + @Autowired(required = false) + private RagBranchIndexRepository branchIndexRepository; + static final String INACTIVITY_TIMEOUT_MINUTES_KEY = "ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES"; static final String ADMISSION_TIMEOUT_MINUTES_KEY = @@ -353,11 +358,16 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re payload.put("previousCommitHash", request.getPreviousCommitHash()); payload.put("currentCommitHash", request.getCurrentCommitHash()); payload.put("baseCommitHash", request.getBaseCommitHash()); - if (branchGenerationRepository != null + if (request.getRagEnabled() + && branchGenerationRepository != null + && branchIndexRepository != null && request.getProjectId() != null && request.getTargetBranchName() != null && request.getBaseCommitHash() != null) { - branchGenerationRepository.findAvailableExactGeneration( + int accessed = branchIndexRepository.markAccessedIfUnclaimed( + request.getProjectId(), request.getTargetBranchName(), OffsetDateTime.now()); + if (accessed > 0) { + branchGenerationRepository.findAvailableExactGeneration( request.getProjectId(), request.getTargetBranchName(), request.getBaseCommitHash(), @@ -371,6 +381,7 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re payload.put("ragBaseGenerationManifestSha256", generation.getManifestDigest()); }); + } } payload.put("previousCodeAnalysisIssues", request.getPreviousCodeAnalysisIssues()); payload.put("reconciliationFileContents", request.getReconciliationFileContents()); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java index 784a2012..3fc411dc 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java @@ -4,6 +4,7 @@ import org.rostilos.codecrow.queue.RedisQueueService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.IOException; @@ -24,10 +25,35 @@ public class AiCommandClient { private final RedisQueueService queueService; private final ObjectMapper objectMapper; private static final int COMMAND_TIMEOUT_MINUTES = 30; - + static final String CONSUMER_HEARTBEAT_KEY = + "codecrow:commands:consumer:heartbeat"; + private static final long DEFAULT_ADMISSION_TIMEOUT_MINUTES = 5L; + private static final String ADMISSION_TIMEOUT_MINUTES_KEY = + "COMMAND_QUEUE_ADMISSION_TIMEOUT_MINUTES"; + private final long commandTimeoutMillis; + private final long admissionTimeoutMillis; + + @Autowired public AiCommandClient(RedisQueueService queueService, ObjectMapper objectMapper) { + this( + queueService, + objectMapper, + TimeUnit.MINUTES.toMillis(COMMAND_TIMEOUT_MINUTES), + resolveAdmissionTimeoutMillis()); + } + + AiCommandClient( + RedisQueueService queueService, + ObjectMapper objectMapper, + long commandTimeoutMillis, + long admissionTimeoutMillis) { this.queueService = queueService; this.objectMapper = objectMapper; + if (commandTimeoutMillis <= 0 || admissionTimeoutMillis <= 0) { + throw new IllegalArgumentException("Command queue timeouts must be positive"); + } + this.commandTimeoutMillis = commandTimeoutMillis; + this.admissionTimeoutMillis = admissionTimeoutMillis; } /** @@ -85,24 +111,44 @@ private Map executeAsyncJob( Consumer> eventHandler) throws IOException { String eventQueueKey = "codecrow:analysis:events:" + jobId; String jobsQueueKey = "codecrow:queue:commands"; + String jsonPayload = null; try { + if (!queueService.hasKey(CONSUMER_HEARTBEAT_KEY)) { + throw new IOException( + "Inference Orchestrator is unavailable: no live command queue consumer"); + } Map jobPayload = Map.of( "job_id", jobId, "command_type", commandType, "request", request); - String jsonPayload = objectMapper.writeValueAsString(jobPayload); + jsonPayload = objectMapper.writeValueAsString(jobPayload); queueService.leftPush(jobsQueueKey, jsonPayload); queueService.setExpiry(eventQueueKey, COMMAND_TIMEOUT_MINUTES + 1); long startTime = System.currentTimeMillis(); - long timeoutMillis = TimeUnit.MINUTES.toMillis(COMMAND_TIMEOUT_MINUTES); + boolean workerAcknowledged = false; while (true) { - if (System.currentTimeMillis() - startTime > timeoutMillis) { + long now = System.currentTimeMillis(); + if (!workerAcknowledged) { + if (!queueService.hasKey(CONSUMER_HEARTBEAT_KEY)) { + throw new IOException( + "Inference Orchestrator became unavailable before " + + "the command job was admitted"); + } + if (now - startTime > admissionTimeoutMillis) { + throw new IOException( + "AI command was not admitted by Inference Orchestrator within " + + TimeUnit.MILLISECONDS.toSeconds(admissionTimeoutMillis) + + " seconds"); + } + } + if (now - startTime > commandTimeoutMillis) { throw new IOException( - "AI command timed out after " + COMMAND_TIMEOUT_MINUTES + " minutes for Job: " + jobId); + "AI command timed out after " + commandTimeoutMillis + + "ms for Job: " + jobId); } String eventJson = queueService.rightPop(eventQueueKey, 5); @@ -110,6 +156,7 @@ private Map executeAsyncJob( if (eventJson == null) { continue; // Timeout on rightPop, continue to check overall timeout } + workerAcknowledged = true; try { Map event = objectMapper.readValue(eventJson, Map.class); @@ -150,6 +197,14 @@ private Map executeAsyncJob( log.error("Failed to communicate with AI async queue", e); throw new IOException("AI queue communication failed: " + e.getMessage(), e); } finally { + if (jsonPayload != null) { + try { + queueService.removeFromList(jobsQueueKey, jsonPayload); + } catch (Exception cleanupError) { + log.warn("Failed to remove command job {} from pending queue: {}", + jobId, cleanupError.getMessage()); + } + } try { queueService.deleteKey(eventQueueKey); } catch (Exception ignored) { @@ -157,6 +212,27 @@ private Map executeAsyncJob( } } + private static long resolveAdmissionTimeoutMillis() { + String configured = System.getProperty(ADMISSION_TIMEOUT_MINUTES_KEY); + if (configured == null || configured.isBlank()) { + configured = System.getenv(ADMISSION_TIMEOUT_MINUTES_KEY); + } + if (configured == null || configured.isBlank()) { + return TimeUnit.MINUTES.toMillis(DEFAULT_ADMISSION_TIMEOUT_MINUTES); + } + try { + long minutes = Long.parseLong(configured.trim()); + if (minutes <= 0) { + throw new IllegalArgumentException( + ADMISSION_TIMEOUT_MINUTES_KEY + " must be a positive integer"); + } + return TimeUnit.MINUTES.toMillis(minutes); + } catch (NumberFormatException invalid) { + throw new IllegalArgumentException( + ADMISSION_TIMEOUT_MINUTES_KEY + " must be a positive integer", invalid); + } + } + /** * Request object for summarize endpoint. */ diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java index ea3e8f93..cb735f3c 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java @@ -195,13 +195,19 @@ private Map process( } List unanalyzedCommits = Collections.emptyList(); + AnalysisLockService.LockLease lockLease = null; try { + lockLease = analysisLockService.maintainLockLease( + lockKey.get(), + analysisLockService.getLeaseMinutes(AnalysisLockType.BRANCH_ANALYSIS)); + requireActiveLease(lockLease); + requireConfirmedLease(lockLease); Optional existingBranchOpt = branchRepository.findByProjectIdAndBranchName( project.getId(), request.getTargetBranchName()); if (request.getSourcePrNumber() == null - && matchCache(request, existingBranchOpt, project, consumer)) { + && matchCache(request, existingBranchOpt, project, consumer, lockLease)) { return Map.of( "status", "skipped", "reason", "commit_already_analyzed", @@ -295,6 +301,7 @@ && matchCache(request, existingBranchOpt, project, consumer)) { // This keeps PullRequestState accurate for commit coverage checks. Set mergedPrNumbers = new LinkedHashSet<>(); if (prNumber != null) { + requireConfirmedLease(lockLease); try { pullRequestService.markPullRequestMerged(project.getId(), prNumber); mergedPrNumbers.add(prNumber); @@ -305,6 +312,7 @@ && matchCache(request, existingBranchOpt, project, consumer)) { } if (prNumber != null || isMergeCommit) { + requireConfirmedLease(lockLease); PullRequestStatusSyncService.SyncResult syncResult = pullRequestStatusSyncService .syncOpenPullRequestStates( project, request.getTargetBranchName(), consumer); @@ -320,6 +328,7 @@ && matchCache(request, existingBranchOpt, project, consumer)) { && prNumber == null && !isMergeCommit && mergedPrNumbers.isEmpty()) { + requireConfirmedLease(lockLease); return skipAlreadyAnalyzedRange(project, request, existingBranchOpt, consumer); } @@ -381,12 +390,16 @@ && matchCache(request, existingBranchOpt, project, consumer)) { } if (changedFiles.isEmpty()) { + requireConfirmedLease(lockLease); branchFileOperationsService.createOrUpdateProjectBranch( project, request, existingBranchOpt.orElse(null)); EventNotificationEmitter.emitStatus(consumer, "skipped", "No changed files match the project analysis scope"); + requireConfirmedLease(lockLease); performIncrementalRagUpdate(request, project, repositoryDiff, consumer, directPushLimited); + requireConfirmedLease(lockLease); branchHealthService.markBranchHealthy(project, request); + requireConfirmedLease(lockLease); branchHealthService.recordCommitsAnalyzed(project, unanalyzedCommits, request.getTargetBranchName()); return Map.of("status", "accepted", "cached", false, @@ -403,7 +416,8 @@ && matchCache(request, existingBranchOpt, project, consumer)) { if (!isFirstAnalysis && !directPushLimited) { performDirectPushAnalysisIfNeeded( project, request, unanalyzedCommits, rawDiff, - changedFiles, provider, consumer, prNumber, isMergeCommit); + changedFiles, provider, consumer, prNumber, isMergeCommit, + lockLease); } else if (isFirstAnalysis) { log.info( "First analysis for branch {} — skipping direct push analysis (establishing baseline, {} files)", @@ -422,12 +436,15 @@ && matchCache(request, existingBranchOpt, project, consumer)) { log.info("Branch archive: {} files extracted for {} changed files", branchFileSnapshot.contents().size(), changedFiles.size()); + requireConfirmedLease(lockLease); Set existingFiles = branchFileOperationsService.updateBranchFiles( changedFiles, project, request.getTargetBranchName(), branchFileSnapshot); + requireConfirmedLease(lockLease); Branch branch = branchFileOperationsService.createOrUpdateProjectBranch( project, request, existingBranchOpt.orElse(null)); + requireConfirmedLease(lockLease); if (mergedPrNumbers.size() > 1) { branchIssueMappingService.mapCodeAnalysisIssuesToBranch( changedFiles, existingFiles, branch, project, mergedPrNumbers); @@ -435,15 +452,18 @@ && matchCache(request, existingBranchOpt, project, consumer)) { branchIssueMappingService.mapCodeAnalysisIssuesToBranch( changedFiles, existingFiles, branch, project, prNumber); } + requireConfirmedLease(lockLease); branchIssueReconciliationService.reconcileIssueLineNumbers(rawDiff, changedFiles, branch); // Update branch issue counts after mapping + requireConfirmedLease(lockLease); Branch refreshedBranch = refreshAndSaveIssueCounts(branch); log.info("Updated branch issue counts after mapping: total={}, high={}, medium={}, low={}, resolved={}", refreshedBranch.getTotalIssues(), refreshedBranch.getHighSeverityCount(), refreshedBranch.getMediumSeverityCount(), refreshedBranch.getLowSeverityCount(), refreshedBranch.getResolvedCount()); + requireConfirmedLease(lockLease); branchIssueReconciliationService.reanalyzeCandidateIssues( changedFiles, existingFiles, refreshedBranch, project, request, consumer, branchFileSnapshot.contents(), rawDiff, @@ -457,17 +477,22 @@ && matchCache(request, existingBranchOpt, project, consumer)) { EventNotificationEmitter.emitStatus(consumer, "finalizing_branch_state", "Saving changed-file reconciliation results"); + requireConfirmedLease(lockLease); branchFileOperationsService.updateFileSnapshotsForBranch(existingFiles, project, request, branchFileSnapshot); Branch branchForVerify = branchRepository.findByProjectIdAndBranchName( project.getId(), request.getTargetBranchName()).orElse(refreshedBranch); + requireConfirmedLease(lockLease); branchIssueReconciliationService.verifyIssueLineNumbersWithSnippets( changedFiles, project, branchForVerify); // ── Post-analysis housekeeping ──────────────────────────────────── + requireConfirmedLease(lockLease); performIncrementalRagUpdate(request, project, repositoryDiff, consumer, directPushLimited); + requireConfirmedLease(lockLease); branchHealthService.markBranchHealthy(project, request); + requireConfirmedLease(lockLease); branchHealthService.recordCommitsAnalyzed(project, unanalyzedCommits, request.getTargetBranchName()); @@ -477,11 +502,36 @@ && matchCache(request, existingBranchOpt, project, consumer)) { return Map.of("status", "accepted", "cached", false, "branch", request.getTargetBranchName()); + } catch (BranchAnalysisLeaseLostException e) { + log.info("Branch analysis stopped after lock lease ownership was lost: project={}, branch={}", + project.getId(), request.getTargetBranchName()); + throw e; } catch (Exception e) { branchHealthService.handleProcessFailure(project, request, unanalyzedCommits, e); throw e; } finally { - analysisLockService.releaseLock(lockKey.get()); + if (lockLease != null) { + try { + lockLease.close(); + } catch (RuntimeException closeFailure) { + log.info( + "Branch analysis lock lease could not be closed after processing; " + + "continuing lock cleanup: project={}, branch={}, detail={}", + project.getId(), request.getTargetBranchName(), + closeFailure.getMessage()); + } + } + try { + analysisLockService.releaseLock(lockKey.get()); + } catch (RuntimeException releaseFailure) { + // The branch outcome is already decided. Cleanup failure is + // observational; the durable lock lease will expire. + log.info( + "Branch analysis lock could not be released after processing; " + + "leaving it to expiry: project={}, branch={}, detail={}", + project.getId(), request.getTargetBranchName(), + releaseFailure.getMessage()); + } } } @@ -490,12 +540,36 @@ public Map fullReconcile(Long projectId, String branchName, return branchFullReconciliationService.fullReconcile(projectId, branchName, consumer); } + private static void requireActiveLease(AnalysisLockService.LockLease lease) throws IOException { + if (lease == null || lease.isOwnershipLost()) { + throw lostLeaseException(); + } + } + + private static void requireConfirmedLease(AnalysisLockService.LockLease lease) throws IOException { + if (lease == null || !lease.confirmOwnership()) { + throw lostLeaseException(); + } + } + + private static BranchAnalysisLeaseLostException lostLeaseException() { + return new BranchAnalysisLeaseLostException( + "Branch analysis lost its lock lease while the worker was active"); + } + + private static final class BranchAnalysisLeaseLostException extends IOException { + private BranchAnalysisLeaseLostException(String message) { + super(message); + } + } + /** * Check if the incoming commit was already SUCCESSFULLY analyzed. * Uses lastSuccessfulCommitHash so that failed attempts are re-processed. */ private boolean matchCache(BranchProcessRequest request, Optional existingBranchOpt, - Project project, Consumer> consumer) { + Project project, Consumer> consumer, + AnalysisLockService.LockLease lockLease) throws IOException { if (request.getCommitHash() == null || existingBranchOpt.isEmpty()) return false; @@ -517,9 +591,12 @@ private boolean matchCache(BranchProcessRequest request, Optional existi BranchFileOperationsService.BranchFileSnapshot branchFileSnapshot = branchFileOperationsService.downloadBranchFileSnapshot( vcsRepoInfoImpl, request.getCommitHash(), branchFiles); + requireConfirmedLease(lockLease); branchFileOperationsService.updateFileSnapshotsForBranch( branchFiles, project, request, branchFileSnapshot); } + } catch (BranchAnalysisLeaseLostException leaseLost) { + throw leaseLost; } catch (Exception snapEx) { log.warn("Failed to refresh file snapshots on skip path (non-critical): {}", snapEx.getMessage()); @@ -699,7 +776,8 @@ private void performDirectPushAnalysisIfNeeded( EVcsProvider provider, Consumer> consumer, Long mergedPrNumber, - boolean isMergeCommit) { + boolean isMergeCommit, + AnalysisLockService.LockLease lockLease) throws IOException { if (unanalyzedCommits.isEmpty()) { log.debug("No unanalyzed commits — skipping direct push analysis check"); @@ -794,9 +872,12 @@ private void performDirectPushAnalysisIfNeeded( project, request, rawDiff, fileContents, new ArrayList<>(changedFiles)); // Call the inference orchestrator using single request + requireConfirmedLease(lockLease); Map aiResponse = aiAnalysisClient.performAnalysis(aiRequests.get(0), event -> { try { - consumer.accept(event); + if (consumer != null) { + consumer.accept(event); + } } catch (Exception ex) { log.debug("Event consumer failed during direct push analysis: {}", ex.getMessage()); @@ -815,6 +896,7 @@ private void performDirectPushAnalysisIfNeeded( } // Save the analysis with DetectionSource.DIRECT_PUSH_ANALYSIS + requireConfirmedLease(lockLease); CodeAnalysis directPushAnalysis = codeAnalysisService.createDirectPushAnalysisFromAiResponse( project, aiResponse, request.getTargetBranchName(), request.getCommitHash(), fileContents); @@ -842,6 +924,8 @@ private void performDirectPushAnalysisIfNeeded( EventNotificationEmitter.emitStatus(consumer, "direct_push_analysis_complete", "Direct push analysis found " + issuesFound + " issues"); + } catch (BranchAnalysisLeaseLostException leaseLost) { + throw leaseLost; } catch (Exception e) { // Direct push analysis failure is non-fatal — reconciliation will still run log.warn("Direct push analysis failed (non-fatal, reconciliation will still run): {}", @@ -913,7 +997,8 @@ private void performIncrementalRagUpdate(BranchProcessRequest request, Project p boolean ragUpdated = ragOperationsService.triggerIncrementalUpdate( project, targetBranch, request.getCommitHash(), commitDiff, consumer); if (!ragUpdated) { - log.warn("RAG incremental update did not complete for project={}, branch={}, commit={}", + log.info("RAG incremental update did not complete; retaining the last usable index " + + "for project={}, branch={}, commit={}", project.getId(), targetBranch, request.getCommitHash()); return; } @@ -921,7 +1006,8 @@ private void performIncrementalRagUpdate(BranchProcessRequest request, Project p log.info("Non-main branch push - updating branch index for project={}, branch={}", project.getId(), targetBranch); if (!ragOperationsService.updateBranchIndex(project, targetBranch, consumer)) { - log.warn("RAG branch index update did not complete for project={}, branch={}", + log.info("RAG branch index update did not complete; retaining the last usable index " + + "for project={}, branch={}", project.getId(), targetBranch); return; } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java index bb6cfc6b..4b458ecd 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java @@ -22,7 +22,6 @@ import org.rostilos.codecrow.analysisengine.exception.AnalysisLockedException; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.analysisengine.service.PullRequestService; -import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; import org.rostilos.codecrow.commitgraph.service.AnalyzedCommitService; import org.rostilos.codecrow.analysisengine.service.vcs.VcsAiClientService; import org.rostilos.codecrow.analysisengine.service.vcs.VcsReportingService; @@ -46,7 +45,6 @@ import java.util.Optional; import java.util.Collections; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.rostilos.codecrow.analysisengine.util.DiffFingerprintUtil; @@ -73,7 +71,6 @@ public class PullRequestAnalysisProcessor { private final AiAnalysisClient aiAnalysisClient; private final VcsServiceFactory vcsServiceFactory; private final AnalysisLockService analysisLockService; - private final RagOperationsService ragOperationsService; private final ApplicationEventPublisher eventPublisher; private final AnalyzedCommitService analyzedCommitService; private final VcsClientProvider vcsClientProvider; @@ -99,7 +96,6 @@ public PullRequestAnalysisProcessor( FileSnapshotService fileSnapshotService, PrIssueTrackingService prIssueTrackingService, AstScopeEnricher astScopeEnricher, - @Autowired(required = false) RagOperationsService ragOperationsService, @Autowired(required = false) ApplicationEventPublisher eventPublisher ) { this.codeAnalysisService = codeAnalysisService; @@ -108,7 +104,6 @@ public PullRequestAnalysisProcessor( this.aiAnalysisClient = aiAnalysisClient; this.vcsServiceFactory = vcsServiceFactory; this.analysisLockService = analysisLockService; - this.ragOperationsService = ragOperationsService; this.eventPublisher = eventPublisher; this.analyzedCommitService = analyzedCommitService; this.vcsClientProvider = vcsClientProvider; @@ -148,7 +143,7 @@ public Map process( AnalysisLockType.PR_ANALYSIS, request.getCommitHash(), request.getPullRequestId(), - consumer::accept); + event -> emitEvent(consumer, event)); if (acquiredLock.isEmpty()) { String message = String.format( @@ -171,7 +166,12 @@ public Map process( lockKey = acquiredLock.get(); } + AnalysisLockService.LockLease lockLease = null; try { + lockLease = analysisLockService.maintainLockLease( + lockKey, + analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS)); + requireActiveLease(lockLease); EVcsProvider provider = ProjectVcsInfoRetriever.getVcsProvider(project); VcsReportingService reportingService = vcsServiceFactory.getReportingService(provider); // Get all previous analyses for this PR to provide full issue history to AI @@ -190,6 +190,7 @@ public Map process( // Request construction acquires and verifies the provider's immutable // full head object ID. Persist the PR only after that canonicalization, // never from an abbreviated or otherwise unverified webhook value. + requireConfirmedLease(lockLease); PullRequest pullRequest = pullRequestService.createOrUpdatePullRequest( request.getProjectId(), request.getPullRequestId(), @@ -202,7 +203,8 @@ public Map process( String message = "No changed files match the project analysis scope"; log.info("Skipping PR analysis for project={}, PR={}: {}", project.getId(), request.getPullRequestId(), message); - consumer.accept(Map.of("type", "info", "message", message)); + emitEvent(consumer, Map.of("type", "info", "message", message)); + requireConfirmedLease(lockLease); publishAnalysisCompletedEvent(project, request, correlationId, startTime, AnalysisCompletedEvent.CompletionStatus.SUCCESS, 0, 0, null); return Map.of("status", "ignored", "message", message); @@ -216,8 +218,9 @@ public Map process( CacheHitType cacheHit = postAnalysisCacheIfExist( project, pullRequest, request.getCommitHash(), request.getPullRequestId(), reportingService, request.getPlaceholderCommentId(), request.getTargetBranchName(), - request.getSourceBranchName(), diffFingerprint); + request.getSourceBranchName(), diffFingerprint, lockLease); if (cacheHit != CacheHitType.NONE) { + requireConfirmedLease(lockLease); publishAnalysisCompletedEvent(project, request, correlationId, startTime, AnalysisCompletedEvent.CompletionStatus.SUCCESS, 0, 0, null); String cacheStatus = cacheHit == CacheHitType.COMMIT_HASH ? "cached_by_commit" : "cached"; @@ -225,8 +228,10 @@ public Map process( } if (postDiffFingerprintCacheIfExist( - request, diffFingerprint, project, pullRequest, aiRequest, reportingService + request, diffFingerprint, project, pullRequest, aiRequest, reportingService, + lockLease )) { + requireConfirmedLease(lockLease); publishAnalysisCompletedEvent(project, request, correlationId, startTime, AnalysisCompletedEvent.CompletionStatus.SUCCESS, 0, 0, null); return Map.of("status", "cached_by_fingerprint", "cached", true); @@ -237,43 +242,18 @@ public Map process( project.getId(), request.getPullRequestId()); } - // Only prepare RAG after the exact snapshot/configuration cache has missed. - ensureRagIndexForTargetBranch(project, request.getTargetBranchName(), consumer); - - AtomicBoolean lockLeaseLost = new AtomicBoolean(false); Map aiResponse = aiAnalysisClient.performAnalysis(aiRequest, event -> { - if ("processing".equals(String.valueOf(event.get("state")))) { - try { - if (!analysisLockService.renewLock( - lockKey, - analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS))) { - lockLeaseLost.set(true); - log.error("PR analysis lost its lock lease: {}", lockKey); - } - } catch (Exception leaseError) { - lockLeaseLost.set(true); - log.error("Failed to renew PR analysis lock lease: {}", lockKey, leaseError); - } - } - try { - log.debug("Received event from AI client: type={}", event.get("type")); - consumer.accept(event); - log.debug("Event forwarded to consumer successfully"); - } catch (Exception ex) { - log.error("Event consumer failed: {}", ex.getMessage(), ex); - } + log.debug("Received event from AI client: type={}", event.get("type")); + emitEvent(consumer, event); }); - if (lockLeaseLost.get()) { - throw new IOException( - "PR analysis lost its lock lease while the review worker was active"); - } + requireConfirmedLease(lockLease); if (AiAnalysisClient.isPromptDryRunResult(aiResponse)) { Object artifact = aiResponse.get("promptArtifact"); log.warn( "Prompt dry run completed for project={}, PR={}; artifact={}", project.getId(), request.getPullRequestId(), artifact); - consumer.accept(Map.of( + emitEvent(consumer, Map.of( "type", "info", "state", "prompt_dry_run_completed", "message", "Prompt dry run completed without publishing an analysis", @@ -297,6 +277,9 @@ public Map process( request.getCommitHash()); } + // Direct VCS fallback can itself be slow. Establish an atomic lease + // barrier immediately before the first durable analysis write. + requireConfirmedLease(lockLease); CodeAnalysis newAnalysis = codeAnalysisService.createAnalysisFromAiResponse( project, aiResponse, @@ -353,6 +336,10 @@ public Map process( log.warn("PR issue tracking failed (non-critical): {}", trackEx.getMessage()); } + // Ownership may have changed after persistence but before external + // publication. Perform an atomic proof instead of waiting for the + // next scheduled heartbeat to observe it. + requireConfirmedLease(lockLease); try { reportingService.postAnalysisResults( newAnalysis, @@ -362,15 +349,17 @@ public Map process( request.getPlaceholderCommentId()); } catch (IOException e) { log.error("Failed to post analysis results to VCS: {}", e.getMessage(), e); - consumer.accept(Map.of( + emitEvent(consumer, Map.of( "type", "warning", "message", "Analysis completed but failed to post results to VCS: " + e.getMessage())); } // === DAG: Mark PR commits as ANALYZED === + requireConfirmedLease(lockLease); markPrCommitsAnalyzed(project, request, newAnalysis); // Publish successful completion event + requireConfirmedLease(lockLease); publishAnalysisCompletedEvent(project, request, correlationId, startTime, AnalysisCompletedEvent.CompletionStatus.SUCCESS, issuesFound, allChangedFiles != null ? allChangedFiles.size() : 0, null); @@ -378,7 +367,7 @@ public Map process( return aiResponse; } catch (IOException e) { log.error("IOException during PR analysis: {}", e.getMessage(), e); - consumer.accept(Map.of( + emitEvent(consumer, Map.of( "type", "error", "message", "Analysis failed due to I/O error: " + e.getMessage())); @@ -388,12 +377,62 @@ public Map process( return Map.of("status", "error", "message", e.getMessage()); } finally { + if (lockLease != null) { + try { + lockLease.close(); + } catch (RuntimeException closeFailure) { + log.info( + "PR analysis lock lease could not be closed after processing; " + + "continuing lock cleanup: project={}, PR={}, detail={}", + project.getId(), request.getPullRequestId(), + closeFailure.getMessage()); + } + } if (!isPreAcquired) { - analysisLockService.releaseLock(lockKey); + try { + analysisLockService.releaseLock(lockKey); + } catch (RuntimeException releaseFailure) { + // The analysis outcome is already decided. Cleanup failure + // is observational; the durable lock lease will expire. + log.info( + "PR analysis lock could not be released after processing; " + + "leaving it to expiry: project={}, PR={}, detail={}", + project.getId(), request.getPullRequestId(), + releaseFailure.getMessage()); + } } } } + private static void requireActiveLease(AnalysisLockService.LockLease lease) throws IOException { + if (lease == null || lease.isOwnershipLost()) { + throw lostLeaseException(); + } + } + + private static void requireConfirmedLease(AnalysisLockService.LockLease lease) throws IOException { + if (lease == null || !lease.confirmOwnership()) { + throw lostLeaseException(); + } + } + + private static IOException lostLeaseException() { + return new IOException("PR analysis lost its lock lease while the review worker was active"); + } + + /** Progress delivery is observational and cannot change review ownership or outcome. */ + private static void emitEvent(EventConsumer consumer, Map event) { + if (consumer == null) { + return; + } + try { + consumer.accept(event); + } catch (RuntimeException observerFailure) { + log.debug("PR progress observer rejected event type={} state={}: {}", + event.get("type"), event.get("state"), observerFailure.getMessage()); + } + } + private String taskContextValue(AiAnalysisRequest aiRequest, String... keys) { Map taskContext = aiRequest.getTaskContext(); if (taskContext == null || taskContext.isEmpty()) { @@ -494,7 +533,8 @@ private Map fetchFileContentsFromVcs(Project project, List changedFiles) { + String commitHash, List changedFiles, + AnalysisLockService.LockLease lockLease) throws IOException { try { // Strategy 1: Copy PR-level snapshots from the source analysis's original PR if (sourceAnalysis.getPrNumber() != null) { @@ -504,6 +544,7 @@ private void persistPrSnapshotsForCacheHit(PullRequest pullRequest, CodeAnalysis Map sourceContents = fileSnapshotService.getFileContentsMapForPr( sourcePr.get().getId()); if (!sourceContents.isEmpty()) { + requireConfirmedLease(lockLease); fileSnapshotService.persistSnapshotsForPr(pullRequest, cloned, sourceContents, commitHash); log.info("Copied {} PR snapshots from source PR {} to PR {} (cache hit)", sourceContents.size(), sourceAnalysis.getPrNumber(), pullRequest.getPrNumber()); @@ -524,9 +565,12 @@ private void persistPrSnapshotsForCacheHit(PullRequest pullRequest, CodeAnalysis if (!filePaths.isEmpty()) { Map fileContents = fetchFileContentsFromVcs(project, filePaths, commitHash); if (!fileContents.isEmpty()) { + requireConfirmedLease(lockLease); fileSnapshotService.persistSnapshotsForPr(pullRequest, cloned, fileContents, commitHash); } } + } catch (IOException ownershipFailure) { + throw ownershipFailure; } catch (Exception e) { log.warn("Failed to persist PR snapshots for cache hit (non-critical): {}", e.getMessage()); } @@ -538,9 +582,10 @@ protected boolean postDiffFingerprintCacheIfExist( Project project, PullRequest pullRequest, AiAnalysisRequest aiRequest, - VcsReportingService reportingService + VcsReportingService reportingService, + AnalysisLockService.LockLease lockLease - ) { + ) throws IOException { // Get analysis cache by diff fingerprint (any PR ID) - less ideal than commit hash but still a win if(diffFingerprint == null) { return false; @@ -554,14 +599,17 @@ protected boolean postDiffFingerprintCacheIfExist( "Diff fingerprint cache hit for project={}, fingerprint={} (source PR={}). Cloning for PR={}.", project.getId(), diffFingerprint.substring(0, 8) + "...", fingerprintHit.get().getPrNumber(), request.getPullRequestId()); + requireConfirmedLease(lockLease); CodeAnalysis cloned = codeAnalysisService.cloneAnalysisForPr( fingerprintHit.get(), project, request.getPullRequestId(), request.getCommitHash(), request.getTargetBranchName(), request.getSourceBranchName(), diffFingerprint); + requireConfirmedLease(lockLease); copyTaskImplementationEvidence(fingerprintHit.get(), cloned); // Persist PR-level snapshots for the source code viewer persistPrSnapshotsForCacheHit(pullRequest, cloned, fingerprintHit.get(), project, - request.getCommitHash(), aiRequest.getChangedFiles()); + request.getCommitHash(), aiRequest.getChangedFiles(), lockLease); + requireConfirmedLease(lockLease); try { reportingService.postAnalysisResults(cloned, project, request.getPullRequestId(), pullRequest.getId(), @@ -584,8 +632,9 @@ protected CacheHitType postAnalysisCacheIfExist( String placeholderCommentId, String targetBranch, String sourceBranch, - String expectedReviewIdentity - ) { + String expectedReviewIdentity, + AnalysisLockService.LockLease lockLease + ) throws IOException { Optional cachedAnalysis = codeAnalysisService.getCodeAnalysisCache( project.getId(), commitHash, @@ -595,6 +644,7 @@ protected CacheHitType postAnalysisCacheIfExist( if (cachedAnalysis.isPresent() && expectedReviewIdentity != null && expectedReviewIdentity.equals(cachedAnalysis.get().getDiffFingerprint())) { + requireConfirmedLease(lockLease); try { reportingService.postAnalysisResults(cachedAnalysis.get(), project, @@ -617,14 +667,17 @@ protected CacheHitType postAnalysisCacheIfExist( project.getId(), commitHash, commitHashHit.get().getPrNumber(), prId ); + requireConfirmedLease(lockLease); CodeAnalysis cloned = codeAnalysisService.cloneAnalysisForPr( commitHashHit.get(), project, prId, commitHash, targetBranch, sourceBranch, commitHashHit.get().getDiffFingerprint()); + requireConfirmedLease(lockLease); copyTaskImplementationEvidence(commitHashHit.get(), cloned); // Persist PR-level snapshots for the source code viewer persistPrSnapshotsForCacheHit(pullRequest, cloned, commitHashHit.get(), project, - commitHash, null); + commitHash, null, lockLease); + requireConfirmedLease(lockLease); try { reportingService.postAnalysisResults( cloned, @@ -884,41 +937,6 @@ static List selectPrEvidenceCommits( return selected; } - /** - * Ensures RAG index is up-to-date for the PR target branch. - *

- * For PRs targeting the main branch: - * - Checks if the main RAG index commit matches the current target branch HEAD - * - If outdated, performs incremental update before analysis - *

- * For PRs targeting non-main branches with multi-branch enabled: - * - First ensures the main index is up to date - * - Then ensures branch index exists and is up to date for the target branch - *

- * This ensures analysis always uses the most current codebase context. - */ - private void ensureRagIndexForTargetBranch(Project project, String targetBranch, EventConsumer consumer) { - if (ragOperationsService == null) { - log.debug("RagOperationsService not available - skipping RAG index check for target branch"); - return; - } - - try { - boolean ready = ragOperationsService.ensureRagIndexUpToDate( - project, - targetBranch, - consumer::accept); - if (ready) { - log.info("RAG index ensured up-to-date for PR target branch: project={}, branch={}", - project.getId(), targetBranch); - } - } catch (Exception e) { - log.warn( - "Failed to ensure RAG index up-to-date for target branch (non-critical): project={}, branch={}, error={}", - project.getId(), targetBranch, e.getMessage()); - } - } - /** * Publishes an AnalysisStartedEvent for PR analysis. */ diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java index c18a613e..3af6d182 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.analysisengine.service; +import jakarta.annotation.PreDestroy; import org.rostilos.codecrow.core.model.analysis.AnalysisLock; import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; import org.rostilos.codecrow.core.model.project.Project; @@ -23,6 +24,14 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @Service @@ -33,6 +42,7 @@ public class AnalysisLockService { private final AnalysisLockRepository lockRepository; private final TransactionTemplate requiresNewTransactionTemplate; private final String instanceId; + private final ScheduledExecutorService leaseHeartbeatExecutor; @Autowired @Lazy @@ -50,13 +60,29 @@ public class AnalysisLockService { @Value("${analysis.lock.wait.retry.interval.seconds:5}") private int lockWaitRetryIntervalSeconds; - public AnalysisLockService(AnalysisLockRepository lockRepository, PlatformTransactionManager transactionManager) { + @Value("${analysis.lock.heartbeat.interval.seconds:60}") + private int lockHeartbeatIntervalSeconds = 60; + + public AnalysisLockService( + AnalysisLockRepository lockRepository, + PlatformTransactionManager transactionManager) { + this(lockRepository, transactionManager, 4); + } + + @Autowired + public AnalysisLockService( + AnalysisLockRepository lockRepository, + PlatformTransactionManager transactionManager, + @Value("${analysis.lock.heartbeat.threads:4}") int heartbeatThreads) { this.lockRepository = lockRepository; this.instanceId = UUID.randomUUID().toString(); // Create a TransactionTemplate with REQUIRES_NEW propagation for fully isolated lock inserts this.requiresNewTransactionTemplate = new TransactionTemplate(transactionManager); this.requiresNewTransactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + this.leaseHeartbeatExecutor = Executors.newScheduledThreadPool( + Math.max(2, heartbeatThreads), + new LeaseHeartbeatThreadFactory()); log.info("AnalysisLockService initialized with instance ID: {}", instanceId); } @@ -162,12 +188,10 @@ public Optional acquireLockWithWait(Project project, String branchName, if (branchName == null || branchName.isBlank()) { log.error("Cannot acquire lock with wait: branchName is required but was null or blank. " + "project={}, lockType={}, prNumber={}", project.getId(), lockType, prNumber); - if (messageConsumer != null) { - messageConsumer.accept(Map.of( + emitLockEvent(messageConsumer, Map.of( "type", "error", "message", "Cannot start analysis: branch information is missing. Please ensure the PR has valid source branch information." )); - } return Optional.empty(); } @@ -183,8 +207,7 @@ public Optional acquireLockWithWait(Project project, String branchName, if (lockKey.isPresent()) { if (attemptCount > 1) { - if (messageConsumer != null) { - Map lockAcquiredMessage = Map.of( + Map lockAcquiredMessage = Map.of( "type", "lock_acquired", "message", String.format("Lock acquired after %d attempts (waited %d seconds)", attemptCount, @@ -192,8 +215,7 @@ public Optional acquireLockWithWait(Project project, String branchName, "lockType", lockType.name(), "branchName", branchName, "attemptCount", attemptCount); - messageConsumer.accept(lockAcquiredMessage); - } + emitLockEvent(messageConsumer, lockAcquiredMessage); log.info("Lock acquired after {} attempts (waited {} seconds)", attemptCount, Duration.between(startTime, OffsetDateTime.now()).getSeconds()); @@ -208,28 +230,18 @@ public Optional acquireLockWithWait(Project project, String branchName, "Lock acquisition attempt {} failed for project={}, branch={}, type={}. Waiting {} seconds before retry...", attemptCount, project.getId(), branchName, lockType, lockWaitRetryIntervalSeconds); - if (messageConsumer != null) { - try { - // Use HashMap instead of Map.of() to allow null values - Map lockWaitMessage = new java.util.HashMap<>(); - lockWaitMessage.put("type", "lock_wait"); - lockWaitMessage.put("message", - String.format("Waiting for lock release... (attempt %d, waited %ds, timeout in %ds)", - attemptCount, waitedSeconds, remainingSeconds)); - lockWaitMessage.put("lockType", lockType.name()); - lockWaitMessage.put("branchName", branchName); - lockWaitMessage.put("attemptCount", attemptCount); - lockWaitMessage.put("waitedSeconds", waitedSeconds); - lockWaitMessage.put("remainingSeconds", remainingSeconds); - log.debug("Sending lock wait message to consumer: {}", lockWaitMessage); - messageConsumer.accept(lockWaitMessage); - log.debug("Lock wait message sent successfully"); - } catch (Exception e) { - log.warn("Failed to send lock wait message: {}", e.getMessage(), e); - } - } else { - log.warn("Message consumer is null, cannot send lock wait message"); - } + // Use HashMap instead of Map.of() to allow null values + Map lockWaitMessage = new java.util.HashMap<>(); + lockWaitMessage.put("type", "lock_wait"); + lockWaitMessage.put("message", + String.format("Waiting for lock release... (attempt %d, waited %ds, timeout in %ds)", + attemptCount, waitedSeconds, remainingSeconds)); + lockWaitMessage.put("lockType", lockType.name()); + lockWaitMessage.put("branchName", branchName); + lockWaitMessage.put("attemptCount", attemptCount); + lockWaitMessage.put("waitedSeconds", waitedSeconds); + lockWaitMessage.put("remainingSeconds", remainingSeconds); + emitLockEvent(messageConsumer, lockWaitMessage); if (OffsetDateTime.now().plusSeconds(lockWaitRetryIntervalSeconds).isAfter(timeout)) { break; @@ -247,18 +259,12 @@ public Optional acquireLockWithWait(Project project, String branchName, log.warn("Failed to acquire lock after {} attempts and {} seconds (timeout exceeded)", attemptCount, Duration.between(startTime, OffsetDateTime.now()).getSeconds()); - if (messageConsumer != null) { - try { - messageConsumer.accept(Map.of( + emitLockEvent(messageConsumer, Map.of( "type", "lock_timeout", "message", "Failed to acquire lock: timeout exceeded", "lockType", lockType.name(), "branchName", branchName, "totalAttempts", attemptCount)); - } catch (Exception e) { - log.warn("Failed to send lock timeout message: {}", e.getMessage()); - } - } return Optional.empty(); } @@ -297,25 +303,170 @@ public boolean extendLock(String lockKey, int additionalMinutes) { return true; } - @Transactional public boolean renewLock(String lockKey, int leaseMinutes) { - Optional lockOpt = lockRepository.findByLockKey(lockKey); - if (lockOpt.isEmpty()) { - log.warn("Cannot renew lock - not found: {}", lockKey); + if (lockKey == null || lockKey.isBlank()) { return false; } - AnalysisLock lock = lockOpt.get(); - if (lock.isExpired()) { - log.warn("Cannot renew expired lock: {}", lockKey); - return false; + int boundedLeaseMinutes = Math.max(1, leaseMinutes); + OffsetDateTime now = OffsetDateTime.now(); + OffsetDateTime expiresAt = now.plusMinutes(boundedLeaseMinutes); + Integer updated = requiresNewTransactionTemplate.execute(status -> + lockRepository.renewActiveLock(lockKey, now, expiresAt)); + boolean renewed = updated != null && updated > 0; + if (renewed) { + log.debug("Renewed lock {} for {} minutes", lockKey, boundedLeaseMinutes); + } else { + // The lease owner records the single contextual ownership-loss + // diagnostic. Keep this primitive reusable without double logging. + log.debug("Cannot renew missing or expired lock: {}", lockKey); } + return renewed; + } + /** + * Maintains a long-running analysis lease independently of worker progress + * events. A quiet but healthy worker must not lose ownership merely because + * it has no streaming message to emit. + */ + public LockLease maintainLockLease(String lockKey, int leaseMinutes) { int boundedLeaseMinutes = Math.max(1, leaseMinutes); - lock.setExpiresAt(OffsetDateTime.now().plusMinutes(boundedLeaseMinutes)); - lockRepository.save(lock); - log.debug("Renewed lock {} for {} minutes", lockKey, boundedLeaseMinutes); - return true; + long leaseSeconds = TimeUnit.MINUTES.toSeconds(boundedLeaseMinutes); + long intervalSeconds = Math.max(1L, Math.min( + Math.max(1, lockHeartbeatIntervalSeconds), + Math.max(1L, leaseSeconds / 3L))); + ActiveLockLease lease = new ActiveLockLease(lockKey, boundedLeaseMinutes); + lease.renew(); + lease.schedule(intervalSeconds); + return lease; + } + + public interface LockLease extends AutoCloseable { + boolean isOwnershipLost(); + + /** Performs a final atomic ownership proof before durable publication. */ + boolean confirmOwnership(); + + @Override + void close(); + } + + private final class ActiveLockLease implements LockLease { + private final String lockKey; + private final int leaseMinutes; + private final AtomicBoolean ownershipLost = new AtomicBoolean(false); + private final AtomicBoolean closed = new AtomicBoolean(false); + private final AtomicBoolean transientFailureReported = new AtomicBoolean(false); + private final AtomicReference knownExpiresAt; + private volatile ScheduledFuture heartbeat; + + private ActiveLockLease(String lockKey, int leaseMinutes) { + this.lockKey = lockKey; + this.leaseMinutes = leaseMinutes; + // A pre-acquired lock may already be old. Do not infer a fresh lease + // window until the first atomic renewal has actually succeeded. + this.knownExpiresAt = new AtomicReference<>(); + } + + private void schedule(long intervalSeconds) { + heartbeat = leaseHeartbeatExecutor.scheduleWithFixedDelay( + this::renew, intervalSeconds, intervalSeconds, TimeUnit.SECONDS); + } + + private boolean renew() { + if (closed.get() || ownershipLost.get()) { + return false; + } + OffsetDateTime renewalStartedAt = OffsetDateTime.now(); + try { + if (!renewLock(lockKey, leaseMinutes)) { + ownershipLost.set(true); + log.error("Analysis lock lease ownership was lost: {}", lockKey); + return false; + } + knownExpiresAt.set( + renewalStartedAt.plusMinutes(leaseMinutes).minusSeconds(1)); + if (transientFailureReported.compareAndSet(true, false)) { + log.info("Analysis lock heartbeat recovered: {}", lockKey); + } + return true; + } catch (RuntimeException renewalFailure) { + // A database outage is not evidence that ownership changed. The + // previous successful lease remains authoritative until its known + // expiry, and the scheduled heartbeat will retry meanwhile. + OffsetDateTime knownExpiry = knownExpiresAt.get(); + boolean previousLeaseStillActive = knownExpiry != null + && OffsetDateTime.now().isBefore(knownExpiry); + if (!previousLeaseStillActive) { + ownershipLost.set(true); + log.error("Analysis lock ownership could not be confirmed before its known expiry: {}", + lockKey, renewalFailure); + return false; + } + if (transientFailureReported.compareAndSet(false, true)) { + log.warn("Analysis lock renewal failed; the prior lease is still active " + + "and renewal will retry: {}", + lockKey, renewalFailure); + } else { + log.debug("Analysis lock heartbeat remains degraded within the active lease: {}", + lockKey); + } + return true; + } + } + + @Override + public boolean isOwnershipLost() { + return ownershipLost.get(); + } + + @Override + public boolean confirmOwnership() { + return renew(); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + ScheduledFuture scheduled = heartbeat; + if (scheduled != null) { + scheduled.cancel(false); + } + } + } + + @PreDestroy + void shutdownLeaseHeartbeatExecutor() { + leaseHeartbeatExecutor.shutdownNow(); + } + + private static final class LeaseHeartbeatThreadFactory implements ThreadFactory { + private final AtomicInteger threadNumber = new AtomicInteger(); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread( + runnable, + "analysis-lock-heartbeat-" + threadNumber.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + } + + private static void emitLockEvent( + Consumer> messageConsumer, + Map event) { + if (messageConsumer == null) { + return; + } + try { + messageConsumer.accept(event); + } catch (RuntimeException observerFailure) { + log.debug("Analysis lock observer rejected event type={}: {}", + event.get("type"), observerFailure.getMessage()); + } } @Transactional(readOnly = true) diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java index 6718f620..08d5069e 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java @@ -29,6 +29,7 @@ public class BranchAnalysisGateService { private static final Logger log = LoggerFactory.getLogger(BranchAnalysisGateService.class); + private static final long WAIT_STATUS_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(60); private final JobRepository jobRepository; @@ -90,6 +91,7 @@ public GateResult awaitTurn( Consumer> consumer) { long timeoutNanos = TimeUnit.MINUTES.toNanos(Math.max(1, waitTimeoutMinutes)); long startedAt = System.nanoTime(); + long nextStatusAt = startedAt; while (true) { if (currentBranchJobId != null && jobRepository.existsNewerBranchAnalysisJob( @@ -113,7 +115,11 @@ public GateResult awaitTurn( AnalysisLockType.PR_ANALYSIS.name(), branchName, projectId); } - emitWait(consumer, branchName, sourcePrNumber, waitedNanos); + long now = System.nanoTime(); + if (now >= nextStatusAt) { + emitWait(consumer, branchName, sourcePrNumber, waitedNanos); + nextStatusAt = now + WAIT_STATUS_INTERVAL_NANOS; + } if (!pause()) { throw new AnalysisLockedException( AnalysisLockType.PR_ANALYSIS.name(), branchName, projectId); @@ -158,6 +164,7 @@ public void awaitOlderBranchAnalyses( long timeoutNanos = TimeUnit.MINUTES.toNanos(Math.max(1, waitTimeoutMinutes)); long startedAt = System.nanoTime(); + long nextStatusAt = startedAt; while (jobRepository.existsActiveBranchAnalysisJobBefore( projectId, branchName, currentPrJobId)) { long waitedNanos = System.nanoTime() - startedAt; @@ -169,7 +176,11 @@ public void awaitOlderBranchAnalyses( AnalysisLockType.BRANCH_ANALYSIS.name(), branchName, projectId); } - emitBranchWait(consumer, branchName, waitedNanos); + long now = System.nanoTime(); + if (now >= nextStatusAt) { + emitBranchWait(consumer, branchName, waitedNanos); + nextStatusAt = now + WAIT_STATUS_INTERVAL_NANOS; + } if (!pause()) { throw new AnalysisLockedException( AnalysisLockType.BRANCH_ANALYSIS.name(), branchName, projectId); @@ -212,9 +223,13 @@ private void emitWait( event.put("type", "pr_analysis_wait"); event.put("state", "waiting_for_pr_analysis"); event.put("message", sourcePrNumber == null - ? "Waiting for earlier PR analyses targeting " + branchName + " to finish" - : "Waiting for PR #" + sourcePrNumber + " analysis targeting " - + branchName + " to finish"); + ? "Branch analysis for " + branchName + + " is waiting for earlier PR analyses targeting that branch" + : "Post-merge branch analysis for " + branchName + + " is waiting for PR #" + sourcePrNumber + + " analysis to finish"); + event.put("waitingJobType", JobType.BRANCH_ANALYSIS.name()); + event.put("blockingJobType", JobType.PR_ANALYSIS.name()); event.put("branchName", branchName); event.put("waitedSeconds", TimeUnit.NANOSECONDS.toSeconds(waitedNanos)); if (sourcePrNumber != null) { @@ -237,8 +252,10 @@ private void emitBranchWait( consumer.accept(Map.of( "type", "branch_analysis_wait", "state", "waiting_for_target_branch", - "message", "Waiting for the earlier " + branchName - + " update and its RAG publication to finish", + "message", "PR analysis targeting " + branchName + + " is waiting for the earlier branch update and its RAG publication", + "waitingJobType", JobType.PR_ANALYSIS.name(), + "blockingJobType", JobType.BRANCH_ANALYSIS.name(), "branchName", branchName, "waitedSeconds", TimeUnit.NANOSECONDS.toSeconds(waitedNanos))); } catch (Exception e) { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java index 69ed3606..07e64b7f 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java @@ -323,6 +323,8 @@ void shouldIncludeSourceAndTargetBranchNamesInQueuedRequestPayload() throws Exce void shouldBindExactTargetBranchGenerationToQueuedReview() throws Exception { var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag .RagBranchIndexGenerationRepository.class); + var branchRepository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexRepository.class); var generation = mock(org.rostilos.codecrow.core.model.rag .RagBranchIndexGeneration.class); when(generation.getCollectionName()).thenReturn("opaque-master-generation"); @@ -332,6 +334,10 @@ void shouldBindExactTargetBranchGenerationToQueuedReview() throws Exception { .thenReturn(List.of(generation)); org.springframework.test.util.ReflectionTestUtils.setField( client, "branchGenerationRepository", repository); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchIndexRepository", branchRepository); + when(branchRepository.markAccessedIfUnclaimed( + eq(1L), eq("main"), any())).thenReturn(1); AiAnalysisRequest exactRequest = new TestAiAnalysisRequest() { @Override public String getBaseCommitHash() { @@ -357,6 +363,81 @@ public String getBaseCommitHash() { assertThat(requestPayload) .containsEntry("ragCollectionTarget", "opaque-master-generation") .containsEntry("ragBaseGenerationManifestSha256", "master-manifest"); + verify(branchRepository).markAccessedIfUnclaimed( + eq(1L), eq("main"), any()); + } + + @Test + @DisplayName("should not bind a generation claimed by transient cleanup") + void shouldNotBindCleanupClaimedGeneration() throws Exception { + var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexGenerationRepository.class); + var branchRepository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexRepository.class); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchGenerationRepository", repository); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchIndexRepository", branchRepository); + when(branchRepository.markAccessedIfUnclaimed( + eq(1L), eq("main"), any())).thenReturn(0); + AiAnalysisRequest exactRequest = new TestAiAnalysisRequest() { + @Override + public String getBaseCommitHash() { + return "master-base"; + } + }; + when(queueService.rightPop(anyString(), anyLong())) + .thenReturn(objectMapper.writeValueAsString(Map.of( + "type", "final", + "result", Map.of( + "comment", "ok", + "issues", List.of())))); + + client.performAnalysis(exactRequest); + + verifyNoInteractions(repository); + } + + @Test + @DisplayName("should not look up or bind a RAG generation when RAG is disabled") + void shouldNotBindRagGenerationWhenDisabled() throws Exception { + var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexGenerationRepository.class); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchGenerationRepository", repository); + AiAnalysisRequest disabledRequest = new TestAiAnalysisRequest() { + @Override + public boolean getRagEnabled() { + return false; + } + + @Override + public String getBaseCommitHash() { + return "master-base"; + } + }; + Map finalEvent = Map.of( + "type", "final", + "result", Map.of("comment", "ok", "issues", List.of())); + when(queueService.rightPop(anyString(), anyLong())) + .thenReturn(objectMapper.writeValueAsString(finalEvent)); + + client.performAnalysis(disabledRequest); + + var payloadCaptor = org.mockito.ArgumentCaptor.forClass(String.class); + verify(queueService).leftPush(eq("codecrow:analysis:jobs"), payloadCaptor.capture()); + @SuppressWarnings("unchecked") + Map queued = objectMapper.readValue( + payloadCaptor.getValue(), Map.class); + @SuppressWarnings("unchecked") + Map requestPayload = + (Map) queued.get("request"); + assertThat(requestPayload) + .containsEntry("ragEnabled", false) + .doesNotContainKeys( + "ragCollectionTarget", + "ragBaseGenerationManifestSha256"); + verifyNoInteractions(repository); } @Test diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java index a187e6b8..e3923ed6 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java @@ -10,6 +10,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.rostilos.codecrow.queue.RedisQueueService; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; import java.io.IOException; import java.util.HashMap; @@ -36,6 +37,28 @@ class AiCommandClientTest { void setUp() { objectMapper = new ObjectMapper(); client = new AiCommandClient(queueService, objectMapper); + lenient().when(queueService.hasKey(AiCommandClient.CONSUMER_HEARTBEAT_KEY)) + .thenReturn(true); + } + + @Test + @DisplayName("should expose an unambiguous Spring injection constructor") + void shouldConstructThroughSpringContext() { + try (AnnotationConfigApplicationContext context = + new AnnotationConfigApplicationContext()) { + context.registerBean( + RedisQueueService.class, + () -> queueService); + context.registerBean( + ObjectMapper.class, + () -> objectMapper); + context.register(AiCommandClient.class); + + context.refresh(); + + assertThat(context.getBean(AiCommandClient.class)) + .isNotNull(); + } } private AiCommandClient.SummarizeRequest createSummarizeRequest() { @@ -84,6 +107,8 @@ void shouldSuccessfullySummarizePR() throws Exception { assertThat(result.diagramType()).isEqualTo("MERMAID"); verify(queueService).leftPush(eq("codecrow:queue:commands"), anyString()); + verify(queueService).removeFromList( + eq("codecrow:queue:commands"), anyString()); verify(queueService).setExpiry(anyString(), anyLong()); verify(queueService).deleteKey(anyString()); } @@ -126,6 +151,57 @@ void shouldDefaultNullSummarizeResultFields() throws Exception { } } + @Test + @DisplayName("should reject commands before enqueue when no consumer is alive") + void shouldRejectWhenConsumerIsUnavailable() { + when(queueService.hasKey(AiCommandClient.CONSUMER_HEARTBEAT_KEY)) + .thenReturn(false); + + assertThatThrownBy(() -> client.summarize(createSummarizeRequest(), null)) + .isInstanceOf(IOException.class) + .hasMessageContaining("no live command queue consumer"); + + verify(queueService, never()).leftPush(anyString(), anyString()); + } + + @Test + @DisplayName("should remove an unadmitted command when the consumer disappears") + void shouldRemovePendingPayloadWhenConsumerDisappears() { + AiCommandClient supervisedClient = new AiCommandClient( + queueService, objectMapper, 1_000L, 1_000L); + when(queueService.hasKey(AiCommandClient.CONSUMER_HEARTBEAT_KEY)) + .thenReturn(true, true, false); + when(queueService.rightPop(anyString(), anyLong())).thenReturn(null); + + assertThatThrownBy(() -> + supervisedClient.summarize(createSummarizeRequest(), null)) + .isInstanceOf(IOException.class) + .hasMessageContaining("became unavailable before the command job was admitted"); + + verify(queueService).removeFromList( + eq("codecrow:queue:commands"), anyString()); + } + + @Test + @DisplayName("should remove a command that exceeds bounded admission wait") + void shouldRemovePayloadWhenAdmissionTimesOut() { + AiCommandClient supervisedClient = new AiCommandClient( + queueService, objectMapper, 1_000L, 10L); + when(queueService.rightPop(anyString(), anyLong())) + .thenAnswer(invocation -> { + Thread.sleep(12L); + return null; + }); + + assertThatThrownBy(() -> + supervisedClient.summarize(createSummarizeRequest(), null)) + .isInstanceOf(IOException.class) + .hasMessageContaining("was not admitted by Inference Orchestrator"); + + verify(queueService).removeFromList( + eq("codecrow:queue:commands"), anyString()); + } + @Nested @DisplayName("ask()") class AskTests { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java index 5781c71f..8ba074a3 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java @@ -76,6 +76,9 @@ class BranchAnalysisProcessorTest { @Mock private AnalysisLockService analysisLockService; + @Mock + private AnalysisLockService.LockLease lockLease; + @Mock private BranchAnalysisGateService branchAnalysisGateService; @@ -144,6 +147,11 @@ class BranchAnalysisProcessorTest { @BeforeEach void setUp() { when(vcsClientProvider.getClient(vcsConnection)).thenReturn(authorizedClient); + lenient().when(analysisLockService.getLeaseMinutes(any())) + .thenReturn(30); + lenient().when(analysisLockService.maintainLockLease(anyString(), anyInt())) + .thenReturn(lockLease); + lenient().when(lockLease.confirmOwnership()).thenReturn(true); processor = new BranchAnalysisProcessor( projectService, branchRepository, @@ -349,6 +357,76 @@ void shouldSkipWhenCommitAlreadyAnalyzed() throws IOException { assertThat(result).containsEntry("status", "skipped"); assertThat(result).containsEntry("reason", "commit_already_analyzed"); + verify(analysisLockService).maintainLockLease("lock-key", 30); + verify(lockLease).close(); + verify(analysisLockService).releaseLock("lock-key"); + } + + @Test + @DisplayName("cleanup failures cannot override a successful branch outcome") + void cleanupFailuresCannotOverrideSuccessfulBranchOutcome() throws IOException { + BranchProcessRequest request = createRequest(); + when(projectService.getProjectWithConnections(1L)).thenReturn(project); + when(project.getId()).thenReturn(1L); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), any(), any())) + .thenReturn(Optional.of("lock-key")); + Branch existingBranch = new Branch(); + existingBranch.setLastSuccessfulCommitHash("abc123"); + when(branchRepository.findByProjectIdAndBranchName(1L, "main")) + .thenReturn(Optional.of(existingBranch)); + when(branchFileOperationsService.getBranchFilePaths(1L, "main")) + .thenReturn(Collections.emptySet()); + doThrow(new IllegalStateException("lease executor unavailable")) + .when(lockLease).close(); + doThrow(new IllegalStateException("lock database unavailable")) + .when(analysisLockService).releaseLock("lock-key"); + + Map result = processor.process(request, null); + + assertThat(result).containsEntry("status", "skipped"); + assertThat(result).containsEntry("reason", "commit_already_analyzed"); + verify(lockLease).close(); + verify(analysisLockService).releaseLock("lock-key"); + } + + @Test + @DisplayName("should fence cached snapshot publication when branch lease is lost") + void shouldFenceCachedSnapshotPublicationWhenLeaseIsLost() throws IOException { + BranchProcessRequest request = createRequest(); + + when(projectService.getProjectWithConnections(1L)).thenReturn(project); + when(project.getId()).thenReturn(1L); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), any(), any())) + .thenReturn(Optional.of("lock-key")); + when(lockLease.confirmOwnership()).thenReturn(true, false); + + Branch existingBranch = new Branch(); + existingBranch.setLastSuccessfulCommitHash("abc123"); + when(branchRepository.findByProjectIdAndBranchName(1L, "main")) + .thenReturn(Optional.of(existingBranch)); + + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(repoInfo.getRepoWorkspace()).thenReturn("ws"); + when(repoInfo.getRepoSlug()).thenReturn("repo"); + when(branchFileOperationsService.getBranchFilePaths(1L, "main")) + .thenReturn(new HashSet<>(Set.of("src/App.java"))); + when(branchFileOperationsService.downloadBranchFileSnapshot( + any(), eq("abc123"), anySet())) + .thenReturn(archiveSnapshot(Map.of("src/App.java", "content"))); + + assertThatThrownBy(() -> processor.process(request, null)) + .isInstanceOf(IOException.class) + .hasMessageContaining("lost its lock lease"); + + verify(branchFileOperationsService, never()).updateFileSnapshotsForBranch( + anySet(), any(), any(), + isA(BranchFileOperationsService.BranchFileSnapshot.class)); + verify(branchHealthService, never()).handleProcessFailure(any(), any(), anyList(), any()); + verify(lockLease).close(); verify(analysisLockService).releaseLock("lock-key"); } diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java index 13668ba2..76523a4b 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java @@ -15,7 +15,6 @@ import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.analysisengine.service.PullRequestService; import org.rostilos.codecrow.commitgraph.service.AnalyzedCommitService; -import org.rostilos.codecrow.analysisengine.service.rag.RagOperationsService; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.rostilos.codecrow.analysisengine.service.vcs.VcsAiClientService; import org.rostilos.codecrow.analysisengine.service.vcs.VcsReportingService; @@ -68,6 +67,9 @@ class PullRequestAnalysisProcessorTest { @Mock private AnalysisLockService analysisLockService; + @Mock + private AnalysisLockService.LockLease lockLease; + @Mock private AnalyzedCommitService analyzedCommitService; @@ -83,9 +85,6 @@ class PullRequestAnalysisProcessorTest { @Mock private AstScopeEnricher astScopeEnricher; - @Mock - private RagOperationsService ragOperationsService; - @Mock private ApplicationEventPublisher eventPublisher; @@ -121,6 +120,12 @@ void setUp() { .thenReturn(TaskImplementationEvidenceService.PersistenceResult.empty()); lenient().when(taskImplementationEvidenceService.copyForAnalysis(any(), any())) .thenReturn(TaskImplementationEvidenceService.PersistenceResult.empty()); + lenient().when(analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS)) + .thenReturn(30); + lenient().when(analysisLockService.maintainLockLease(anyString(), eq(30))) + .thenReturn(lockLease); + lenient().when(lockLease.isOwnershipLost()).thenReturn(false); + lenient().when(lockLease.confirmOwnership()).thenReturn(true); processor = new PullRequestAnalysisProcessor( pullRequestService, codeAnalysisService, @@ -133,7 +138,6 @@ void setUp() { fileSnapshotService, prIssueTrackingService, astScopeEnricher, - ragOperationsService, eventPublisher); } @@ -253,6 +257,43 @@ void shouldPersistProviderCanonicalizedFullPrHead() throws Exception { verify(analysisLockService).releaseLock("lock-key-123"); } + @Test + @DisplayName("cleanup failures cannot override a successful PR outcome") + void cleanupFailuresCannotOverrideSuccessfulPrOutcome() throws Exception { + PrProcessRequest request = createRequest(); + PullRequestAnalysisProcessor.EventConsumer consumer = mock( + PullRequestAnalysisProcessor.EventConsumer.class); + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(project.getId()).thenReturn(1L); + when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenReturn(Optional.of("lock-key-123")); + when(vcsServiceFactory.getReportingService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(reportingService); + when(vcsServiceFactory.getAiClientService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(aiClientService); + when(codeAnalysisService.getAllPrAnalyses(anyLong(), anyLong())) + .thenReturn(List.of()); + when(pullRequestService.createOrUpdatePullRequest( + anyLong(), anyLong(), anyString(), anyString(), anyString(), any())) + .thenReturn(pullRequest); + when(aiClientService.buildAiAnalysisRequests(any(), any(), any(), anyList())) + .thenReturn(List.of()); + doThrow(new IllegalStateException("lease executor unavailable")) + .when(lockLease).close(); + doThrow(new IllegalStateException("lock database unavailable")) + .when(analysisLockService).releaseLock("lock-key-123"); + + Map result = processor.process(request, consumer, project); + + assertThat(result).containsEntry("status", "ignored"); + verify(lockLease).close(); + verify(analysisLockService).releaseLock("lock-key-123"); + } + @Test @DisplayName("should successfully process PR analysis") void shouldSuccessfullyProcessPRAnalysis() throws Exception { @@ -401,10 +442,6 @@ void shouldNotPersistOrPublishPromptDryRun() throws Exception { "filename", "capture.json", "containerPath", "/app/logs/prompt-dry-runs/capture.json")); - when(analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS)) - .thenReturn(30); - when(analysisLockService.renewLock("lock-key-123", 30)) - .thenReturn(true); when(aiAnalysisClient.performAnalysis(any(), any())) .thenAnswer(invocation -> { @SuppressWarnings("unchecked") @@ -426,13 +463,15 @@ void shouldNotPersistOrPublishPromptDryRun() throws Exception { verify(codeAnalysisService, never()).createAnalysisFromAiResponse( any(), any(), anyLong(), anyString(), anyString(), anyString(), any(), any(), any(), any(), any(), any()); - verify(analysisLockService).renewLock("lock-key-123", 30); + verify(analysisLockService).maintainLockLease("lock-key-123", 30); + verify(lockLease, times(2)).confirmOwnership(); + verify(lockLease).close(); verify(reportingService, never()).postAnalysisResults( any(), any(), anyLong(), any(), any()); } @Test - @DisplayName("should reject a completed review after lock lease ownership is lost") + @DisplayName("should reject a quiet completed review when the independent lease heartbeat lost ownership") void shouldRejectCompletedReviewAfterLockLeaseIsLost() throws Exception { PrProcessRequest request = createRequest(); PullRequestAnalysisProcessor.EventConsumer consumer = mock( @@ -458,22 +497,10 @@ void shouldRejectCompletedReviewAfterLockLeaseIsLost() throws Exception { .thenReturn(List.of(aiAnalysisRequest)); when(aiAnalysisRequest.getRawDiff()).thenReturn("diff"); when(aiAnalysisRequest.getChangedFiles()).thenReturn(List.of("file.java")); - when(analysisLockService.getLeaseMinutes(AnalysisLockType.PR_ANALYSIS)) - .thenReturn(30); - when(analysisLockService.renewLock("lock-key-123", 30)) - .thenReturn(false); - when(aiAnalysisClient.performAnalysis(any(), any())) - .thenAnswer(invocation -> { - @SuppressWarnings("unchecked") - java.util.function.Consumer> eventHandler = - invocation.getArgument(1); - eventHandler.accept(Map.of( - "type", "status", - "state", "processing")); - return Map.of( - "comment", "must not be published", - "issues", List.of()); - }); + when(lockLease.confirmOwnership()).thenReturn(true, false); + when(aiAnalysisClient.performAnalysis(any(), any())).thenReturn(Map.of( + "comment", "must not be published", + "issues", List.of())); Map result = processor.process(request, consumer, project); @@ -487,9 +514,263 @@ void shouldRejectCompletedReviewAfterLockLeaseIsLost() throws Exception { any(), any(), any(), any(), any(), any()); verify(reportingService, never()).postAnalysisResults( any(), any(), anyLong(), any(), any()); + verify(analysisLockService).maintainLockLease("lock-key-123", 30); + verify(lockLease, times(2)).confirmOwnership(); + verify(lockLease).close(); verify(analysisLockService).releaseLock("lock-key-123"); } + @Test + @DisplayName("should re-confirm ownership after direct VCS fallback before persistence") + void shouldRejectLeaseLostDuringDirectVcsFallbackBeforePersistence() throws Exception { + PrProcessRequest request = createRequest(); + PullRequestAnalysisProcessor.EventConsumer consumer = mock( + PullRequestAnalysisProcessor.EventConsumer.class); + Map aiResponse = Map.of( + "comment", "must not be persisted", + "issues", List.of()); + stubReviewThroughAi(aiResponse); + when(lockLease.confirmOwnership()).thenReturn(true, true, false); + + Map result = processor.process(request, consumer, project); + + assertThat(result).containsEntry("status", "error"); + verify(lockLease, times(3)).confirmOwnership(); + verify(codeAnalysisService, never()).createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any()); + verify(reportingService, never()).postAnalysisResults( + any(), any(), anyLong(), any(), any()); + } + + @Test + @DisplayName("should atomically re-confirm ownership before VCS publication") + void shouldRejectLeaseLostBeforeVcsPublication() throws Exception { + PrProcessRequest request = createRequest(); + PullRequestAnalysisProcessor.EventConsumer consumer = mock( + PullRequestAnalysisProcessor.EventConsumer.class); + Map aiResponse = Map.of( + "comment", "must not be published", + "issues", List.of()); + stubReviewThroughAi(aiResponse); + when(lockLease.confirmOwnership()).thenReturn(true, true, true, false); + when(codeAnalysisService.createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any())) + .thenReturn(codeAnalysis); + + Map result = processor.process(request, consumer, project); + + assertThat(result).containsEntry("status", "error"); + verify(lockLease, times(4)).confirmOwnership(); + verify(codeAnalysisService).createAnalysisFromAiResponse( + any(), eq(aiResponse), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any()); + verify(reportingService, never()).postAnalysisResults( + any(), any(), anyLong(), any(), any()); + } + + private void stubReviewThroughAi(Map aiResponse) throws Exception { + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(project.getId()).thenReturn(1L); + when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenReturn(Optional.of("lock-key-123")); + when(pullRequestService.createOrUpdatePullRequest( + anyLong(), anyLong(), anyString(), anyString(), anyString(), any())) + .thenReturn(pullRequest); + when(vcsServiceFactory.getReportingService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(reportingService); + when(vcsServiceFactory.getAiClientService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(aiClientService); + when(codeAnalysisService.getAllPrAnalyses(anyLong(), anyLong())) + .thenReturn(List.of()); + when(aiClientService.buildAiAnalysisRequests(any(), any(), any(), anyList())) + .thenReturn(List.of(aiAnalysisRequest)); + when(aiAnalysisRequest.getRawDiff()).thenReturn("diff"); + when(aiAnalysisRequest.getChangedFiles()).thenReturn(List.of("file.java")); + when(aiAnalysisClient.performAnalysis(any(), any())).thenReturn(aiResponse); + } + + private String stubCacheLookupPrerequisites() throws Exception { + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(project.getId()).thenReturn(1L); + when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenReturn(Optional.of("lock-key-cache")); + when(vcsServiceFactory.getReportingService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(reportingService); + when(vcsServiceFactory.getAiClientService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(aiClientService); + when(codeAnalysisService.getAllPrAnalyses(anyLong(), anyLong())) + .thenReturn(List.of()); + when(aiClientService.buildAiAnalysisRequests(any(), any(), any(), anyList())) + .thenReturn(List.of(aiAnalysisRequest)); + when(pullRequestService.createOrUpdatePullRequest( + anyLong(), anyLong(), anyString(), anyString(), anyString(), any())) + .thenReturn(pullRequest); + when(aiAnalysisRequest.getRawDiff()).thenReturn( + "diff --git a/file.java b/file.java\n@@ -1 +1 @@\n-old\n+new\n"); + when(aiAnalysisRequest.getChangedFiles()).thenReturn(List.of("file.java")); + return processor.computeReviewIdentity(aiAnalysisRequest); + } + + @Test + @DisplayName("should fence the first durable PR write after request construction") + void shouldRejectLeaseLostBeforePullRequestPersistence() throws Exception { + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(project.getId()).thenReturn(1L); + when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenReturn(Optional.of("lock-key-before-pr")); + when(vcsServiceFactory.getReportingService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(reportingService); + when(vcsServiceFactory.getAiClientService(EVcsProvider.BITBUCKET_CLOUD)) + .thenReturn(aiClientService); + when(codeAnalysisService.getAllPrAnalyses(anyLong(), anyLong())) + .thenReturn(List.of()); + when(aiClientService.buildAiAnalysisRequests(any(), any(), any(), anyList())) + .thenReturn(List.of(aiAnalysisRequest)); + when(lockLease.confirmOwnership()).thenReturn(false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(pullRequestService, never()).createOrUpdatePullRequest( + anyLong(), anyLong(), anyString(), anyString(), anyString(), any()); + } + + @Test + @DisplayName("should fence exact cache publication") + void shouldRejectLeaseLostBeforeExactCachePublication() throws Exception { + String identity = stubCacheLookupPrerequisites(); + when(codeAnalysisService.getCodeAnalysisCache(1L, "abc123", 42L)) + .thenReturn(Optional.of(codeAnalysis)); + when(codeAnalysis.getDiffFingerprint()).thenReturn(identity); + when(lockLease.confirmOwnership()).thenReturn(true, false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(reportingService, never()).postAnalysisResults( + any(), any(), anyLong(), any(), any()); + } + + @Test + @DisplayName("should fence commit cache cloning") + void shouldRejectLeaseLostBeforeCommitCacheClone() throws Exception { + String identity = stubCacheLookupPrerequisites(); + when(codeAnalysisService.getCodeAnalysisCache(1L, "abc123", 42L)) + .thenReturn(Optional.empty()); + CodeAnalysis source = mock(CodeAnalysis.class); + when(source.getDiffFingerprint()).thenReturn(identity); + when(source.getPrNumber()).thenReturn(99L); + when(codeAnalysisService.getAnalysisByCommitHash(1L, "abc123")) + .thenReturn(Optional.of(source)); + when(lockLease.confirmOwnership()).thenReturn(true, false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(codeAnalysisService, never()).cloneAnalysisForPr( + any(), any(), anyLong(), anyString(), anyString(), anyString(), anyString()); + } + + @Test + @DisplayName("should fence fingerprint cache cloning") + void shouldRejectLeaseLostBeforeFingerprintCacheClone() throws Exception { + stubCacheLookupPrerequisites(); + when(codeAnalysisService.getCodeAnalysisCache(1L, "abc123", 42L)) + .thenReturn(Optional.empty()); + when(codeAnalysisService.getAnalysisByCommitHash(1L, "abc123")) + .thenReturn(Optional.empty()); + when(codeAnalysisService.getAnalysisByDiffFingerprint(eq(1L), anyString())) + .thenReturn(Optional.of(mock(CodeAnalysis.class))); + when(lockLease.confirmOwnership()).thenReturn(true, false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(codeAnalysisService, never()).cloneAnalysisForPr( + any(), any(), anyLong(), anyString(), anyString(), anyString(), anyString()); + } + + @Test + @DisplayName("should re-confirm after VCS publication before commit receipts") + void shouldRejectLeaseLostAfterVcsPublicationBeforeCommitReceipts() throws Exception { + Map aiResponse = Map.of("comment", "review", "issues", List.of()); + stubReviewThroughAi(aiResponse); + when(codeAnalysisService.createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any())) + .thenReturn(codeAnalysis); + when(lockLease.confirmOwnership()).thenReturn(true, true, true, true, false); + + Map result = processor.process( + createRequest(), mock(PullRequestAnalysisProcessor.EventConsumer.class), project); + + assertThat(result).containsEntry("status", "error"); + verify(reportingService).postAnalysisResults( + eq(codeAnalysis), any(), anyLong(), any(), any()); + verifyNoInteractions(analyzedCommitService); + } + + @Test + @DisplayName("observer failure cannot turn a successful review into failure") + void observerFailureDoesNotChangeReviewOutcome() throws Exception { + Map aiResponse = Map.of("comment", "review", "issues", List.of()); + stubReviewThroughAi(aiResponse); + when(codeAnalysisService.createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any())) + .thenReturn(codeAnalysis); + PullRequestAnalysisProcessor.EventConsumer observer = mock( + PullRequestAnalysisProcessor.EventConsumer.class); + doThrow(new IllegalStateException("observer disconnected")) + .when(observer).accept(anyMap()); + when(analysisLockService.acquireLockWithWait( + any(), anyString(), any(), anyString(), anyLong(), any())) + .thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + java.util.function.Consumer> progress = + invocation.getArgument(5); + progress.accept(Map.of( + "type", "lock_wait", + "message", "waiting")); + return Optional.of("lock-key-123"); + }); + when(aiAnalysisClient.performAnalysis(any(), any())).thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + java.util.function.Consumer> progress = + invocation.getArgument(1); + progress.accept(Map.of( + "type", "status", + "state", "processing")); + return aiResponse; + }); + + Map result = processor.process(createRequest(), observer, project); + + assertThat(result).isEqualTo(aiResponse); + verify(reportingService).postAnalysisResults( + eq(codeAnalysis), any(), anyLong(), any(), any()); + verify(observer).accept(argThat(event -> "lock_wait".equals(event.get("type")))); + verify(observer).accept(argThat(event -> "processing".equals(event.get("state")))); + } + @Test @DisplayName("should throw AnalysisLockedException when lock cannot be acquired") void shouldThrowAnalysisLockedExceptionWhenLockCannotBeAcquired() { @@ -884,7 +1165,7 @@ void shouldReturnTrueAndPostWhenCacheExists() throws IOException { PullRequestAnalysisProcessor.CacheHitType result = processor.postAnalysisCacheIfExist( project, pullRequest, "abc123", 42L, reportingService, "placeholder-id", - "main", "feature-branch", "identity"); + "main", "feature-branch", "identity", lockLease); assertThat(result).isEqualTo(PullRequestAnalysisProcessor.CacheHitType.EXACT); verify(reportingService).postAnalysisResults(eq(codeAnalysis), eq(project), eq(42L), eq(100L), @@ -902,7 +1183,7 @@ void shouldReturnFalseWhenNoCacheExists() throws IOException { PullRequestAnalysisProcessor.CacheHitType result = processor.postAnalysisCacheIfExist( project, pullRequest, "abc123", 42L, reportingService, "placeholder-id", - "main", "feature-branch", "identity"); + "main", "feature-branch", "identity", lockLease); assertThat(result).isEqualTo(PullRequestAnalysisProcessor.CacheHitType.NONE); verify(reportingService, never()).postAnalysisResults(any(), any(), anyLong(), any(), any()); @@ -923,7 +1204,7 @@ void shouldRejectCacheEntriesWithDifferentReviewIdentity() throws IOException { PullRequestAnalysisProcessor.CacheHitType result = processor.postAnalysisCacheIfExist( project, pullRequest, "abc123", 42L, reportingService, "placeholder-id", - "main", "feature-branch", "current-identity"); + "main", "feature-branch", "current-identity", lockLease); assertThat(result).isEqualTo(PullRequestAnalysisProcessor.CacheHitType.NONE); verify(reportingService, never()).postAnalysisResults(any(), any(), anyLong(), any(), any()); @@ -944,7 +1225,7 @@ void shouldReturnTrueEvenWhenPostingFails() throws IOException { PullRequestAnalysisProcessor.CacheHitType result = processor.postAnalysisCacheIfExist( project, pullRequest, "abc123", 42L, reportingService, "placeholder-id", - "main", "feature-branch", "identity"); + "main", "feature-branch", "identity", lockLease); // Should still return EXACT (cache existed) assertThat(result).isEqualTo(PullRequestAnalysisProcessor.CacheHitType.EXACT); @@ -970,7 +1251,6 @@ void shouldWorkWithoutOptionalDependencies() { fileSnapshotService, prIssueTrackingService, null, // astScopeEnricher - null, // ragOperationsService null // eventPublisher ); diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockServiceTest.java index d4439cba..45aec1bd 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockServiceTest.java @@ -1,6 +1,11 @@ package org.rostilos.codecrow.analysisengine.service; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -13,11 +18,14 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.support.TransactionTemplate; +import org.slf4j.LoggerFactory; import java.lang.reflect.Field; import java.time.OffsetDateTime; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -50,6 +58,11 @@ void setUp() throws Exception { setField(lockService, "ragLockTimeoutMinutes", 360); } + @AfterEach + void tearDown() { + lockService.shutdownLeaseHeartbeatExecutor(); + } + private void setId(Object obj, Long id) throws Exception { Field field = obj.getClass().getDeclaredField("id"); field.setAccessible(true); @@ -354,33 +367,151 @@ void testExtendLock_ExpiredLock_ReturnsFalse() { @Test void testRenewLock_UsesLeaseFromCurrentTime() { String lockKey = "lock-1-main-RAG_INDEXING"; - AnalysisLock lock = mock(AnalysisLock.class); - when(lock.isExpired()).thenReturn(false); - when(lockRepository.findByLockKey(lockKey)).thenReturn(Optional.of(lock)); + when(lockRepository.renewActiveLock(eq(lockKey), any(), any())).thenReturn(1); OffsetDateTime before = OffsetDateTime.now().plusMinutes(29); boolean result = lockService.renewLock(lockKey, 30); assertThat(result).isTrue(); + ArgumentCaptor now = ArgumentCaptor.forClass(OffsetDateTime.class); ArgumentCaptor expiry = ArgumentCaptor.forClass(OffsetDateTime.class); - verify(lock).setExpiresAt(expiry.capture()); + verify(lockRepository).renewActiveLock(eq(lockKey), now.capture(), expiry.capture()); assertThat(expiry.getValue()).isAfter(before); assertThat(expiry.getValue()).isBefore(OffsetDateTime.now().plusMinutes(31)); - verify(lockRepository).save(lock); + assertThat(expiry.getValue()).isAfter(now.getValue()); } @Test void testRenewLock_RefusesMissingOrExpiredOwnership() { - AnalysisLock expiredLock = mock(AnalysisLock.class); - when(expiredLock.isExpired()).thenReturn(true); - when(lockRepository.findByLockKey("missing")).thenReturn(Optional.empty()); - when(lockRepository.findByLockKey("expired")).thenReturn(Optional.of(expiredLock)); + when(lockRepository.renewActiveLock(eq("missing"), any(), any())).thenReturn(0); + when(lockRepository.renewActiveLock(eq("expired"), any(), any())).thenReturn(0); assertThat(lockService.renewLock("missing", 30)).isFalse(); assertThat(lockService.renewLock("expired", 30)).isFalse(); verify(lockRepository, never()).save(any()); } + @Test + void lockLeaseHeartbeatRenewsWithoutWorkerProgressAndStopsWhenClosed() throws Exception { + setField(lockService, "lockHeartbeatIntervalSeconds", 1); + AtomicInteger renewals = new AtomicInteger(); + when(lockRepository.renewActiveLock(eq("quiet-review"), any(), any())) + .thenAnswer(invocation -> { + renewals.incrementAndGet(); + return 1; + }); + + AnalysisLockService.LockLease lease = lockService.maintainLockLease("quiet-review", 1); + + verify(lockRepository, timeout(2500).atLeast(2)) + .renewActiveLock(eq("quiet-review"), any(), any()); + assertThat(lease.isOwnershipLost()).isFalse(); + lease.close(); + int afterClose = renewals.get(); + Thread.sleep(1200); + assertThat(renewals).hasValue(afterClose); + } + + @Test + void blockedHeartbeatDoesNotStarveOtherActiveLeases() throws Exception { + setField(lockService, "lockHeartbeatIntervalSeconds", 1); + AtomicInteger blockedRenewals = new AtomicInteger(); + CountDownLatch blockedHeartbeatEntered = new CountDownLatch(1); + CountDownLatch releaseBlockedHeartbeat = new CountDownLatch(1); + when(lockRepository.renewActiveLock(anyString(), any(), any())) + .thenAnswer(invocation -> { + String key = invocation.getArgument(0); + if ("blocked-review".equals(key) + && blockedRenewals.incrementAndGet() > 1) { + blockedHeartbeatEntered.countDown(); + releaseBlockedHeartbeat.await(5, TimeUnit.SECONDS); + } + return 1; + }); + + AnalysisLockService.LockLease blocked = + lockService.maintainLockLease("blocked-review", 1); + AnalysisLockService.LockLease independent = + lockService.maintainLockLease("independent-review", 1); + try { + assertThat(blockedHeartbeatEntered.await(2500, TimeUnit.MILLISECONDS)).isTrue(); + verify(lockRepository, timeout(1500).atLeast(2)) + .renewActiveLock(eq("independent-review"), any(), any()); + } finally { + releaseBlockedHeartbeat.countDown(); + blocked.close(); + independent.close(); + } + } + + @Test + void transientRenewalExceptionDoesNotProveOwnershipLossAfterSuccessfulRenewal() { + when(lockRepository.renewActiveLock(eq("db-blip"), any(), any())) + .thenReturn(1) + .thenThrow(new RuntimeException("temporary database timeout")); + + AnalysisLockService.LockLease lease = lockService.maintainLockLease("db-blip", 30); + + assertThat(lease.isOwnershipLost()).isFalse(); + assertThat(lease.confirmOwnership()).isTrue(); + lease.close(); + } + + @Test + void repeatedTransientRenewalFailuresWarnOnceUntilRecovery() { + when(lockRepository.renewActiveLock(eq("db-outage"), any(), any())) + .thenReturn(1) + .thenThrow(new RuntimeException("database unavailable")) + .thenThrow(new RuntimeException("database unavailable")) + .thenReturn(1); + Logger logger = (Logger) LoggerFactory.getLogger(AnalysisLockService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + try (AnalysisLockService.LockLease lease = + lockService.maintainLockLease("db-outage", 30)) { + assertThat(lease.confirmOwnership()).isTrue(); + assertThat(lease.confirmOwnership()).isTrue(); + assertThat(lease.confirmOwnership()).isTrue(); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage)) + .containsExactly("Analysis lock renewal failed; the prior lease is still active " + + "and renewal will retry: db-outage"); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.INFO) + .map(ILoggingEvent::getFormattedMessage)) + .contains("Analysis lock heartbeat recovered: db-outage"); + } + + @Test + void initialRenewalExceptionCannotAssumeAFullLeaseForPreAcquiredLock() { + when(lockRepository.renewActiveLock(eq("unknown-age"), any(), any())) + .thenThrow(new RuntimeException("database unavailable")); + + AnalysisLockService.LockLease lease = lockService.maintainLockLease("unknown-age", 30); + + assertThat(lease.isOwnershipLost()).isTrue(); + assertThat(lease.confirmOwnership()).isFalse(); + lease.close(); + } + + @Test + void atomicRenewalMissProvesLeaseOwnershipWasLost() { + when(lockRepository.renewActiveLock(eq("replaced"), any(), any())).thenReturn(0); + + AnalysisLockService.LockLease lease = lockService.maintainLockLease("replaced", 30); + + assertThat(lease.isOwnershipLost()).isTrue(); + assertThat(lease.confirmOwnership()).isFalse(); + lease.close(); + } + @Test void testIsLocked_ReturnsTrue_WhenActiveLockExists() { Long projectId = 1L; diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java index 98bd2709..32ba054b 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java @@ -63,7 +63,7 @@ void branchJobWithoutPrContextWaitsOnlyForEarlierPrJobs() { assertThat(result).isEqualTo(BranchAnalysisGateService.GateResult.READY); verify(jobRepository, times(4)).existsActivePrAnalysisJobBefore(1L, "main", 103L); - verify(consumer, times(3)).accept(org.mockito.ArgumentMatchers.argThat( + verify(consumer).accept(org.mockito.ArgumentMatchers.argThat( event -> "pr_analysis_wait".equals(event.get("type")))); var ordered = inOrder(jobRepository); @@ -111,7 +111,11 @@ void mergeWaitsOnlyForNewestAttemptOfItsOwnPr() { assertThat(result).isEqualTo(BranchAnalysisGateService.GateResult.READY); verify(consumer).accept(org.mockito.ArgumentMatchers.argThat( event -> Long.valueOf(41L).equals(event.get("prNumber")) - && event.get("message").toString().contains("PR #41"))); + && event.get("message").toString().contains("PR #41") + && JobType.BRANCH_ANALYSIS.name().equals( + event.get("waitingJobType")) + && JobType.PR_ANALYSIS.name().equals( + event.get("blockingJobType")))); verify(jobRepository, times(0)).existsActivePrAnalysisJobBefore(1L, "main", 103L); } @@ -164,9 +168,33 @@ void prWaitsOnlyForOlderBranchJobsOnItsTargetBranch() { assertThat(result).isEqualTo(BranchAnalysisGateService.GateResult.READY); verify(jobRepository, times(3)) .existsActiveBranchAnalysisJobBefore(1L, "main", 104L); - verify(consumer, times(2)).accept(org.mockito.ArgumentMatchers.argThat( + verify(consumer).accept(org.mockito.ArgumentMatchers.argThat( event -> "branch_analysis_wait".equals(event.get("type")) - && "main".equals(event.get("branchName")))); + && "main".equals(event.get("branchName")) + && JobType.PR_ANALYSIS.name().equals( + event.get("waitingJobType")) + && JobType.BRANCH_ANALYSIS.name().equals( + event.get("blockingJobType")))); + } + + @Test + void prDoesNotWaitForBranchWorkOnItsSourceOrAnotherTarget() { + Job prJob = job(JobStatus.RUNNING); + ReflectionTestUtils.setField(prJob, "id", 104L); + prJob.setBranchName("main"); + + when(jobRepository.existsActiveBranchAnalysisJobBefore(1L, "main", 104L)) + .thenReturn(false); + + BranchAnalysisGateService.GateResult result = service.awaitDependencies( + 1L, prJob, event -> { }); + + assertThat(result).isEqualTo(BranchAnalysisGateService.GateResult.READY); + verify(jobRepository).existsActiveBranchAnalysisJobBefore(1L, "main", 104L); + verify(jobRepository, times(0)) + .existsActiveBranchAnalysisJobBefore(1L, "1.8.1-rc", 104L); + verify(jobRepository, times(0)) + .existsActiveBranchAnalysisJobBefore(1L, "feature/public-share-links", 104L); } private static Job job(JobStatus status) { diff --git a/java-ecosystem/libs/core/src/main/java/module-info.java b/java-ecosystem/libs/core/src/main/java/module-info.java index 9b44d3f4..8e792eee 100644 --- a/java-ecosystem/libs/core/src/main/java/module-info.java +++ b/java-ecosystem/libs/core/src/main/java/module-info.java @@ -166,4 +166,6 @@ to org.hibernate.orm.core, spring.beans, spring.context, spring.core; opens org.rostilos.codecrow.core.persistence.repository.qadoc to spring.core, spring.beans, spring.context; + + exports org.rostilos.codecrow.core.service.qadoc; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatus.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatus.java index 0fa6c745..19656c33 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatus.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatus.java @@ -63,6 +63,10 @@ public class RagIndexStatus { @Column(name = "chunk_count") private Integer chunkCount; + /** Durable producer identity used to keep stale recovery from touching a newer run. */ + @Column(name = "active_job_id") + private Long activeJobId; + @PreUpdate protected void onUpdate() { updatedAt = OffsetDateTime.now(); @@ -198,4 +202,12 @@ public Integer getChunkCount() { public void setChunkCount(Integer chunkCount) { this.chunkCount = chunkCount; } + + public Long getActiveJobId() { + return activeJobId; + } + + public void setActiveJobId(Long activeJobId) { + this.activeJobId = activeJobId; + } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java index 89c5b5c9..f0479472 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java @@ -69,6 +69,12 @@ public class RagBranchIndex { @Column(name = "error_message", columnDefinition = "TEXT") private String errorMessage; + @Column(name = "cleanup_claim_token", length = 64) + private String cleanupClaimToken; + + @Column(name = "cleanup_claimed_at") + private OffsetDateTime cleanupClaimedAt; + /** * Files that were deleted in this branch (for query-time filtering). * These files should be excluded when querying the branch's context. @@ -216,6 +222,22 @@ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } + public String getCleanupClaimToken() { + return cleanupClaimToken; + } + + public void setCleanupClaimToken(String cleanupClaimToken) { + this.cleanupClaimToken = cleanupClaimToken; + } + + public OffsetDateTime getCleanupClaimedAt() { + return cleanupClaimedAt; + } + + public void setCleanupClaimedAt(OffsetDateTime cleanupClaimedAt) { + this.cleanupClaimedAt = cleanupClaimedAt; + } + public Set getDeletedFiles() { return deletedFiles; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java index 120b59c1..be055a07 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java @@ -43,6 +43,10 @@ public class RagIndexOperation { @Column(name = "job_id") private Long jobId; + /** Exact analysis-lock owner captured for safe abandoned-producer cleanup. */ + @Column(name = "analysis_lock_key", length = 500) + private String analysisLockKey; + @Column(name = "attempt_count", nullable = false) private int attemptCount; @@ -124,6 +128,8 @@ public void heartbeat() { public void setGeneration(RagBranchIndexGeneration generation) { this.generation = generation; } public Long getJobId() { return jobId; } public void setJobId(Long jobId) { this.jobId = jobId; } + public String getAnalysisLockKey() { return analysisLockKey; } + public void setAnalysisLockKey(String analysisLockKey) { this.analysisLockKey = analysisLockKey; } public int getAttemptCount() { return attemptCount; } public void setAttemptCount(int attemptCount) { this.attemptCount = attemptCount; } public OffsetDateTime getCreatedAt() { return createdAt; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java index 8a786830..f3e49ae3 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java @@ -36,6 +36,19 @@ Optional findByProjectIdAndBranchNameAndAnalysisType( @Query("DELETE FROM AnalysisLock l WHERE l.lockKey = :lockKey") int deleteByLockKey(@Param("lockKey") String lockKey); + /** + * Atomically renews an owned, unexpired lease. A read/check/save sequence can + * race the scheduled expired-lock delete at the lease boundary; the guarded + * update makes the winner explicit to the caller. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query("UPDATE AnalysisLock l SET l.expiresAt = :expiresAt " + + "WHERE l.lockKey = :lockKey AND l.expiresAt >= :now") + int renewActiveLock( + @Param("lockKey") String lockKey, + @Param("now") OffsetDateTime now, + @Param("expiresAt") OffsetDateTime expiresAt); + @Query("SELECT CASE WHEN COUNT(l) > 0 THEN true ELSE false END FROM AnalysisLock l " + "WHERE l.project.id = :projectId AND l.branchName = :branchName " + "AND l.analysisType = :analysisType AND l.expiresAt > :now") diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java index 4b7da2d7..3be5fb73 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java @@ -1,8 +1,11 @@ package org.rostilos.codecrow.core.persistence.repository.analysis; +import jakarta.persistence.LockModeType; import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; +import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @@ -15,6 +18,10 @@ public interface RagIndexStatusRepository extends JpaRepository findByProjectId(Long projectId); + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT r FROM RagIndexStatus r WHERE r.project.id = :projectId") + Optional findByProjectIdForUpdate(@Param("projectId") Long projectId); + Optional findByWorkspaceNameAndProjectName(String workspaceName, String projectName); List findByStatus(RagIndexingStatus status); @@ -35,6 +42,24 @@ List findByWorkspaceAndStatus(@Param("workspace") String workspa boolean isProjectIndexed(@Param("projectId") Long projectId); + /** + * Restore the last usable checkpoint after a legacy incremental producer + * expires, but only while that exact job still owns the active status. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query("UPDATE RagIndexStatus r SET " + + "r.status = org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.INDEXED, " + + "r.errorMessage = CONCAT('Incremental update failed: ', :errorMessage), " + + "r.failedIncrementalCount = COALESCE(r.failedIncrementalCount, 0) + 1, " + + "r.activeJobId = NULL " + + "WHERE r.project.id = :projectId AND r.activeJobId = :jobId " + + "AND r.status IN (" + + "org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.INDEXING, " + + "org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.UPDATING)") + int recoverAbandonedIncrementalUpdate( + @Param("projectId") Long projectId, + @Param("jobId") Long jobId, + @Param("errorMessage") String errorMessage); + void deleteByProjectId(Long projectId); } - diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java index f96cc670..e51badda 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java @@ -13,6 +13,16 @@ @Repository public interface BranchRepository extends JpaRepository { + interface StaleRetryCandidate { + Long getBranchId(); + Long getProjectId(); + String getBranchName(); + String getCommitHash(); + int getConsecutiveFailures(); + java.time.OffsetDateTime getLastHealthCheckAt(); + boolean getBranchAnalysisEnabled(); + } + Optional findByProjectIdAndBranchName(Long projectId, String branchName); Optional findByProjectIdAndCommitHash(Long projectId, String commitHash); @@ -32,4 +42,22 @@ public interface BranchRepository extends JpaRepository { */ @Query("SELECT b FROM Branch b JOIN FETCH b.project WHERE b.healthStatus = :status") List findByHealthStatusWithProject(@Param("status") BranchHealthStatus status); + + /** + * Materializes only scheduler inputs so the repository's short read + * transaction is closed before a retry performs VCS, RAG, or AI work. + */ + @Query(""" + SELECT b.id AS branchId, + b.project.id AS projectId, + b.branchName AS branchName, + b.commitHash AS commitHash, + b.consecutiveFailures AS consecutiveFailures, + b.lastHealthCheckAt AS lastHealthCheckAt, + b.project.branchAnalysisEnabled AS branchAnalysisEnabled + FROM Branch b + WHERE b.healthStatus = :status + """) + List findStaleRetryCandidates( + @Param("status") BranchHealthStatus status); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobLogRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobLogRepository.java index e17eeb05..8e1a91cb 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobLogRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobLogRepository.java @@ -44,6 +44,13 @@ List findByJobIdAndStep( @Param("step") String step ); + @Query("SELECT CASE WHEN COUNT(l) > 0 THEN true ELSE false END " + + "FROM JobLog l WHERE l.job.id = :jobId AND l.step = :step") + boolean existsByJobIdAndStep( + @Param("jobId") Long jobId, + @Param("step") String step + ); + @Query("SELECT COALESCE(MAX(l.sequenceNumber), 0) + 1 FROM JobLog l WHERE l.job.id = :jobId") Long getNextSequenceNumber(@Param("jobId") Long jobId); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java index 28570720..87f3a6ba 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java @@ -1,11 +1,13 @@ package org.rostilos.codecrow.core.persistence.repository.job; +import jakarta.persistence.LockModeType; import org.rostilos.codecrow.core.model.job.Job; import org.rostilos.codecrow.core.model.job.JobStatus; import org.rostilos.codecrow.core.model.job.JobType; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -16,8 +18,25 @@ public interface JobRepository extends JpaRepository { + /** + * Scalar identity for a legacy incremental RAG job whose producer may + * have disappeared. Keeping recovery coordinates scalar avoids carrying + * detached project proxies out of the repository transaction. + */ + interface LegacyRagJobRecoveryCoordinates { + Long getJobId(); + Long getProjectId(); + String getBranchName(); + String getCommitHash(); + String getErrorMessage(); + } + Optional findByExternalId(String externalId); + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT j FROM Job j WHERE j.id = :jobId") + Optional findByIdForUpdate(@Param("jobId") Long jobId); + @Query("SELECT j FROM Job j WHERE j.project.id = :projectId ORDER BY j.createdAt DESC") Page findByProjectId(@Param("projectId") Long projectId, Pageable pageable); @@ -190,6 +209,8 @@ boolean existsNewerBranchAnalysisJob( @Query("SELECT j FROM Job j WHERE " + "j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED) " + "AND j.updatedAt < :threshold ORDER BY j.createdAt ASC") @@ -199,6 +220,8 @@ List findRecoverableWebhookJobs( @Query("SELECT j FROM Job j WHERE " + "j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING " + "AND j.updatedAt < :threshold ORDER BY j.updatedAt ASC") List findAbandonedRunningWebhookJobs( @@ -208,6 +231,9 @@ List findAbandonedRunningWebhookJobs( @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + "j.updatedAt = :claimedAt WHERE j.id = :jobId " + + "AND j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED) " + "AND j.updatedAt < :threshold") @@ -220,9 +246,129 @@ int claimRecoverableWebhookJob( @Query("UPDATE Job j SET j.updatedAt = :activityAt WHERE j.id = :jobId") int touchJob(@Param("jobId") Long jobId, @Param("activityAt") OffsetDateTime activityAt); + /** + * Atomically renew a live legacy incremental RAG producer. Exact-generation + * jobs are owned by {@code RagIndexOperation} and must never share this + * recovery path. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query(value = """ + UPDATE job j + SET updated_at = :renewedAt + WHERE j.id = :jobId + AND j.job_type = 'RAG_INCREMENTAL_INDEX' + AND j.status = 'RUNNING' + AND j.updated_at >= :validAfter + AND NOT EXISTS ( + SELECT 1 + FROM rag_index_operation o + WHERE o.job_id = j.id + ) + """, nativeQuery = true) + int renewLegacyRagJobLease( + @Param("jobId") Long jobId, + @Param("validAfter") OffsetDateTime validAfter, + @Param("renewedAt") OffsetDateTime renewedAt); + + /** + * Find legacy incremental RAG jobs with no durable producer activity. The + * guarded update below remains the authority; this projection only bounds + * and prioritizes recovery work. + */ + @Query("SELECT j.id AS jobId, j.project.id AS projectId, " + + "j.branchName AS branchName, j.commitHash AS commitHash, " + + "j.errorMessage AS errorMessage " + + "FROM Job j WHERE " + + "j.jobType = org.rostilos.codecrow.core.model.job.JobType.RAG_INCREMENTAL_INDEX " + + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + + "org.rostilos.codecrow.core.model.job.JobStatus.RUNNING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.WAITING) " + + "AND j.updatedAt < :threshold " + + "AND NOT EXISTS (SELECT o.id FROM RagIndexOperation o WHERE o.jobId = j.id) " + + "ORDER BY j.updatedAt ASC") + List findAbandonedLegacyRagJobs( + @Param("threshold") OffsetDateTime threshold, + Pageable pageable); + + /** + * Win recovery only if neither a heartbeat nor a terminal transition moved + * the row after selection. This CAS is the fencing point between a live + * producer and the recovery scheduler. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query(value = """ + UPDATE job j + SET status = 'FAILED', + completed_at = :failedAt, + error_message = :diagnostic, + updated_at = :failedAt + WHERE j.id = :jobId + AND j.job_type = 'RAG_INCREMENTAL_INDEX' + AND j.status IN ('PENDING', 'QUEUED', 'RUNNING', 'WAITING') + AND j.updated_at < :threshold + AND NOT EXISTS ( + SELECT 1 + FROM rag_index_operation o + WHERE o.job_id = j.id + ) + """, nativeQuery = true) + int failAbandonedLegacyRagJob( + @Param("jobId") Long jobId, + @Param("threshold") OffsetDateTime threshold, + @Param("failedAt") OffsetDateTime failedAt, + @Param("diagnostic") String diagnostic); + + /** + * Atomically win the terminal transition for a still-owned legacy RAG + * producer. The surrounding transaction commits this row together with + * project and branch checkpoints, so recovery cannot interleave between + * ownership proof and publication. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query(value = """ + UPDATE job j + SET status = 'COMPLETED', + completed_at = :completedAt, + progress = 100, + updated_at = :completedAt + WHERE j.id = :jobId + AND j.job_type = 'RAG_INCREMENTAL_INDEX' + AND j.status = 'RUNNING' + AND j.updated_at >= :validAfter + AND NOT EXISTS ( + SELECT 1 + FROM rag_index_operation o + WHERE o.job_id = j.id + ) + """, nativeQuery = true) + int completeOwnedLegacyRagJob( + @Param("jobId") Long jobId, + @Param("validAfter") OffsetDateTime validAfter, + @Param("completedAt") OffsetDateTime completedAt); + + /** Retry projection repair if the process stopped after failing the job. */ + @Query("SELECT j.id AS jobId, j.project.id AS projectId, " + + "j.branchName AS branchName, j.commitHash AS commitHash, " + + "j.errorMessage AS errorMessage " + + "FROM Job j WHERE " + + "j.jobType = org.rostilos.codecrow.core.model.job.JobType.RAG_INCREMENTAL_INDEX " + + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.FAILED " + + "AND NOT EXISTS (SELECT o.id FROM RagIndexOperation o WHERE o.jobId = j.id) " + + "AND EXISTS (SELECT s.id FROM RagIndexStatus s WHERE " + + "s.project.id = j.project.id AND s.activeJobId = j.id " + + "AND s.status IN (org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.INDEXING, " + + "org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.UPDATING)) " + + "ORDER BY j.updatedAt ASC") + List findFailedLegacyRagJobsWithActiveStatus( + Pageable pageable); + @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + "j.updatedAt = :claimedAt WHERE j.id = :jobId " + + "AND j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING " + "AND j.updatedAt < :threshold") int claimAbandonedRunningWebhookJob( diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java index a7347032..322d6de5 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java @@ -13,6 +13,14 @@ @Repository public interface RagBranchIndexGenerationRepository extends JpaRepository { + interface CleanupGenerationCandidate { + Long getGenerationId(); + String getCollectionName(); + String getRevision(); + String getManifestDigest(); + RagBranchIndexGenerationStatus getStatus(); + } + Optional findFirstByBranchIndexIdAndRevisionAndStatusInOrderByCreatedAtDesc( Long branchIndexId, String revision, @@ -20,13 +28,34 @@ Optional findFirstByBranchIndexIdAndRevisionAndStatusI List findByBranchIndexIdOrderByCreatedAtDesc(Long branchIndexId); + @Query("SELECT g.id AS generationId, g.collectionName AS collectionName, " + + "g.revision AS revision, g.manifestDigest AS manifestDigest, " + + "g.status AS status " + + "FROM RagBranchIndexGeneration g WHERE g.branchIndex.id = :branchIndexId " + + "ORDER BY g.createdAt DESC") + List findCleanupCandidatesByBranchIndexId( + @Param("branchIndexId") Long branchIndexId); + @Query("SELECT g FROM RagBranchIndexGeneration g " + "JOIN g.branchIndex b WHERE b.project.id = :projectId " + "AND b.branchName = :branchName AND g.revision = :revision " + + "AND b.cleanupClaimToken IS NULL " + "AND g.status IN :statuses ORDER BY g.createdAt DESC") List findAvailableExactGeneration( @Param("projectId") Long projectId, @Param("branchName") String branchName, @Param("revision") String revision, @Param("statuses") List statuses); + + /** + * Physical targets that may still contain a project's PR overlays. Both + * active and superseded published generations are relevant: a generation + * can be superseded between review indexing and the close webhook. + */ + @Query("SELECT DISTINCT g.collectionName FROM RagBranchIndexGeneration g " + + "JOIN g.branchIndex b WHERE b.project.id = :projectId " + + "AND g.status IN :statuses") + List findCollectionNamesByProjectIdAndStatusIn( + @Param("projectId") Long projectId, + @Param("statuses") List statuses); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java index 5660ee45..1757bfbb 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java @@ -3,13 +3,16 @@ import jakarta.persistence.LockModeType; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import java.time.OffsetDateTime; import java.util.List; import java.util.Optional; @@ -19,8 +22,66 @@ @Repository public interface RagBranchIndexRepository extends JpaRepository { + interface OperatorAliasCandidate { + Long getBranchIndexId(); + Long getGenerationId(); + Long getProjectId(); + String getWorkspaceName(); + String getProjectNamespace(); + String getBranchName(); + String getRevision(); + String getCollectionName(); + String getManifestDigest(); + RagBranchIndexKind getIndexKind(); + } + + /** + * Immutable coordinates required to advance the currently published + * generation. Keeping this boundary scalar prevents callers from carrying + * a lazy generation proxy into long-running VCS or RAG operations after the + * repository transaction has closed. + */ + interface ActiveGenerationCoordinates { + Long getGenerationId(); + String getRevision(); + String getCollectionName(); + String getRepresentationFingerprint(); + Integer getFileCount(); + Integer getChunkCount(); + } + + interface TransientCleanupCandidate { + Long getBranchIndexId(); + Long getProjectId(); + String getWorkspaceName(); + String getProjectNamespace(); + String getBranchName(); + OffsetDateTime getLastAccessedAt(); + OffsetDateTime getUpdatedAt(); + String getCleanupClaimToken(); + OffsetDateTime getCleanupClaimedAt(); + ProjectConfig getProjectConfiguration(); + } + Optional findByProjectIdAndBranchName(Long projectId, String branchName); + @Query(""" + SELECT g.id AS generationId, + g.revision AS revision, + g.collectionName AS collectionName, + g.representationFingerprint AS representationFingerprint, + g.fileCount AS fileCount, + g.chunkCount AS chunkCount + FROM RagBranchIndex b + JOIN b.activeGeneration g + WHERE b.project.id = :projectId + AND b.branchName = :branchName + AND b.cleanupClaimToken IS NULL + """) + Optional findActiveGenerationCoordinates( + @Param("projectId") Long projectId, + @Param("branchName") String branchName); + @Lock(LockModeType.PESSIMISTIC_WRITE) @Query("SELECT b FROM RagBranchIndex b WHERE b.project.id = :projectId AND b.branchName = :branchName") Optional findByProjectIdAndBranchNameForUpdate( @@ -35,6 +96,82 @@ Optional findByProjectIdAndBranchNameForUpdate( List findByIndexKind(RagBranchIndexKind indexKind); + /** Scalar scheduler inputs, materialized before any remote cleanup call. */ + @Query(""" + SELECT b.id AS branchIndexId, + b.project.id AS projectId, + b.project.workspace.name AS workspaceName, + b.project.namespace AS projectNamespace, + b.branchName AS branchName, + b.lastAccessedAt AS lastAccessedAt, + b.updatedAt AS updatedAt, + b.cleanupClaimToken AS cleanupClaimToken, + b.cleanupClaimedAt AS cleanupClaimedAt, + b.project.configuration AS projectConfiguration + FROM RagBranchIndex b + WHERE b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.TRANSIENT + """) + List findTransientCleanupCandidates(); + + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("UPDATE RagBranchIndex b SET b.cleanupClaimToken = :claimToken, " + + "b.cleanupClaimedAt = :claimedAt " + + "WHERE b.id = :branchIndexId " + + "AND b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.TRANSIENT " + + "AND b.lifecycleStatus = " + + "org.rostilos.codecrow.core.model.rag.RagBranchIndexLifecycleStatus.READY " + + "AND (b.cleanupClaimToken IS NULL OR b.cleanupClaimToken = :claimToken " + + "OR b.cleanupClaimedAt < :staleBefore) " + + "AND ((b.lastAccessedAt IS NOT NULL AND b.lastAccessedAt < :cutoff) " + + "OR (b.lastAccessedAt IS NULL AND b.updatedAt < :cutoff))") + int claimExpiredTransientForDeletion( + @Param("branchIndexId") Long branchIndexId, + @Param("cutoff") OffsetDateTime cutoff, + @Param("staleBefore") OffsetDateTime staleBefore, + @Param("claimToken") String claimToken, + @Param("claimedAt") OffsetDateTime claimedAt); + + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("UPDATE RagBranchIndex b SET b.cleanupClaimedAt = :claimedAt " + + "WHERE b.id = :branchIndexId AND b.cleanupClaimToken = :claimToken") + int heartbeatTransientDeletionClaim( + @Param("branchIndexId") Long branchIndexId, + @Param("claimToken") String claimToken, + @Param("claimedAt") OffsetDateTime claimedAt); + + /** + * Atomically records exact-generation use unless cleanup already owns the + * branch. A successful touch makes any expiry claim fail its cutoff check. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("UPDATE RagBranchIndex b SET b.lastAccessedAt = :accessedAt " + + "WHERE b.project.id = :projectId AND b.branchName = :branchName " + + "AND b.cleanupClaimToken IS NULL") + int markAccessedIfUnclaimed( + @Param("projectId") Long projectId, + @Param("branchName") String branchName, + @Param("accessedAt") OffsetDateTime accessedAt); + + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("UPDATE RagBranchIndex b SET b.cleanupClaimToken = NULL, b.cleanupClaimedAt = NULL " + + "WHERE b.id = :branchIndexId AND b.cleanupClaimToken = :claimToken") + int cancelTransientDeletion( + @Param("branchIndexId") Long branchIndexId, + @Param("claimToken") String claimToken); + + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Transactional + @Query("DELETE FROM RagBranchIndex b WHERE b.id = :branchIndexId " + + "AND b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.TRANSIENT " + + "AND b.cleanupClaimToken = :claimToken") + int deleteClaimedTransientById( + @Param("branchIndexId") Long branchIndexId, + @Param("claimToken") String claimToken); + @Query("SELECT CASE WHEN COUNT(b) > 0 THEN true ELSE false END FROM RagBranchIndex b " + "WHERE b.project.id = :projectId AND b.branchName = :branchName") boolean existsByProjectIdAndBranchName(@Param("projectId") Long projectId, @Param("branchName") String branchName); @@ -47,4 +184,56 @@ Optional findByProjectIdAndBranchNameForUpdate( @Query("SELECT b.branchName FROM RagBranchIndex b WHERE b.project.id = :projectId") List findBranchNamesByProjectId(@Param("projectId") Long projectId); + + /** + * Reads the immutable values needed by optional operator-alias repair. + * Returning a scalar projection lets the database transaction finish + * before the caller performs any potentially slow RAG/Qdrant request. + */ + @Query(""" + SELECT b.project.id AS projectId, + b.id AS branchIndexId, + b.activeGeneration.id AS generationId, + b.project.workspace.name AS workspaceName, + b.project.namespace AS projectNamespace, + b.branchName AS branchName, + b.activeGeneration.revision AS revision, + b.activeGeneration.collectionName AS collectionName, + b.activeGeneration.manifestDigest AS manifestDigest, + b.indexKind AS indexKind + FROM RagBranchIndex b + WHERE b.activeGeneration IS NOT NULL + AND b.indexKind IN ( + org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.PRIMARY, + org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE + ) + """) + List findOperatorAliasCandidates(); + + /** + * Re-reads one branch's current publication coordinates without retaining + * an entity or transaction across the remote alias request. + */ + @Query(""" + SELECT b.project.id AS projectId, + b.id AS branchIndexId, + b.activeGeneration.id AS generationId, + b.project.workspace.name AS workspaceName, + b.project.namespace AS projectNamespace, + b.branchName AS branchName, + b.activeGeneration.revision AS revision, + b.activeGeneration.collectionName AS collectionName, + b.activeGeneration.manifestDigest AS manifestDigest, + b.indexKind AS indexKind + FROM RagBranchIndex b + WHERE b.id = :branchIndexId + AND b.activeGeneration IS NOT NULL + AND b.indexKind IN ( + org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.PRIMARY, + org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE + ) + """) + Optional findOperatorAliasCandidateById( + @Param("branchIndexId") Long branchIndexId); + } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java index 92d4b4c1..4f2670c6 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java @@ -1,8 +1,12 @@ package org.rostilos.codecrow.core.persistence.repository.rag; +import jakarta.persistence.LockModeType; import org.rostilos.codecrow.core.model.rag.RagIndexOperation; import org.rostilos.codecrow.core.model.rag.RagIndexOperationStatus; +import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import java.time.OffsetDateTime; @@ -12,9 +16,129 @@ @Repository public interface RagIndexOperationRepository extends JpaRepository { + /** Detached-safe coordinates for abandoned and failed-operation recovery. */ + interface RecoveryOperationProjection { + Long getOperationId(); + Long getProjectId(); + String getBranchName(); + String getToRevision(); + Long getJobId(); + String getAnalysisLockKey(); + String getErrorMessage(); + } + + interface SucceededOperationProjection { + Long getOperationId(); + Long getProjectId(); + String getBranchName(); + String getToRevision(); + Long getJobId(); + String getAnalysisLockKey(); + Integer getFileCount(); + Integer getChunkCount(); + Boolean getActiveGeneration(); + } + Optional findByProjectIdAndOperationKey(Long projectId, String operationKey); - List findByStatusInAndUpdatedAtBefore( - List statuses, - OffsetDateTime updatedBefore); + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT o FROM RagIndexOperation o WHERE o.id = :operationId") + Optional findByIdForUpdate(@Param("operationId") Long operationId); + + @Query("SELECT o.id AS operationId, o.project.id AS projectId, " + + "o.branchName AS branchName, o.toRevision AS toRevision, " + + "o.jobId AS jobId, o.analysisLockKey AS analysisLockKey, " + + "o.errorMessage AS errorMessage FROM RagIndexOperation o " + + "WHERE o.status IN :statuses AND o.updatedAt < :updatedBefore") + List findRecoverableOperationProjections( + @Param("statuses") List statuses, + @Param("updatedBefore") OffsetDateTime updatedBefore); + + boolean existsByProjectIdAndBranchNameAndStatusIn( + Long projectId, + String branchName, + List statuses); + + /** + * Finds already-failed operations whose durable projections still say the + * producer is active. This repairs drift created by older recovery code or + * by a partial recovery failure on the next scan. + */ + @Query(value = """ + SELECT o.id AS "operationId", + o.project_id AS "projectId", + o.branch_name AS "branchName", + o.to_revision AS "toRevision", + o.job_id AS "jobId", + o.analysis_lock_key AS "analysisLockKey", + o.error_message AS "errorMessage" + FROM rag_index_operation o + WHERE o.status = 'FAILED' + AND ( + EXISTS ( + SELECT 1 FROM job j + WHERE j.id = o.job_id + AND j.status IN ('PENDING', 'QUEUED', 'RUNNING', 'WAITING') + ) + OR EXISTS ( + SELECT 1 FROM rag_index_status s + WHERE s.project_id = o.project_id + AND s.indexed_branch = o.branch_name + AND s.status IN ('INDEXING', 'UPDATING') + AND s.active_job_id = o.job_id + ) + OR EXISTS ( + SELECT 1 FROM analysis_lock l + WHERE l.project_id = o.project_id + AND l.branch_name = o.branch_name + AND l.analysis_type = 'RAG_INDEXING' + AND l.lock_key = o.analysis_lock_key + ) + ) + ORDER BY o.completed_at DESC, o.id DESC + """, nativeQuery = true) + List findFailedOperationsWithActiveProjections(); + + /** + * Scalar recovery coordinates for a generation that was published before + * its job/status/lock projections were terminalized. + */ + @Query(value = """ + SELECT o.id AS "operationId", + o.project_id AS "projectId", + o.branch_name AS "branchName", + o.to_revision AS "toRevision", + o.job_id AS "jobId", + o.analysis_lock_key AS "analysisLockKey", + g.file_count AS "fileCount", + g.chunk_count AS "chunkCount", + (b.active_generation_id = g.id) AS "activeGeneration" + FROM rag_index_operation o + JOIN rag_branch_index_generation g ON g.id = o.generation_id + JOIN rag_branch_index b ON b.id = g.branch_index_id + WHERE o.status = 'SUCCEEDED' + AND ( + EXISTS ( + SELECT 1 FROM job j + WHERE j.id = o.job_id + AND j.status IN ('PENDING', 'QUEUED', 'RUNNING', 'WAITING') + ) + OR EXISTS ( + SELECT 1 FROM rag_index_status s + WHERE s.project_id = o.project_id + AND s.indexed_branch = o.branch_name + AND s.status IN ('INDEXING', 'UPDATING') + AND s.active_job_id = o.job_id + ) + OR EXISTS ( + SELECT 1 FROM analysis_lock l + WHERE l.project_id = o.project_id + AND l.branch_name = o.branch_name + AND l.analysis_type = 'RAG_INDEXING' + AND l.lock_key = o.analysis_lock_key + ) + ) + ORDER BY o.completed_at DESC, o.id DESC + """, nativeQuery = true) + List findSucceededOperationsWithActiveProjections(); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java index 043b4fbe..7606b377 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java @@ -91,6 +91,22 @@ default Job createRagIndexJob( */ void failJob(Job job, String errorMessage); + /** Finish an intentionally skipped job without representing it as a failure. */ + default void skipJob(Job job, String reason) { + completeJob(job, Map.of("status", "skipped", "reason", reason)); + } + + /** + * Announce a success already committed by a fenced repository transition. + * Persistent hosts should override this without updating terminal status. + */ + default void recordExternallyCompletedJob( + Job job, + String state, + String message) { + info(job, state, message); + } + /** * Log an INFO level message to a job. * @param job The job to log to diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java index 5382c051..5602fc7b 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java @@ -358,6 +358,53 @@ public Job completeJob(Job job) { return job; } + /** + * Record and announce a successful terminal transition performed by an + * atomic repository CAS. This never changes job status, so a stale caller + * cannot overwrite a recovery outcome. Repeated calls reuse the durable + * terminal log and still release local streaming subscribers. + */ + @Transactional(propagation = org.springframework.transaction.annotation.Propagation.REQUIRES_NEW) + public void recordExternallyCompletedJob( + Job job, + String step, + String message) { + if (job == null || job.getId() == null) { + return; + } + Job persisted = jobRepository.findByIdForUpdate(job.getId()).orElse(null); + if (persisted == null || persisted.getStatus() != JobStatus.COMPLETED) { + log.info("Ignoring terminal success notification for non-completed job {}", + job.getId()); + return; + } + if (!jobLogRepository.existsByJobIdAndStep(persisted.getId(), step)) { + addLog(persisted, JobLogLevel.INFO, step, message); + } + notifyJobComplete(persisted); + } + + /** Record a failure already committed by an atomic recovery CAS. */ + @Transactional(propagation = org.springframework.transaction.annotation.Propagation.REQUIRES_NEW) + public void recordExternallyFailedJob( + Job job, + String step, + String message) { + if (job == null || job.getId() == null) { + return; + } + Job persisted = jobRepository.findByIdForUpdate(job.getId()).orElse(null); + if (persisted == null || persisted.getStatus() != JobStatus.FAILED) { + log.info("Ignoring terminal failure notification for non-failed job {}", + job.getId()); + return; + } + if (!jobLogRepository.existsByJobIdAndStep(persisted.getId(), step)) { + addLog(persisted, JobLogLevel.ERROR, step, message); + } + notifyJobComplete(persisted); + } + /** * Complete a job and link it to a code analysis. */ @@ -754,6 +801,41 @@ public boolean claimAbandonedRunningWebhookJob(Long jobId, OffsetDateTime thresh jobId, threshold, OffsetDateTime.now()) == 1; } + /** Renew the database-backed lease for one live legacy RAG producer. */ + @Transactional + public boolean renewLegacyRagJobLease( + Long jobId, + OffsetDateTime validAfter, + OffsetDateTime renewedAt) { + return jobRepository.renewLegacyRagJobLease( + jobId, validAfter, renewedAt) == 1; + } + + public List + findAbandonedLegacyRagJobs(OffsetDateTime threshold, int limit) { + return jobRepository.findAbandonedLegacyRagJobs( + threshold, PageRequest.of(0, Math.max(1, limit))); + } + + /** + * Atomically terminalize an abandoned legacy RAG job only while its lease + * is still stale and no exact-generation operation owns it. + */ + @Transactional + public boolean failAbandonedLegacyRagJob( + Long jobId, + OffsetDateTime threshold, + String diagnostic) { + return jobRepository.failAbandonedLegacyRagJob( + jobId, threshold, OffsetDateTime.now(), diagnostic) == 1; + } + + public List + findFailedLegacyRagJobsWithActiveStatus(int limit) { + return jobRepository.findFailedLegacyRagJobsWithActiveStatus( + PageRequest.of(0, Math.max(1, limit))); + } + private void touchJobActivity(Job job) { if (job == null || job.getId() == null) { return; diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/QaDocDocumentService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/QaDocDocumentService.java index 155a944f..f098ad96 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/QaDocDocumentService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/QaDocDocumentService.java @@ -52,6 +52,14 @@ public Optional findLatestDocument(Long projectId, Long prNumber) return qaDocDocumentRepository.findByProjectIdAndPrNumber(projectId, prNumber); } + @Transactional(readOnly = true) + public Optional findDocumentById(Long documentId) { + if (documentId == null) { + return Optional.empty(); + } + return qaDocDocumentRepository.findById(documentId); + } + @Transactional(readOnly = true) public Optional findLatestDocumentForTask(Long projectId, String taskId, diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocContent.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocContent.java new file mode 100644 index 00000000..940c549e --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocContent.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.service.qadoc; + +import java.util.List; + +public record QaDocContent( + String overviewMarkdown, + List testCases, + String environmentMarkdown +) { +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocContentParser.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocContentParser.java new file mode 100644 index 00000000..e481c496 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocContentParser.java @@ -0,0 +1,390 @@ +package org.rostilos.codecrow.core.service.qadoc; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Splits rendered QA markdown into overview, test cases, and environment/setup notes. */ +public final class QaDocContentParser { + + public static final String TEST_CASES_START = ""; + public static final String TEST_CASES_END = ""; + + private static final Pattern SCENARIO_PATTERN = Pattern.compile( + "(?m)^\\s*\\*\\*(.+?)\\*\\*\\s*\\((HIGH|MEDIUM|LOW)\\)[^\\r\\n]*$", + Pattern.CASE_INSENSITIVE + ); + private static final Pattern HEADING_PATTERN = Pattern.compile("(?m)^\\s*(#{2,6})\\s+(.+?)\\s*$"); + private static final Pattern LEGACY_SECTION_PATTERN = Pattern.compile( + "(?im)^\\s*(#{2,4})\\s+(?:\\d+\\.\\s*)?Test Scenarios(?:\\s+by Area)?\\s*$" + ); + private static final Pattern NUMBERED_SECTION_HEADING = Pattern.compile("^\\d+\\.\\s+.+$"); + private static final Pattern KNOWN_LATER_SECTION_HEADING = Pattern.compile( + "^(?:Edge Cases(?: and Negative Testing)?|Negative Testing|Regression Risks|" + + "Environment(?: and Setup Notes)?|Setup Notes).*$", + Pattern.CASE_INSENSITIVE + ); + private static final Pattern ENVIRONMENT_SECTION_HEADING = Pattern.compile( + "^(?:6\\.\\s+.+|(?:\\d+\\.\\s*)?(?:Environment and Setup Notes|" + + "Setup and Environment Notes|Environment Setup Notes|Environment Notes|Setup Notes))$", + Pattern.CASE_INSENSITIVE + ); + private static final Pattern GENERATED_QA_FOOTER = Pattern.compile( + "(?ms)\\n\\s*---\\s*\\R\\s*\\*🐦 Generated by \\[CodeCrow]\\([^\\r\\n]+\\) " + + "QA Auto-Documentation\\*\\s*\\R\\s*" + + "\\s*$" + ); + + private QaDocContentParser() { + } + + public static QaDocContent parse(String markdown) { + String source = markdown == null + ? "" + : GENERATED_QA_FOOTER.matcher(markdown.trim()).replaceFirst("").trim(); + Section section = findTestCaseSection(source); + String withoutTestCases; + List testCases; + if (section == null) { + withoutTestCases = source; + testCases = parseTestCases(source); + } else { + withoutTestCases = (source.substring(0, section.start()) + + "\n\n" + + source.substring(section.end())) + .replace(TEST_CASES_START, "") + .replace(TEST_CASES_END, ""); + testCases = parseTestCases(section.content()); + } + + Section environmentSection = findEnvironmentSection(withoutTestCases); + String overview = environmentSection == null + ? normalizeDocumentPart(withoutTestCases) + : normalizeDocumentPart( + withoutTestCases.substring(0, environmentSection.start()) + + "\n\n" + + withoutTestCases.substring(environmentSection.end())); + String environment = environmentSection == null + ? null + : stripFirstHeading(environmentSection.content()); + + return new QaDocContent(overview, testCases, normalizeNullablePart(environment)); + } + + /** + * Parses only an explicitly marked section for unauthenticated disclosure. + * Legacy heading inference is intentionally excluded from this boundary. + */ + public static List parseMarkedTestCases(String markdown) { + String source = markdown == null ? "" : markdown.trim(); + MarkedSection section = findMarkedSection(source); + if (section == null) { + return List.of(); + } + return parseStructuredTestCases( + source.substring(section.testContentStart(), section.testContentEnd()).trim()); + } + + /** + * Keeps the compact Jira-facing parts of the rendered QA document while + * replacing the broad test-case and environment/setup bodies with their + * respective public-preview links. Original section headings and the + * generated ownership footer are retained. + */ + public static String replaceShareableSections( + String markdown, + String testCasesReplacement, + String environmentReplacement) { + String withTestCasesReplaced = replaceMarkedTestCaseBody(markdown, testCasesReplacement); + return replaceEnvironmentBody(withTestCasesReplaced, environmentReplacement); + } + + private static String replaceMarkedTestCaseBody(String markdown, String replacementMarkdown) { + String source = markdown == null ? "" : markdown.trim(); + String replacement = replacementMarkdown == null ? "" : replacementMarkdown.trim(); + if (replacement.isBlank()) { + throw new IllegalArgumentException("Test-case replacement is required."); + } + + MarkedSection section = findMarkedSection(source); + if (section == null) { + throw new IllegalArgumentException("A marked QA test-case section is required."); + } + + String laterSections = source + .substring(section.testContentEnd(), section.markerEnd()) + .trim(); + String afterMarker = source + .substring(section.markerEnd() + TEST_CASES_END.length()) + .trim(); + + StringBuilder result = new StringBuilder(source + .substring(0, section.contentStart()).stripTrailing()) + .append("\n") + .append(section.heading()) + .append("\n\n") + .append(replacement) + .append("\n") + .append(TEST_CASES_END); + if (!laterSections.isBlank()) { + result.append("\n\n").append(laterSections); + } + if (!afterMarker.isBlank()) { + result.append("\n\n").append(afterMarker); + } + return result.toString().trim(); + } + + private static String replaceEnvironmentBody(String markdown, String replacementMarkdown) { + String replacement = replacementMarkdown == null ? "" : replacementMarkdown.trim(); + if (replacement.isBlank()) { + throw new IllegalArgumentException("Environment replacement is required."); + } + + Section section = findEnvironmentSection(markdown); + if (section == null) { + return markdown; + } + + Matcher heading = HEADING_PATTERN.matcher(section.content()); + if (!heading.find()) { + return markdown; + } + + StringBuilder result = new StringBuilder(markdown + .substring(0, section.start()) + .stripTrailing()) + .append("\n\n") + .append(heading.group().trim()) + .append("\n\n") + .append(replacement); + String suffix = markdown.substring(section.end()).stripLeading(); + if (!suffix.isBlank()) { + result.append("\n\n").append(suffix); + } + return result.toString().trim(); + } + + private static Section findTestCaseSection(String markdown) { + MarkedSection marked = findMarkedSection(markdown); + if (marked != null) { + int sectionEnd = marked.testContentEnd() == marked.markerEnd() + ? marked.markerEnd() + TEST_CASES_END.length() + : marked.testContentEnd(); + return new Section( + marked.markerStart(), + sectionEnd, + markdown.substring(marked.testContentStart(), marked.testContentEnd()).trim()); + } + + Matcher legacy = LEGACY_SECTION_PATTERN.matcher(markdown); + if (!legacy.find()) { + return null; + } + int headingLevel = legacy.group(1).length(); + int sectionEnd = markdown.length(); + Matcher headings = HEADING_PATTERN.matcher(markdown); + headings.region(legacy.end(), markdown.length()); + while (headings.find()) { + if (headings.group(1).length() <= headingLevel) { + sectionEnd = headings.start(); + break; + } + } + return new Section(legacy.start(), sectionEnd, markdown.substring(legacy.start(), sectionEnd).trim()); + } + + /** + * Models occasionally put the closing marker after later peer sections. + * Treat the next heading at the test-section level (or higher) as the real + * boundary, while retaining the marker as the outer disclosure boundary. + */ + private static MarkedSection findMarkedSection(String markdown) { + int markerStart = markdown.indexOf(TEST_CASES_START); + if (markerStart < 0) { + return null; + } + int contentStart = markerStart + TEST_CASES_START.length(); + int markerEnd = markdown.indexOf(TEST_CASES_END, contentStart); + if (markerEnd < 0) { + return null; + } + + String markedContent = markdown.substring(contentStart, markerEnd); + Matcher testHeading = LEGACY_SECTION_PATTERN.matcher(markedContent); + if (!testHeading.find()) { + return new MarkedSection( + markerStart, + contentStart, + markerEnd, + contentStart, + markerEnd, + "### 3. Test Scenarios"); + } + + int testContentStart = contentStart + testHeading.start(); + int testHeadingEnd = contentStart + testHeading.end(); + int testContentEnd = markerEnd; + int headingLevel = testHeading.group(1).length(); + + Matcher followingHeadings = HEADING_PATTERN.matcher(markdown); + followingHeadings.region(testHeadingEnd, markerEnd); + while (followingHeadings.find()) { + int followingLevel = followingHeadings.group(1).length(); + String followingTitle = followingHeadings.group(2).trim(); + if (followingLevel < headingLevel + || (followingLevel == headingLevel + && isLaterDocumentSection(followingTitle))) { + testContentEnd = followingHeadings.start(); + break; + } + } + + return new MarkedSection( + markerStart, + contentStart, + markerEnd, + testContentStart, + testContentEnd, + testHeading.group().trim()); + } + + private static boolean isLaterDocumentSection(String headingTitle) { + return NUMBERED_SECTION_HEADING.matcher(headingTitle).matches() + || KNOWN_LATER_SECTION_HEADING.matcher(headingTitle).matches(); + } + + private static Section findEnvironmentSection(String markdown) { + Matcher headings = HEADING_PATTERN.matcher(markdown); + while (headings.find()) { + String title = headings.group(2).trim(); + if (!ENVIRONMENT_SECTION_HEADING.matcher(title).matches()) { + continue; + } + + int headingLevel = headings.group(1).length(); + int sectionStart = headings.start(); + int sectionEnd = findGeneratedFooterStart(markdown, headings.end()); + Matcher followingHeadings = HEADING_PATTERN.matcher(markdown); + followingHeadings.region(headings.end(), sectionEnd); + while (followingHeadings.find()) { + if (followingHeadings.group(1).length() <= headingLevel) { + sectionEnd = followingHeadings.start(); + break; + } + } + return new Section( + sectionStart, + sectionEnd, + markdown.substring(sectionStart, sectionEnd).trim()); + } + return null; + } + + private static int findGeneratedFooterStart(String markdown, int fromIndex) { + Matcher footer = GENERATED_QA_FOOTER.matcher(markdown); + return footer.find(fromIndex) ? footer.start() : markdown.length(); + } + + private static String stripFirstHeading(String section) { + return HEADING_PATTERN.matcher(section).replaceFirst("").trim(); + } + + private static String normalizeDocumentPart(String value) { + return value == null + ? "" + : value.trim().replaceAll("\\n{3,}", "\n\n"); + } + + private static String normalizeNullablePart(String value) { + String normalized = normalizeDocumentPart(value); + return normalized.isBlank() ? null : normalized; + } + + private static List parseTestCases(String section) { + List structured = parseStructuredTestCases(section); + if (!structured.isEmpty()) { + return structured; + } + + String fallback = stripSectionHeading(section); + if (fallback.isBlank()) { + return List.of(); + } + return List.of(new QaDocTestCase("Test scenarios", null, null, fallback)); + } + + private static List parseStructuredTestCases(String section) { + Matcher matcher = SCENARIO_PATTERN.matcher(section); + List matches = new ArrayList<>(); + while (matcher.find()) { + matches.add(new ScenarioMatch( + matcher.start(), + matcher.end(), + matcher.group(1).trim(), + matcher.group(2).toUpperCase(Locale.ROOT) + )); + } + + if (matches.isEmpty()) { + return List.of(); + } + + List testCases = new ArrayList<>(); + for (int index = 0; index < matches.size(); index++) { + ScenarioMatch current = matches.get(index); + int descriptionEnd = index + 1 < matches.size() + ? matches.get(index + 1).start() + : section.length(); + + Matcher interveningHeading = HEADING_PATTERN.matcher(section); + interveningHeading.region(current.end(), descriptionEnd); + if (interveningHeading.find()) { + descriptionEnd = interveningHeading.start(); + } + + String description = section.substring(current.end(), descriptionEnd).trim(); + testCases.add(new QaDocTestCase( + current.title(), + current.priority(), + findFunctionalArea(section, current.start()), + description + )); + } + return List.copyOf(testCases); + } + + private static String findFunctionalArea(String section, int beforePosition) { + Matcher headings = HEADING_PATTERN.matcher(section); + String latest = null; + while (headings.find() && headings.start() < beforePosition) { + String candidate = headings.group(2).trim(); + if (!candidate.toLowerCase(Locale.ROOT).contains("test scenarios")) { + latest = candidate; + } + } + return latest; + } + + private static String stripSectionHeading(String section) { + return LEGACY_SECTION_PATTERN.matcher(section).replaceFirst("").trim(); + } + + private record Section(int start, int end, String content) { + } + + private record MarkedSection( + int markerStart, + int contentStart, + int markerEnd, + int testContentStart, + int testContentEnd, + String heading + ) { + } + + private record ScenarioMatch(int start, int end, String title, String priority) { + } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocPublicShareResource.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocPublicShareResource.java new file mode 100644 index 00000000..e4a8e827 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocPublicShareResource.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.service.qadoc; + +public final class QaDocPublicShareResource { + + /** Public, read-only preview of the intentionally shareable QA document tabs. */ + public static final String DOCUMENT = "qa-document"; + + private QaDocPublicShareResource() { + } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocTestCase.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocTestCase.java new file mode 100644 index 00000000..8bea1504 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qadoc/QaDocTestCase.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.service.qadoc; + +public record QaDocTestCase( + String title, + String priority, + String functionalArea, + String descriptionMarkdown +) { +} + diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.27.0__rag_index_active_job.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.27.0__rag_index_active_job.sql new file mode 100644 index 00000000..74714a2e --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.27.0__rag_index_active_job.sql @@ -0,0 +1,8 @@ +-- Correlate the project-level RAG status with the durable job that currently +-- owns it. Recovery uses this identity to preserve a newer producer even +-- before that producer has registered its exact-generation operation. +ALTER TABLE rag_index_status + ADD COLUMN IF NOT EXISTS active_job_id BIGINT; + +CREATE INDEX IF NOT EXISTS idx_rag_index_status_active_job + ON rag_index_status(active_job_id); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.28.0__rag_operation_lock_owner.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.28.0__rag_operation_lock_owner.sql new file mode 100644 index 00000000..ff423438 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.28.0__rag_operation_lock_owner.sql @@ -0,0 +1,5 @@ +-- Persist the exact analysis-lock owner used by a RAG generation attempt. +-- Recovery releases only this key, so a same-revision retry cannot lose its +-- newly acquired lock to a delayed recovery pass. +ALTER TABLE rag_index_operation + ADD COLUMN IF NOT EXISTS analysis_lock_key VARCHAR(500); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.29.0__rag_transient_cleanup_claim.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.29.0__rag_transient_cleanup_claim.sql new file mode 100644 index 00000000..a4f06efa --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.29.0__rag_transient_cleanup_claim.sql @@ -0,0 +1,9 @@ +-- Durable claim for transient-generation cleanup. Readers and rebuilds reject +-- a claimed row while remote physical collections are being removed. +ALTER TABLE rag_branch_index + ADD COLUMN IF NOT EXISTS cleanup_claim_token VARCHAR(64), + ADD COLUMN IF NOT EXISTS cleanup_claimed_at TIMESTAMP WITH TIME ZONE; + +CREATE INDEX IF NOT EXISTS idx_rag_branch_cleanup_claim + ON rag_branch_index(index_kind, cleanup_claimed_at) + WHERE cleanup_claim_token IS NOT NULL; diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java new file mode 100644 index 00000000..ea7006be --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java @@ -0,0 +1,65 @@ +package org.rostilos.codecrow.core.persistence.repository.job; + +import org.junit.jupiter.api.Test; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.Query; + +import java.lang.reflect.Method; +import java.time.OffsetDateTime; + +import static org.assertj.core.api.Assertions.assertThat; + +class JobRepositoryRecoveryQueryTest { + + @Test + void genericWebhookSelectionIsLimitedToPrAndBranchAnalysis() throws Exception { + assertGenericWebhookTypes(query( + "findRecoverableWebhookJobs", + OffsetDateTime.class, + Pageable.class)); + assertGenericWebhookTypes(query( + "findAbandonedRunningWebhookJobs", + OffsetDateTime.class, + Pageable.class)); + } + + @Test + void genericWebhookAtomicClaimsRepeatTheSameTypeBoundary() throws Exception { + assertGenericWebhookTypes(query( + "claimRecoverableWebhookJob", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class)); + assertGenericWebhookTypes(query( + "claimAbandonedRunningWebhookJob", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class)); + } + + private static String query(String methodName, Class... parameters) + throws Exception { + Method method = JobRepository.class.getDeclaredMethod(methodName, parameters); + Query annotation = method.getAnnotation(Query.class); + assertThat(annotation).as("@Query on %s", methodName).isNotNull(); + return annotation.value(); + } + + private static void assertGenericWebhookTypes(String query) { + assertThat(query) + .contains("JobTriggerSource.WEBHOOK") + .contains("JobType.PR_ANALYSIS") + .contains("JobType.BRANCH_ANALYSIS") + .doesNotContain("JobType.BRANCH_RECONCILIATION") + .doesNotContain("JobType.RAG_INITIAL_INDEX") + .doesNotContain("JobType.RAG_INCREMENTAL_INDEX") + .doesNotContain("JobType.MANUAL_ANALYSIS") + .doesNotContain("JobType.REPO_SYNC") + .doesNotContain("JobType.SUMMARIZE_COMMAND") + .doesNotContain("JobType.ASK_COMMAND") + .doesNotContain("JobType.ANALYZE_COMMAND") + .doesNotContain("JobType.REVIEW_COMMAND") + .doesNotContain("JobType.QA_DOC_COMMAND") + .doesNotContain("JobType.IGNORED_COMMENT"); + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/LegacyRagJobRecoveryQueryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/LegacyRagJobRecoveryQueryTest.java new file mode 100644 index 00000000..86cd5e73 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/LegacyRagJobRecoveryQueryTest.java @@ -0,0 +1,110 @@ +package org.rostilos.codecrow.core.persistence.repository.job; + +import org.junit.jupiter.api.Test; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.Query; + +import java.lang.reflect.Method; +import java.time.OffsetDateTime; + +import static org.assertj.core.api.Assertions.assertThat; + +class LegacyRagJobRecoveryQueryTest { + + @Test + void leaseRenewalIsGuardedByTypeStatusAgeAndMissingExactOperation() + throws Exception { + Query query = query( + "renewLegacyRagJobLease", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class); + + assertThat(query.nativeQuery()).isTrue(); + assertThat(query.value()) + .contains("RAG_INCREMENTAL_INDEX") + .contains("RUNNING") + .contains("updated_at >= :validAfter") + .contains("NOT EXISTS") + .contains("rag_index_operation"); + } + + @Test + void abandonedSelectionUsesScalarCoordinatesAndExcludesExactOperations() + throws Exception { + Query query = query( + "findAbandonedLegacyRagJobs", + OffsetDateTime.class, + Pageable.class); + + assertThat(query.value()) + .contains("j.id AS jobId") + .contains("j.project.id AS projectId") + .contains("RAG_INCREMENTAL_INDEX") + .contains("JobStatus.PENDING") + .contains("JobStatus.RUNNING") + .contains("j.updatedAt < :threshold") + .contains("NOT EXISTS") + .contains("RagIndexOperation"); + } + + @Test + void abandonmentClaimRepeatsEveryLeaseFenceAtomically() throws Exception { + Query query = query( + "failAbandonedLegacyRagJob", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class, + String.class); + + assertThat(query.nativeQuery()).isTrue(); + assertThat(query.value()) + .contains("status = 'FAILED'") + .contains("RAG_INCREMENTAL_INDEX") + .contains("'PENDING', 'QUEUED', 'RUNNING', 'WAITING'") + .contains("updated_at < :threshold") + .contains("NOT EXISTS") + .contains("rag_index_operation"); + } + + @Test + void completionRepeatsEveryLeaseFenceAtomically() throws Exception { + Query query = query( + "completeOwnedLegacyRagJob", + Long.class, + OffsetDateTime.class, + OffsetDateTime.class); + + assertThat(query.nativeQuery()).isTrue(); + assertThat(query.value()) + .contains("status = 'COMPLETED'") + .contains("RAG_INCREMENTAL_INDEX") + .contains("status = 'RUNNING'") + .contains("updated_at >= :validAfter") + .contains("NOT EXISTS") + .contains("rag_index_operation"); + } + + @Test + void failedProjectionRepairIsOwnedByTheSameJob() throws Exception { + Query query = query( + "findFailedLegacyRagJobsWithActiveStatus", + Pageable.class); + + assertThat(query.value()) + .contains("JobStatus.FAILED") + .contains("s.activeJobId = j.id") + .contains("RagIndexingStatus.INDEXING") + .contains("RagIndexingStatus.UPDATING") + .contains("NOT EXISTS") + .contains("RagIndexOperation"); + } + + private static Query query(String name, Class... parameterTypes) + throws Exception { + Method method = JobRepository.class.getDeclaredMethod(name, parameterTypes); + Query query = method.getAnnotation(Query.class); + assertThat(query).as("@Query on %s", name).isNotNull(); + return query; + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRecoveryQueryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRecoveryQueryTest.java new file mode 100644 index 00000000..4dca8202 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRecoveryQueryTest.java @@ -0,0 +1,60 @@ +package org.rostilos.codecrow.core.persistence.repository.rag; + +import org.junit.jupiter.api.Test; +import org.springframework.data.jpa.repository.Query; + +import java.lang.reflect.Method; +import java.time.OffsetDateTime; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class RagIndexOperationRecoveryQueryTest { + + @Test + void abandonedSelectionReturnsOnlyDetachedSafeScalarCoordinates() + throws Exception { + Method method = RagIndexOperationRepository.class.getDeclaredMethod( + "findRecoverableOperationProjections", List.class, OffsetDateTime.class); + Query annotation = method.getAnnotation(Query.class); + + assertThat(annotation).isNotNull(); + assertThat(method.getGenericReturnType().getTypeName()) + .contains("RecoveryOperationProjection") + .doesNotContain("RagIndexOperation>"); + assertThat(annotation.value()) + .contains("o.project.id AS projectId") + .doesNotContain("SELECT o FROM"); + } + + @Test + void failedProjectionRecoveryRequiresExactStatusAndLockOwnership() + throws Exception { + String query = query("findFailedOperationsWithActiveProjections"); + + assertThat(query) + .contains("o.project_id AS \"projectId\"") + .contains("s.active_job_id = o.job_id") + .contains("l.lock_key = o.analysis_lock_key") + .doesNotContain("s.active_job_id IS NULL"); + } + + @Test + void succeededProjectionRecoveryRequiresExactStatusAndLockOwnership() + throws Exception { + String query = query("findSucceededOperationsWithActiveProjections"); + + assertThat(query) + .contains("s.active_job_id = o.job_id") + .contains("l.lock_key = o.analysis_lock_key") + .doesNotContain("s.active_job_id IS NULL"); + } + + private static String query(String methodName) throws Exception { + Method method = RagIndexOperationRepository.class.getDeclaredMethod(methodName); + Query annotation = method.getAnnotation(Query.class); + assertThat(annotation).as("@Query on %s", methodName).isNotNull(); + assertThat(annotation.nativeQuery()).isTrue(); + return annotation.value(); + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java index f2b64cb5..b10fdffa 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java @@ -51,6 +51,34 @@ void errorShouldCallLogToJobWithErrorLevel() { verify(service).logToJob(job, JobLogLevel.ERROR, "test-state", "error message"); } + @Test + @DisplayName("skipJob() should complete compatibility implementations without failing") + void skipJobShouldUseNonFailureCompletionResult() { + TestAnalysisJobService service = spy(new TestAnalysisJobService()); + Job job = new Job(); + + service.skipJob(job, "deferred until a later trigger"); + + verify(service).completeJob(job, Map.of( + "status", "skipped", + "reason", "deferred until a later trigger")); + verify(service, never()).failJob(any(), anyString()); + } + + @Test + void externallyCompletedJobDefaultsToNotificationOnly() { + TestAnalysisJobService service = spy(new TestAnalysisJobService()); + Job job = new Job(); + + service.recordExternallyCompletedJob( + job, "rag_complete", "RAG index updated"); + + verify(service).logToJob( + job, JobLogLevel.INFO, "rag_complete", "RAG index updated"); + verify(service, never()).completeJob(any(), any()); + verify(service, never()).failJob(any(), anyString()); + } + } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceLegacyRagRecoveryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceLegacyRagRecoveryTest.java new file mode 100644 index 00000000..70a01e29 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceLegacyRagRecoveryTest.java @@ -0,0 +1,139 @@ +package org.rostilos.codecrow.core.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobLog; +import org.rostilos.codecrow.core.model.job.JobStatus; +import org.rostilos.codecrow.core.persistence.repository.job.JobLogRepository; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.springframework.data.domain.Pageable; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.OffsetDateTime; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class JobServiceLegacyRagRecoveryTest { + @Mock private JobRepository jobs; + @Mock private JobLogRepository logs; + + private JobService service; + + @BeforeEach + void setUp() { + service = new JobService(jobs, logs, new ObjectMapper()); + } + + @Test + void renewsTheExactLeaseWindowProvidedByTheProducer() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + OffsetDateTime renewedAt = OffsetDateTime.now(); + when(jobs.renewLegacyRagJobLease(91L, validAfter, renewedAt)) + .thenReturn(1); + + assertThat(service.renewLegacyRagJobLease( + 91L, validAfter, renewedAt)).isTrue(); + } + + @Test + void exposesOnlyTheBoundedRecoveryBatch() { + JobRepository.LegacyRagJobRecoveryCoordinates coordinates = + org.mockito.Mockito.mock( + JobRepository.LegacyRagJobRecoveryCoordinates.class); + when(jobs.findAbandonedLegacyRagJobs( + any(OffsetDateTime.class), any(Pageable.class))) + .thenReturn(List.of(coordinates)); + + assertThat(service.findAbandonedLegacyRagJobs( + OffsetDateTime.now(), 17)).containsExactly(coordinates); + verify(jobs).findAbandonedLegacyRagJobs( + any(OffsetDateTime.class), argThat(page -> page.getPageSize() == 17)); + } + + @Test + void reportsWhetherRecoveryWonTheAtomicFence() { + when(jobs.failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), any(), eq("abandoned"))) + .thenReturn(1); + + assertThat(service.failAbandonedLegacyRagJob( + 91L, OffsetDateTime.now(), "abandoned")).isTrue(); + } + + @Test + void recordsAnAlreadyCompletedJobWithoutRepeatingItsTerminalTransition() { + Job supplied = new Job(); + ReflectionTestUtils.setField(supplied, "id", 91L); + Job persisted = new Job(); + ReflectionTestUtils.setField(persisted, "id", 91L); + persisted.setExternalId("job-91"); + persisted.setStatus(JobStatus.COMPLETED); + when(jobs.findByIdForUpdate(91L)).thenReturn(java.util.Optional.of(persisted)); + when(logs.existsByJobIdAndStep(91L, "rag_complete")).thenReturn(false); + when(logs.getNextSequenceNumber(91L)).thenReturn(4L); + when(logs.save(any(JobLog.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + service.recordExternallyCompletedJob( + supplied, "rag_complete", "RAG index updated"); + + verify(logs).save(argThat(entry -> + entry.getJob() == persisted + && "rag_complete".equals(entry.getStep()) + && "RAG index updated".equals(entry.getMessage()))); + verify(jobs, never()).save(any()); + } + + @Test + void repeatedTerminalNotificationDoesNotDuplicateTheDurableLog() { + Job supplied = new Job(); + ReflectionTestUtils.setField(supplied, "id", 91L); + Job persisted = new Job(); + ReflectionTestUtils.setField(persisted, "id", 91L); + persisted.setExternalId("job-91"); + persisted.setStatus(JobStatus.COMPLETED); + when(jobs.findByIdForUpdate(91L)).thenReturn(java.util.Optional.of(persisted)); + when(logs.existsByJobIdAndStep(91L, "rag_complete")).thenReturn(true); + + service.recordExternallyCompletedJob( + supplied, "rag_complete", "RAG index updated"); + + verify(logs, never()).save(any()); + verify(jobs, never()).save(any()); + } + + @Test + void recordsAnAlreadyFailedJobWithoutRepeatingItsTerminalTransition() { + Job supplied = new Job(); + ReflectionTestUtils.setField(supplied, "id", 91L); + Job persisted = new Job(); + ReflectionTestUtils.setField(persisted, "id", 91L); + persisted.setExternalId("job-91"); + persisted.setStatus(JobStatus.FAILED); + when(jobs.findByIdForUpdate(91L)) + .thenReturn(java.util.Optional.of(persisted)); + when(logs.existsByJobIdAndStep(91L, "rag_recovery")).thenReturn(false); + when(logs.getNextSequenceNumber(91L)).thenReturn(4L); + when(logs.save(any(JobLog.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + service.recordExternallyFailedJob( + supplied, "rag_recovery", "Job failed: producer expired"); + + verify(logs).save(argThat(entry -> + entry.getJob() == persisted + && entry.getLevel() + == org.rostilos.codecrow.core.model.job.JobLogLevel.ERROR + && "rag_recovery".equals(entry.getStep()))); + verify(jobs, never()).save(any()); + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qadoc/QaDocContentParserTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qadoc/QaDocContentParserTest.java new file mode 100644 index 00000000..596e8bc9 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qadoc/QaDocContentParserTest.java @@ -0,0 +1,251 @@ +package org.rostilos.codecrow.core.service.qadoc; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class QaDocContentParserTest { + + @Test + void separatesMarkedTestCasesFromOverviewAndPreservesStructuredDetails() { + String markdown = """ + # QA Guide + + ## What Changed + A user-visible flow changed. + + + ## 3. Test Scenarios by Area + + ### Checkout + **Complete checkout** (HIGH) + - **Preconditions:** A product is in the cart + - **Steps:** + 1. Submit the order + - **Expected Result:** The confirmation is shown + + **Reject an empty address** (MEDIUM) + - **Expected Result:** A validation message is shown + + + ## Regression Risks + Verify saved carts. + """; + + QaDocContent result = QaDocContentParser.parse(markdown); + + assertThat(result.overviewMarkdown()) + .contains("What Changed", "Regression Risks") + .doesNotContain("Complete checkout", "codecrow-test-cases"); + assertThat(result.environmentMarkdown()).isNull(); + assertThat(result.testCases()).hasSize(2); + assertThat(result.testCases().get(0)) + .extracting(QaDocTestCase::title, QaDocTestCase::priority, QaDocTestCase::functionalArea) + .containsExactly("Complete checkout", "HIGH", "Checkout"); + assertThat(result.testCases().get(0).descriptionMarkdown()) + .contains("Preconditions", "Submit the order", "Expected Result"); + } + + @Test + void extractsLegacyEnglishTestScenarioSections() { + QaDocContent result = QaDocContentParser.parse(""" + ### 1. Change Summary + Summary + ### 3. Test Scenarios + **Open settings** (LOW) + - **Expected Result:** Settings appear + ### 4. Edge Cases + Try an empty state. + """); + + assertThat(result.testCases()).singleElement() + .extracting(QaDocTestCase::title) + .isEqualTo("Open settings"); + assertThat(result.overviewMarkdown()).doesNotContain("Open settings"); + } + + @Test + void publicParsingRequiresMarkersAndAStructuredScenario() { + assertThat(QaDocContentParser.parseMarkedTestCases(""" + + Secret overview text without a scenario + + """)).isEmpty(); + assertThat(QaDocContentParser.parseMarkedTestCases(""" + **Unmarked scenario** (HIGH) + - **Expected Result:** It works + """)).isEmpty(); + } + + @Test + void replacesOnlyTheMarkedTestCaseBodyAndKeepsTheNumberedSection() { + String result = QaDocContentParser.replaceShareableSections(""" + ### 1. Change Summary + Checkout now supports gift cards. + + ### 2. Scope + Web checkout only. + + + ### 3. Test Scenarios + + **Pay with a gift card** (HIGH) + - **Steps:** Enter a valid gift card. + - **Expected Result:** The balance is applied. + + + ### 4. Edge Cases + Test an expired gift card. + """, + "https://codecrow.cloud/share#token=ccs_public-token&tab=test-cases", + "https://codecrow.cloud/share#token=ccs_public-token&tab=environment"); + + assertThat(result) + .contains("### 1. Change Summary", "Checkout now supports gift cards") + .contains("### 2. Scope", "Web checkout only") + .contains("### 3. Test Scenarios\n\nhttps://codecrow.cloud/share#token=ccs_public-token") + .contains("### 4. Edge Cases", "Test an expired gift card") + .doesNotContain("Pay with a gift card", "The balance is applied"); + } + + @Test + void preservesLaterPeerSectionsWhenTheModelPlacesTheEndMarkerTooLate() { + String markdown = """ + ### 1. Change Summary + Checkout changed. + + + ### 3. Test Scenarios + ### Checkout + **Pay with a gift card** (HIGH) + - **Expected Result:** The balance is applied. + + ### 4. Edge Cases and Negative Testing + - Try an expired gift card. + + ### 5. Regression Risks + - Existing card payments. + + ### 6. Environment and Setup Notes + - Use the QA payment environment. + + """; + + QaDocContent parsed = QaDocContentParser.parse(markdown); + String comment = QaDocContentParser.replaceShareableSections( + markdown, + "https://codecrow.cloud/share#token=ccs_public-token&tab=test-cases", + "https://codecrow.cloud/share#token=ccs_public-token&tab=environment"); + + assertThat(parsed.testCases()).singleElement() + .extracting(QaDocTestCase::title, QaDocTestCase::functionalArea) + .containsExactly("Pay with a gift card", "Checkout"); + assertThat(parsed.overviewMarkdown()) + .contains( + "### 4. Edge Cases and Negative Testing", + "### 5. Regression Risks") + .doesNotContain( + "Pay with a gift card", + "codecrow-test-cases", + "### 6. Environment and Setup Notes", + "Use the QA payment environment"); + assertThat(parsed.environmentMarkdown()) + .isEqualTo("- Use the QA payment environment."); + assertThat(comment) + .contains("### 3. Test Scenarios\n\nhttps://codecrow.cloud/share#token=ccs_public-token&tab=test-cases") + .contains( + "### 4. Edge Cases and Negative Testing", + "Try an expired gift card", + "### 5. Regression Risks", + "Existing card payments", + "### 6. Environment and Setup Notes", + "https://codecrow.cloud/share#token=ccs_public-token&tab=environment") + .doesNotContain( + "Pay with a gift card", + "The balance is applied", + "Use the QA payment environment") + .contains("tab=test-cases\n\n\n### 4."); + } + + @Test + void replacesEnvironmentBodyAndPreservesTheGeneratedFooter() { + String markdown = """ + ### 1. Change Summary + Checkout changed. + + + ### 3. Test Scenarios + **Pay successfully** (HIGH) + - **Expected Result:** Payment succeeds. + + + ### 4. Edge Cases + Try an expired card. + + ### 5. Regression Risks + Verify saved cards. + + ### 6. Environment and Setup Notes + Use the QA payment environment. + + --- + *🐦 Generated by [CodeCrow](https://codecrow.app) QA Auto-Documentation* + + """; + QaDocContent parsed = QaDocContentParser.parse(markdown); + String comment = QaDocContentParser.replaceShareableSections( + markdown, + "https://codecrow.cloud/share#token=ccs_public-token&tab=test-cases", + "https://codecrow.cloud/share#token=ccs_public-token&tab=environment"); + + assertThat(parsed.overviewMarkdown()) + .contains("Change Summary", "Edge Cases", "Regression Risks") + .doesNotContain("Generated by", "codecrow-qa-autodoc"); + assertThat(parsed.environmentMarkdown()).isEqualTo("Use the QA payment environment."); + assertThat(comment) + .contains("### 4. Edge Cases", "Try an expired card") + .contains("### 5. Regression Risks", "Verify saved cards") + .contains("### 6. Environment and Setup Notes\n\n" + + "https://codecrow.cloud/share#token=ccs_public-token&tab=environment") + .contains("Generated by [CodeCrow]", "") + .doesNotContain("Use the QA payment environment."); + } + + @Test + void separatesSetupAndEnvironmentNotesFromAnUnmarkedDocument() { + QaDocContent parsed = QaDocContentParser.parse(""" + # QA Testing Guide — Checkout + + ## 1. What Changed + Saved cards can now be selected at checkout. + + ## 6. Setup and Environment Notes + - Enable the saved-card feature flag. + - Use a customer with an existing payment method. + """); + + assertThat(parsed.overviewMarkdown()) + .contains("What Changed", "Saved cards") + .doesNotContain("Setup and Environment Notes", "feature flag"); + assertThat(parsed.environmentMarkdown()) + .contains("Enable the saved-card feature flag", "existing payment method") + .doesNotContain("## 6."); + } + + @Test + void preservesSectionsAfterEnvironmentNotesInTheOverview() { + QaDocContent parsed = QaDocContentParser.parse(""" + ## Overview + Summary. + + ## Environment and Setup Notes + Use staging. + + ## Appendix + Contact the QA lead. + """); + + assertThat(parsed.environmentMarkdown()).isEqualTo("Use staging."); + assertThat(parsed.overviewMarkdown()).contains("Overview", "Appendix", "Contact the QA lead"); + } +} diff --git a/java-ecosystem/libs/events/src/main/java/org/rostilos/codecrow/events/EventNotificationEmitter.java b/java-ecosystem/libs/events/src/main/java/org/rostilos/codecrow/events/EventNotificationEmitter.java index 9474205b..3adcae92 100644 --- a/java-ecosystem/libs/events/src/main/java/org/rostilos/codecrow/events/EventNotificationEmitter.java +++ b/java-ecosystem/libs/events/src/main/java/org/rostilos/codecrow/events/EventNotificationEmitter.java @@ -4,11 +4,25 @@ import java.util.function.Consumer; public class EventNotificationEmitter { + private EventNotificationEmitter() { + } + + /** + * Delivers an observational progress event. A missing or disconnected + * observer must never change the durable analysis outcome. + */ public static void emitStatus(Consumer> consumer, String state, String description) { - consumer.accept( - Map.of( + if (consumer == null) { + return; + } + try { + consumer.accept(Map.of( "type", "status", "state", state, "message", description)); + } catch (RuntimeException ignored) { + // Observers are optional transport adapters. Their lifecycle is not + // part of the durable operation result. + } } } diff --git a/java-ecosystem/libs/events/src/test/java/org/rostilos/codecrow/events/EventNotificationEmitterTest.java b/java-ecosystem/libs/events/src/test/java/org/rostilos/codecrow/events/EventNotificationEmitterTest.java new file mode 100644 index 00000000..a7ec8a1d --- /dev/null +++ b/java-ecosystem/libs/events/src/test/java/org/rostilos/codecrow/events/EventNotificationEmitterTest.java @@ -0,0 +1,28 @@ +package org.rostilos.codecrow.events; + +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThatCode; + +class EventNotificationEmitterTest { + + @Test + void missingObserverIsANoOp() { + assertThatCode(() -> EventNotificationEmitter.emitStatus(null, "started", "Started")) + .doesNotThrowAnyException(); + } + + @Test + void observerFailureCannotChangeOperationOutcome() { + Consumer> disconnected = event -> { + throw new IllegalStateException("stream disconnected"); + }; + + assertThatCode(() -> EventNotificationEmitter.emitStatus( + disconnected, "completed", "Completed")) + .doesNotThrowAnyException(); + } +} diff --git a/java-ecosystem/libs/public-share/pom.xml b/java-ecosystem/libs/public-share/pom.xml new file mode 100644 index 00000000..53c27784 --- /dev/null +++ b/java-ecosystem/libs/public-share/pom.xml @@ -0,0 +1,53 @@ + + + 4.0.0 + + + org.rostilos.codecrow + codecrow-parent + 1.0 + ../../pom.xml + + + codecrow-public-share + Purpose-neutral opaque public-share credentials and persistence + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-autoconfigure + + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-junit-jupiter + test + + + org.assertj + assertj-core + test + + + org.springframework.boot + spring-boot-starter-test + test + + + com.h2database + h2 + test + + + diff --git a/java-ecosystem/libs/public-share/src/main/java/module-info.java b/java-ecosystem/libs/public-share/src/main/java/module-info.java new file mode 100644 index 00000000..67019a59 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/java/module-info.java @@ -0,0 +1,24 @@ +module org.rostilos.codecrow.publicshare { + requires jakarta.persistence; + requires spring.data.jpa; + requires spring.data.commons; + requires spring.context; + requires spring.boot.autoconfigure; + requires spring.beans; + requires spring.core; + requires spring.tx; + requires org.hibernate.orm.core; + + exports org.rostilos.codecrow.publicshare.api; + exports org.rostilos.codecrow.publicshare.config; + exports org.rostilos.codecrow.publicshare.service; + + opens org.rostilos.codecrow.publicshare.config + to spring.core, spring.beans, spring.context; + opens org.rostilos.codecrow.publicshare.model + to org.hibernate.orm.core, spring.core, spring.beans, spring.context; + opens org.rostilos.codecrow.publicshare.persistence + to spring.core, spring.beans, spring.context; + opens org.rostilos.codecrow.publicshare.service + to spring.core, spring.beans, spring.context; +} diff --git a/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/api/IssuedPublicShare.java b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/api/IssuedPublicShare.java new file mode 100644 index 00000000..d7e40da3 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/api/IssuedPublicShare.java @@ -0,0 +1,14 @@ +package org.rostilos.codecrow.publicshare.api; + +/** A raw public-share credential. The value is returned once and is never persisted. */ +public record IssuedPublicShare(String token) { + + public String toFrontendUrl(String frontendBaseUrl) { + if (frontendBaseUrl == null || frontendBaseUrl.isBlank()) { + throw new IllegalArgumentException("Frontend base URL is required for a public share link."); + } + String base = frontendBaseUrl.trim().replaceAll("/+$", ""); + // Fragments are not sent in HTTP request targets or Referer headers. + return base + "/share#token=" + token; + } +} diff --git a/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/api/ResolvedPublicShare.java b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/api/ResolvedPublicShare.java new file mode 100644 index 00000000..a3c7b7ef --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/api/ResolvedPublicShare.java @@ -0,0 +1,4 @@ +package org.rostilos.codecrow.publicshare.api; + +public record ResolvedPublicShare(String resourceType, String resourceKey) { +} diff --git a/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/config/PublicShareAutoConfiguration.java b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/config/PublicShareAutoConfiguration.java new file mode 100644 index 00000000..95c68f35 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/config/PublicShareAutoConfiguration.java @@ -0,0 +1,24 @@ +package org.rostilos.codecrow.publicshare.config; + +import org.rostilos.codecrow.publicshare.persistence.PublicShareLinkRepository; +import org.rostilos.codecrow.publicshare.service.PublicShareLinkService; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.Bean; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +/** + * Self-contained Spring Boot integration for opaque public-share credentials. + * Applications opt in by depending on this artifact and do not scan its + * entity, repository, or service implementation packages themselves. + */ +@AutoConfiguration +@EnableJpaRepositories(basePackages = "org.rostilos.codecrow.publicshare.persistence") +@EntityScan(basePackages = "org.rostilos.codecrow.publicshare.model") +public class PublicShareAutoConfiguration { + + @Bean + public PublicShareLinkService publicShareLinkService(PublicShareLinkRepository repository) { + return new PublicShareLinkService(repository); + } +} diff --git a/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/model/PublicShareLink.java b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/model/PublicShareLink.java new file mode 100644 index 00000000..dfdb3028 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/model/PublicShareLink.java @@ -0,0 +1,89 @@ +package org.rostilos.codecrow.publicshare.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.PrePersist; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; + +import java.time.OffsetDateTime; + +@Entity +@Table( + name = "public_share_link", + uniqueConstraints = @UniqueConstraint( + name = "uq_public_share_link_token_hash", + columnNames = "token_hash" + ), + indexes = @Index( + name = "idx_public_share_link_resource", + columnList = "resource_type, resource_key" + ) +) +public class PublicShareLink { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(nullable = false, updatable = false) + private Long id; + + @Column(name = "resource_type", nullable = false, length = 80, updatable = false) + private String resourceType; + + @Column(name = "resource_key", nullable = false, length = 255, updatable = false) + private String resourceKey; + + @Column(name = "token_hash", nullable = false, length = 64, updatable = false) + private String tokenHash; + + @Column(name = "expires_at") + private OffsetDateTime expiresAt; + + @Column(name = "revoked_at") + private OffsetDateTime revokedAt; + + @Column(name = "created_at", nullable = false, updatable = false) + private OffsetDateTime createdAt; + + protected PublicShareLink() { + } + + public PublicShareLink(String resourceType, + String resourceKey, + String tokenHash, + OffsetDateTime expiresAt) { + this.resourceType = resourceType; + this.resourceKey = resourceKey; + this.tokenHash = tokenHash; + this.expiresAt = expiresAt; + } + + @PrePersist + void onCreate() { + if (createdAt == null) { + createdAt = OffsetDateTime.now(); + } + } + + public boolean isActiveAt(OffsetDateTime now) { + return revokedAt == null && (expiresAt == null || expiresAt.isAfter(now)); + } + + public void revoke(OffsetDateTime now) { + if (revokedAt == null) { + revokedAt = now; + } + } + + public Long getId() { return id; } + public String getResourceType() { return resourceType; } + public String getResourceKey() { return resourceKey; } + public String getTokenHash() { return tokenHash; } + public OffsetDateTime getExpiresAt() { return expiresAt; } + public OffsetDateTime getRevokedAt() { return revokedAt; } + public OffsetDateTime getCreatedAt() { return createdAt; } +} diff --git a/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/persistence/PublicShareLinkRepository.java b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/persistence/PublicShareLinkRepository.java new file mode 100644 index 00000000..f7f6052a --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/persistence/PublicShareLinkRepository.java @@ -0,0 +1,11 @@ +package org.rostilos.codecrow.publicshare.persistence; + +import org.rostilos.codecrow.publicshare.model.PublicShareLink; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; + +public interface PublicShareLinkRepository extends JpaRepository { + + Optional findByTokenHash(String tokenHash); +} diff --git a/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/service/PublicShareLinkService.java b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/service/PublicShareLinkService.java new file mode 100644 index 00000000..3cbe95e0 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/java/org/rostilos/codecrow/publicshare/service/PublicShareLinkService.java @@ -0,0 +1,118 @@ +package org.rostilos.codecrow.publicshare.service; + +import org.rostilos.codecrow.publicshare.api.IssuedPublicShare; +import org.rostilos.codecrow.publicshare.api.ResolvedPublicShare; +import org.rostilos.codecrow.publicshare.model.PublicShareLink; +import org.rostilos.codecrow.publicshare.persistence.PublicShareLinkRepository; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.OffsetDateTime; +import java.util.Base64; +import java.util.HexFormat; +import java.util.Optional; + +/** + * Issues and resolves opaque public-share credentials. + * + *

The token is deliberately unrelated to application JWTs: it contains no + * claims or identifiers, is generated from 256 bits of randomness, and only a + * one-way SHA-256 hash is stored.

+ */ +@Transactional +public class PublicShareLinkService { + + private static final String TOKEN_PREFIX = "ccs_"; + private static final int TOKEN_BYTES = 32; + + private final PublicShareLinkRepository repository; + private final SecureRandom secureRandom; + + public PublicShareLinkService(PublicShareLinkRepository repository) { + this(repository, new SecureRandom()); + } + + PublicShareLinkService(PublicShareLinkRepository repository, SecureRandom secureRandom) { + this.repository = repository; + this.secureRandom = secureRandom; + } + + public IssuedPublicShare issue(String resourceType, String resourceKey) { + return issue(resourceType, resourceKey, null); + } + + public IssuedPublicShare issue(String resourceType, + String resourceKey, + OffsetDateTime expiresAt) { + String normalizedType = requireValue(resourceType, "Resource type", 80); + String normalizedKey = requireValue(resourceKey, "Resource key", 255); + if (expiresAt != null && !expiresAt.isAfter(OffsetDateTime.now())) { + throw new IllegalArgumentException("Public share expiry must be in the future."); + } + + byte[] randomBytes = new byte[TOKEN_BYTES]; + secureRandom.nextBytes(randomBytes); + String token = TOKEN_PREFIX + Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes); + + repository.save(new PublicShareLink( + normalizedType, + normalizedKey, + hash(token), + expiresAt + )); + return new IssuedPublicShare(token); + } + + @Transactional(readOnly = true) + public Optional resolve(String token) { + if (!hasValidShape(token)) { + return Optional.empty(); + } + OffsetDateTime now = OffsetDateTime.now(); + return repository.findByTokenHash(hash(token)) + .filter(link -> link.isActiveAt(now)) + .map(link -> new ResolvedPublicShare( + link.getResourceType(), + link.getResourceKey() + )); + } + + public boolean revoke(String token) { + if (!hasValidShape(token)) { + return false; + } + Optional link = repository.findByTokenHash(hash(token)); + link.ifPresent(value -> value.revoke(OffsetDateTime.now())); + return link.isPresent(); + } + + private static boolean hasValidShape(String token) { + return token != null + && token.startsWith(TOKEN_PREFIX) + && token.length() >= TOKEN_PREFIX.length() + 40 + && token.length() <= 128; + } + + private static String requireValue(String value, String fieldName, int maxLength) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(fieldName + " is required."); + } + String normalized = value.trim(); + if (normalized.length() > maxLength) { + throw new IllegalArgumentException(fieldName + " is too long."); + } + return normalized; + } + + static String hash(String token) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest(token.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } +} diff --git a/java-ecosystem/libs/public-share/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/java-ecosystem/libs/public-share/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..0789fd57 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +org.rostilos.codecrow.publicshare.config.PublicShareAutoConfiguration diff --git a/java-ecosystem/libs/public-share/src/main/resources/db/migration/managed/V2.25.0__public_share_links.sql b/java-ecosystem/libs/public-share/src/main/resources/db/migration/managed/V2.25.0__public_share_links.sql new file mode 100644 index 00000000..d6d22654 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/resources/db/migration/managed/V2.25.0__public_share_links.sql @@ -0,0 +1,23 @@ +-- Generic opaque public-share credentials. +-- +-- Tokens are random bearer credentials. Only their SHA-256 hashes are stored; +-- the resource type/key pair is resolved server-side and is never encoded in +-- the public URL. + +CREATE TABLE IF NOT EXISTS public_share_link ( + id BIGSERIAL PRIMARY KEY, + resource_type VARCHAR(80) NOT NULL, + resource_key VARCHAR(255) NOT NULL, + token_hash VARCHAR(64) NOT NULL, + expires_at TIMESTAMPTZ, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT uq_public_share_link_token_hash UNIQUE (token_hash) +); + +CREATE INDEX IF NOT EXISTS idx_public_share_link_resource + ON public_share_link(resource_type, resource_key); + +COMMENT ON TABLE public_share_link IS + 'Opaque, purpose-neutral public-share credentials. Raw bearer tokens are never persisted.'; diff --git a/java-ecosystem/libs/public-share/src/main/resources/db/migration/managed/V2.26.0__rename_qa_document_share_resource.sql b/java-ecosystem/libs/public-share/src/main/resources/db/migration/managed/V2.26.0__rename_qa_document_share_resource.sql new file mode 100644 index 00000000..be0cf571 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/main/resources/db/migration/managed/V2.26.0__rename_qa_document_share_resource.sql @@ -0,0 +1,6 @@ +-- The QA share exposes the deliberately public Overview, Test cases, and +-- Environment tabs. Keep previously issued opaque credentials resolvable while +-- aligning the persisted resource type with that complete contract. +UPDATE public_share_link +SET resource_type = 'qa-document' +WHERE resource_type = 'qa-doc-test-cases'; diff --git a/java-ecosystem/libs/public-share/src/test/java/org/rostilos/codecrow/publicshare/config/PublicShareAutoConfigurationContextTest.java b/java-ecosystem/libs/public-share/src/test/java/org/rostilos/codecrow/publicshare/config/PublicShareAutoConfigurationContextTest.java new file mode 100644 index 00000000..980a155b --- /dev/null +++ b/java-ecosystem/libs/public-share/src/test/java/org/rostilos/codecrow/publicshare/config/PublicShareAutoConfigurationContextTest.java @@ -0,0 +1,47 @@ +package org.rostilos.codecrow.publicshare.config; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.publicshare.persistence.PublicShareLinkRepository; +import org.rostilos.codecrow.publicshare.service.PublicShareLinkService; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; +import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +class PublicShareAutoConfigurationContextTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of( + DataSourceAutoConfiguration.class, + HibernateJpaAutoConfiguration.class, + TransactionAutoConfiguration.class, + PublicShareAutoConfiguration.class + )) + .withPropertyValues( + "spring.datasource.url=jdbc:h2:mem:public-share;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", + "spring.datasource.username=sa", + "spring.datasource.password=", + "spring.jpa.hibernate.ddl-auto=create-drop" + ); + + @Test + void registersAndPersistsWithoutApplicationPackageScanning() { + contextRunner.run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context).hasSingleBean(PublicShareLinkRepository.class); + assertThat(context).hasSingleBean(PublicShareLinkService.class); + + PublicShareLinkService service = context.getBean(PublicShareLinkService.class); + String token = service.issue("test-resource", "internal-42").token(); + + assertThat(service.resolve(token)) + .hasValueSatisfying(resolved -> { + assertThat(resolved.resourceType()).isEqualTo("test-resource"); + assertThat(resolved.resourceKey()).isEqualTo("internal-42"); + }); + }); + } +} diff --git a/java-ecosystem/libs/public-share/src/test/java/org/rostilos/codecrow/publicshare/config/PublicShareAutoConfigurationTest.java b/java-ecosystem/libs/public-share/src/test/java/org/rostilos/codecrow/publicshare/config/PublicShareAutoConfigurationTest.java new file mode 100644 index 00000000..b4b4ec07 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/test/java/org/rostilos/codecrow/publicshare/config/PublicShareAutoConfigurationTest.java @@ -0,0 +1,27 @@ +package org.rostilos.codecrow.publicshare.config; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +import static org.assertj.core.api.Assertions.assertThat; + +class PublicShareAutoConfigurationTest { + + @Test + void ownsItsSpringIntegrationBoundaries() { + assertThat(PublicShareAutoConfiguration.class) + .hasAnnotation(AutoConfiguration.class); + + EnableJpaRepositories repositories = PublicShareAutoConfiguration.class + .getAnnotation(EnableJpaRepositories.class); + assertThat(repositories.basePackages()) + .containsExactly("org.rostilos.codecrow.publicshare.persistence"); + + EntityScan entities = PublicShareAutoConfiguration.class + .getAnnotation(EntityScan.class); + assertThat(entities.basePackages()) + .containsExactly("org.rostilos.codecrow.publicshare.model"); + } +} diff --git a/java-ecosystem/libs/public-share/src/test/java/org/rostilos/codecrow/publicshare/service/PublicShareLinkServiceTest.java b/java-ecosystem/libs/public-share/src/test/java/org/rostilos/codecrow/publicshare/service/PublicShareLinkServiceTest.java new file mode 100644 index 00000000..b90ad0f7 --- /dev/null +++ b/java-ecosystem/libs/public-share/src/test/java/org/rostilos/codecrow/publicshare/service/PublicShareLinkServiceTest.java @@ -0,0 +1,58 @@ +package org.rostilos.codecrow.publicshare.service; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.publicshare.api.IssuedPublicShare; +import org.rostilos.codecrow.publicshare.api.ResolvedPublicShare; +import org.rostilos.codecrow.publicshare.model.PublicShareLink; +import org.rostilos.codecrow.publicshare.persistence.PublicShareLinkRepository; + +import java.security.SecureRandom; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class PublicShareLinkServiceTest { + + @Mock + private PublicShareLinkRepository repository; + @Mock + private SecureRandom secureRandom; + + @Test + void storesOnlyAHashAndResolvesTheOpaqueResourceReference() { + when(repository.save(any(PublicShareLink.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + PublicShareLinkService service = new PublicShareLinkService(repository, secureRandom); + + IssuedPublicShare issued = service.issue("report-preview", "internal-key-42"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(PublicShareLink.class); + verify(repository).save(captor.capture()); + PublicShareLink stored = captor.getValue(); + assertThat(issued.token()).startsWith("ccs_"); + assertThat(stored.getTokenHash()) + .hasSize(64) + .isNotEqualTo(issued.token()) + .isEqualTo(PublicShareLinkService.hash(issued.token())); + + when(repository.findByTokenHash(stored.getTokenHash())).thenReturn(Optional.of(stored)); + assertThat(service.resolve(issued.token())) + .contains(new ResolvedPublicShare("report-preview", "internal-key-42")); + } + + @Test + void rejectsValuesThatAreNotPublicShareTokensWithoutQueryingByRawValue() { + PublicShareLinkService service = new PublicShareLinkService(repository, secureRandom); + + assertThat(service.resolve("eyJhbGciOiJIUzI1NiJ9.jwt.payload")).isEmpty(); + assertThat(service.resolve("ccs_short")).isEmpty(); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java new file mode 100644 index 00000000..d21a3dbf --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java @@ -0,0 +1,173 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.service.AnalysisJobService; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** + * Commits the durable ownership boundary for an exact branch snapshot. + * + *

The RAG operation is registered before its child job is created. The job + * and operation are then linked and moved to RUNNING in this one transaction, + * so recovery can never observe a committed running exact-generation job + * without a corresponding operation.

+ */ +@Service +public class BranchIndexBuildAdmissionService { + + public enum BuildOrigin { + AUTOMATIC("automatic"), + OPERATOR("operator"); + + private final String fingerprintLabel; + + BuildOrigin(String fingerprintLabel) { + this.fingerprintLabel = fingerprintLabel; + } + } + + public enum ProjectStatusAdmission { + NONE, + INDEXING, + UPDATING + } + + public record AdmittedBuild( + Job job, + BranchIndexGenerationBuildService.PreparedBuild preparedBuild, + ProjectStatusAdmission statusAdmission) { + } + + private final RagBranchIndexRegistryService registryService; + private final AnalysisJobService jobService; + private final RagIndexTrackingService trackingService; + + public BranchIndexBuildAdmissionService( + RagBranchIndexRegistryService registryService, + AnalysisJobService jobService, + RagIndexTrackingService trackingService) { + this.registryService = registryService; + this.jobService = jobService; + this.trackingService = trackingService; + } + + @Transactional + public AdmittedBuild admit( + Project project, + String branch, + String revision, + RagBranchIndexKind kind, + JobTriggerSource triggerSource, + String analysisLockKey, + BuildOrigin origin) { + String lockKey = requireText(analysisLockKey, "analysisLockKey"); + BuildOrigin buildOrigin = origin != null ? origin : BuildOrigin.AUTOMATIC; + + var registration = registryService.registerBuild( + project, + branch, + kind, + null, + revision, + operationFingerprint(buildOrigin, lockKey)); + if (registration.existingOperation()) { + // A lock key identifies one acquisition. Seeing it again means a + // previous admission committed and recovery already owns that + // operation; creating a second child job would split ownership. + throw new IllegalStateException( + "Exact RAG build for this lock was already admitted"); + } + + BranchIndexGenerationBuildService.PreparedBuild prepared = + BranchIndexGenerationBuildService.prepare(registration, lockKey); + var activeSource = registration.generation().getParentGeneration(); + ProjectStatusAdmission projectStatus = kind != RagBranchIndexKind.PRIMARY + ? ProjectStatusAdmission.NONE + : (activeSource == null + ? ProjectStatusAdmission.INDEXING + : ProjectStatusAdmission.UPDATING); + + if (projectStatus == ProjectStatusAdmission.UPDATING) { + // The active exact generation is the authoritative completed + // checkpoint. Align a stale/failed legacy status under the branch + // lock before switching it to UPDATING, all in this transaction. + trackingService.preparePublishedGenerationForUpdate( + project, + branch, + activeSource.getRevision(), + activeSource.getFileCount(), + activeSource.getChunkCount()); + } + Job job = jobService.createRagIndexJob( + project, + projectStatus == ProjectStatusAdmission.INDEXING, + triggerSource, + branch, + revision); + if (job == null || job.getId() == null) { + throw new IllegalStateException( + "A durable branch-bound RAG job could not be created"); + } + + // Attach the operation before starting the job. The surrounding + // transaction commits both state transitions together. + registryService.startBuild(prepared.operationId(), job.getId(), lockKey); + jobService.startJob(job); + if (projectStatus == ProjectStatusAdmission.INDEXING) { + trackingService.markIndexingStarted( + project, branch, revision, job.getId()); + } else if (projectStatus == ProjectStatusAdmission.UPDATING) { + trackingService.markUpdatingStarted( + project, branch, revision, job.getId()); + } + return new AdmittedBuild(job, prepared, projectStatus); + } + + /** + * Immediately terminalizes an admitted build that cannot reach execute. + * This is only for failures in the narrow local hand-off after admission; + * failures inside execute are owned by the build service itself. + */ + public void abortOperation(AdmittedBuild admission, String diagnostic) { + if (admission == null) { + return; + } + String failure = diagnostic == null || diagnostic.isBlank() + ? "Exact RAG build failed before execution" + : diagnostic; + registryService.fail(admission.preparedBuild().operationId(), failure); + } + + static String operationFingerprint(BuildOrigin origin, String analysisLockKey) { + return "exact-full-snapshot:" + origin.fingerprintLabel + ":" + + digest(requireText(analysisLockKey, "analysisLockKey")); + } + + private static String requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " is required"); + } + return value.trim(); + } + + private static String digest(String value) { + try { + return HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException failure) { + throw new IllegalStateException("SHA-256 is unavailable", failure); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java index fa465d3d..1853e132 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.ragengine.branch; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; @@ -10,6 +11,8 @@ import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.IOException; @@ -18,11 +21,8 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; /** @@ -34,21 +34,68 @@ public class BranchIndexGenerationBuildService { private static final Logger log = LoggerFactory.getLogger( BranchIndexGenerationBuildService.class); - private static final long HEARTBEAT_INTERVAL_SECONDS = 15; - private final BranchArchiveService archiveService; private final RagPipelineClient pipelineClient; private final RagBranchIndexRegistryService registryService; - private final ScheduledExecutorService heartbeatExecutor; + private final RagIndexOperationHeartbeatService heartbeatService; + private final AnalysisLockService analysisLockService; + private final int ragLockLeaseMinutes; + + /** + * The durable coordinates needed after build admission commits. Keeping + * this boundary scalar prevents callers from carrying detached registry + * entities (and their lazy associations) into archive or RAG I/O. + */ + public record PreparedBuild( + long operationId, + String collectionTarget, + boolean alreadySucceeded, + String manifestDigest, + String analysisLockKey, + String sourceCollectionTarget) { + + public PreparedBuild( + long operationId, + String collectionTarget, + boolean alreadySucceeded, + String manifestDigest, + String analysisLockKey) { + this(operationId, collectionTarget, alreadySucceeded, manifestDigest, + analysisLockKey, null); + } + + public PreparedBuild { + if (operationId <= 0) { + throw new IllegalArgumentException("operationId must be positive"); + } + if (collectionTarget == null || collectionTarget.isBlank()) { + throw new IllegalArgumentException("collectionTarget is required"); + } + } + } + + public BranchIndexGenerationBuildService( + BranchArchiveService archiveService, + RagPipelineClient pipelineClient, + RagBranchIndexRegistryService registryService, + RagIndexOperationHeartbeatService heartbeatService) { + this(archiveService, pipelineClient, registryService, heartbeatService, null, 360); + } + @Autowired public BranchIndexGenerationBuildService( BranchArchiveService archiveService, RagPipelineClient pipelineClient, - RagBranchIndexRegistryService registryService) { + RagBranchIndexRegistryService registryService, + RagIndexOperationHeartbeatService heartbeatService, + AnalysisLockService analysisLockService, + @Value("${analysis.lock.rag.timeout.minutes:360}") int ragLockLeaseMinutes) { this.archiveService = archiveService; this.pipelineClient = pipelineClient; this.registryService = registryService; - this.heartbeatExecutor = Executors.newSingleThreadScheduledExecutor(new HeartbeatThreadFactory()); + this.heartbeatService = heartbeatService; + this.analysisLockService = analysisLockService; + this.ragLockLeaseMinutes = Math.max(1, ragLockLeaseMinutes); } public Map build( @@ -77,8 +124,26 @@ public Map build( List excludePatterns, Long jobId, Consumer> progressEvents) throws IOException { + return build(project, connection, vcsWorkspace, repoSlug, branch, revision, + kind, includePatterns, excludePatterns, jobId, null, progressEvents); + } + + public Map build( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId, + String analysisLockKey, + Consumer> progressEvents) throws IOException { return buildInternal(project, connection, vcsWorkspace, repoSlug, branch, - revision, kind, includePatterns, excludePatterns, jobId, progressEvents, false); + revision, kind, includePatterns, excludePatterns, jobId, + analysisLockKey, progressEvents, false); } /** @@ -98,8 +163,26 @@ public Map rebuild( List excludePatterns, Long jobId, Consumer> progressEvents) throws IOException { + return rebuild(project, connection, vcsWorkspace, repoSlug, branch, revision, + kind, includePatterns, excludePatterns, jobId, null, progressEvents); + } + + public Map rebuild( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId, + String analysisLockKey, + Consumer> progressEvents) throws IOException { return buildInternal(project, connection, vcsWorkspace, repoSlug, branch, - revision, kind, includePatterns, excludePatterns, jobId, progressEvents, true); + revision, kind, includePatterns, excludePatterns, jobId, + analysisLockKey, progressEvents, true); } public Map build( @@ -114,7 +197,7 @@ public Map build( List excludePatterns, Long jobId) throws IOException { return buildInternal(project, connection, vcsWorkspace, repoSlug, branch, - revision, kind, includePatterns, excludePatterns, jobId, null, false); + revision, kind, includePatterns, excludePatterns, jobId, null, null, false); } private Map buildInternal( @@ -128,28 +211,62 @@ private Map buildInternal( List includePatterns, List excludePatterns, Long jobId, + String analysisLockKey, Consumer> progressEvents, boolean forceRebuild) throws IOException { var registration = registryService.registerBuild( project, branch, kind, null, revision, - forceRebuild ? "operator-refresh:" + requireJobId(jobId) : null); - if (registration.existingOperation() - && registration.operation().getStatus() - == RagIndexOperationStatus.SUCCEEDED) { + forceRebuild + ? "full-snapshot:job:" + requireJobId(jobId) + ":" + UUID.randomUUID() + : null); + PreparedBuild prepared = prepare(registration, analysisLockKey); + if (prepared.alreadySucceeded()) { + return Map.of( + "status", "reused", + "collection_target", prepared.collectionTarget(), + "generation_manifest_sha256", prepared.manifestDigest()); + } + + registryService.startBuild( + prepared.operationId(), jobId, analysisLockKey); + return execute(project, connection, vcsWorkspace, repoSlug, branch, revision, + kind, includePatterns, excludePatterns, prepared, progressEvents); + } + + /** + * Executes an already-admitted build. The operation must have been linked + * to its durable job and transitioned to RUNNING in the admission + * transaction before this method is called. + */ + public Map execute( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + PreparedBuild prepared, + Consumer> progressEvents) throws IOException { + if (prepared == null) { + throw new IllegalArgumentException("prepared build is required"); + } + if (prepared.alreadySucceeded()) { return Map.of( "status", "reused", - "collection_target", registration.generation().getCollectionName(), - "generation_manifest_sha256", registration.generation().getManifestDigest()); + "collection_target", prepared.collectionTarget(), + "generation_manifest_sha256", prepared.manifestDigest()); } - registryService.startBuild(registration.operation().getId(), jobId); - ScheduledFuture heartbeat = heartbeatExecutor.scheduleAtFixedRate( - () -> heartbeat(registration.operation().getId()), - HEARTBEAT_INTERVAL_SECONDS, - HEARTBEAT_INTERVAL_SECONDS, - TimeUnit.SECONDS); + RagIndexOperationHeartbeatService.HeartbeatScope heartbeat = null; + AnalysisLockService.LockLease lockLease = null; Path snapshot = null; + AtomicBoolean snapshotOwnershipTransferred = new AtomicBoolean(false); try { + lockLease = startAnalysisLockLease(prepared); + heartbeat = heartbeatService.start(prepared.operationId()); snapshot = Files.createTempDirectory("codecrow-rag-branch-generation-"); archiveService.downloadAndExtractSnapshotToDirectory( connection, @@ -161,34 +278,56 @@ private Map buildInternal( boolean publishBranchAlias = kind == RagBranchIndexKind.PRIMARY || kind == RagBranchIndexKind.DURABLE; boolean publishLegacyProjectAlias = kind == RagBranchIndexKind.PRIMARY; - Map result = progressEvents == null - ? pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, registration.generation().getCollectionName(), - false, false) - : pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, registration.generation().getCollectionName(), - false, false, - progressEvents); + Map result; + if (progressEvents == null) { + result = prepared.sourceCollectionTarget() == null + ? pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, prepared.collectionTarget(), + false, false) + : pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, prepared.collectionTarget(), + false, false, prepared.sourceCollectionTarget()); + } else { + result = prepared.sourceCollectionTarget() == null + ? pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, prepared.collectionTarget(), + false, false, true, + () -> snapshotOwnershipTransferred.set(true), + progressEvents) + : pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, prepared.collectionTarget(), + false, false, true, prepared.sourceCollectionTarget(), + () -> snapshotOwnershipTransferred.set(true), + progressEvents); + } Object manifest = result.get("generation_manifest_sha256"); if (!(manifest instanceof String digest) || digest.isBlank()) { throw new IOException("RAG full branch generation has no manifest digest"); } + if (lockLease != null && !lockLease.confirmOwnership()) { + throw new IOException( + "RAG indexing lock ownership was lost before generation publication"); + } var published = registryService.publish( - registration.operation().getId(), + prepared.operationId(), digest, number(result.get("document_count")), number(result.get("chunk_count"))); publishReadableAliasesIfActive( - project, branch, revision, registration.generation().getCollectionName(), + project, branch, revision, prepared.collectionTarget(), published, publishBranchAlias, publishLegacyProjectAlias); return result; } catch (Throwable failure) { registryService.fail( - registration.operation().getId(), + prepared.operationId(), failure.getMessage() != null ? failure.getMessage() : failure.getClass().getSimpleName()); @@ -200,13 +339,59 @@ private Map buildInternal( } throw new IOException("Failed to build exact branch generation", failure); } finally { - heartbeat.cancel(false); - if (snapshot != null) { + if (lockLease != null) { + lockLease.close(); + } + if (heartbeat != null) { + heartbeat.close(); + } + if (snapshot != null && !snapshotOwnershipTransferred.get()) { deleteTree(snapshot); } } } + /** Maps a managed registration to the scalar post-transaction boundary. */ + public static PreparedBuild prepare( + RagBranchIndexRegistryService.BuildRegistration registration) { + return prepare(registration, null); + } + + public static PreparedBuild prepare( + RagBranchIndexRegistryService.BuildRegistration registration, + String analysisLockKey) { + if (registration == null || registration.operation() == null + || registration.generation() == null) { + throw new IllegalArgumentException("A complete build registration is required"); + } + boolean succeeded = registration.existingOperation() + && registration.operation().getStatus() == RagIndexOperationStatus.SUCCEEDED; + return new PreparedBuild( + registration.operation().getId(), + registration.generation().getCollectionName(), + succeeded, + registration.generation().getManifestDigest(), + analysisLockKey, + registration.sourceCollectionTarget()); + } + + private AnalysisLockService.LockLease startAnalysisLockLease(PreparedBuild prepared) + throws IOException { + if (prepared.analysisLockKey() == null || prepared.analysisLockKey().isBlank()) { + return null; + } + if (analysisLockService == null) { + throw new IOException("Analysis-lock lease service is unavailable for exact RAG build"); + } + AnalysisLockService.LockLease lease = analysisLockService.maintainLockLease( + prepared.analysisLockKey(), ragLockLeaseMinutes); + if (lease.isOwnershipLost()) { + lease.close(); + throw new IOException("RAG indexing lock ownership was lost before snapshot build"); + } + return lease; + } + private void publishReadableAliasesIfActive( Project project, String branch, @@ -224,11 +409,13 @@ private void publishReadableAliasesIfActive( pipelineClient.publishGenerationAliases( project.getWorkspace().getName(), project.getNamespace(), branch, revision, collectionTarget, + published.getManifestDigest(), true, publishLegacyProjectAlias); - } catch (IOException aliasFailure) { + } catch (IOException | RuntimeException aliasFailure) { // Readable aliases are operator convenience. Exact retrieval uses // the registry target, and reconciliation repairs this alias later. - log.warn("Could not publish readable aliases for RAG generation {}: {}", + log.info("Readable aliases were not published for active RAG generation {}; " + + "the reconciliation scheduler will retry: {}", published.getId(), aliasFailure.getMessage()); } } @@ -244,24 +431,6 @@ private static String requireJobId(Long jobId) { return jobId.toString(); } - private void heartbeat(long operationId) { - try { - registryService.heartbeatBuild(operationId); - } catch (Exception ignored) { - // A later heartbeat may still succeed. If the producer actually - // stops, recovery turns the durable operation into a failure. - } - } - - private static final class HeartbeatThreadFactory implements ThreadFactory { - @Override - public Thread newThread(Runnable runnable) { - Thread thread = new Thread(runnable, "rag-generation-heartbeat"); - thread.setDaemon(true); - return thread; - } - } - private static void deleteTree(Path root) { try { if (!Files.exists(root)) { diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java index d5813aed..f6665d54 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java @@ -13,6 +13,8 @@ import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -39,9 +41,13 @@ */ @Service public class BranchIndexMaintenanceService { + private static final Logger log = LoggerFactory.getLogger( + BranchIndexMaintenanceService.class); + private final RagOperationsService ragOperationsService; private final VcsClientProvider vcsClientProvider; private final BranchIndexGenerationBuildService generationBuildService; + private final BranchIndexBuildAdmissionService buildAdmissionService; private final RagIndexTrackingService trackingService; private final AnalysisLockService lockService; private final AnalysisJobService jobService; @@ -52,6 +58,7 @@ public BranchIndexMaintenanceService( RagOperationsService ragOperationsService, VcsClientProvider vcsClientProvider, BranchIndexGenerationBuildService generationBuildService, + BranchIndexBuildAdmissionService buildAdmissionService, RagIndexTrackingService trackingService, AnalysisLockService lockService, AnalysisJobService jobService, @@ -60,6 +67,7 @@ public BranchIndexMaintenanceService( this.ragOperationsService = ragOperationsService; this.vcsClientProvider = vcsClientProvider; this.generationBuildService = generationBuildService; + this.buildAdmissionService = buildAdmissionService; this.trackingService = trackingService; this.lockService = lockService; this.jobService = jobService; @@ -86,7 +94,7 @@ public Map rebuild(Project project, String requestedBranch, bool String message = failure.getMessage() != null ? failure.getMessage() : failure.getClass().getSimpleName(); failures.put(branch, message); - events.accept(Map.of("type", "progress", "stage", "branch_failed", + emitEvent(events, Map.of("type", "progress", "stage", "branch_failed", "branch", branch, "message", "RAG snapshot failed for branch '" + branch + "': " + message)); } @@ -101,7 +109,7 @@ public Map rebuild(Project project, String requestedBranch, bool for (BranchBuildPlan plan : plans.subList(from, to)) { String branch = plan.branch(); wave.put(branch, CompletableFuture.runAsync(() -> { - events.accept(Map.of("type", "progress", "stage", "branch", + emitEvent(events, Map.of("type", "progress", "stage", "branch", "branch", branch, "message", "Building exact RAG snapshot for branch '" + branch + "'")); rebuildOne(project, plan, events); @@ -116,7 +124,7 @@ public Map rebuild(Project project, String requestedBranch, bool String message = cause.getMessage() != null ? cause.getMessage() : cause.getClass().getSimpleName(); failures.put(build.getKey(), message); - events.accept(Map.of("type", "progress", "stage", "branch_failed", + emitEvent(events, Map.of("type", "progress", "stage", "branch_failed", "branch", build.getKey(), "message", "RAG snapshot failed for branch '" + build.getKey() + "': " + message)); @@ -172,36 +180,45 @@ private void rebuildOne(Project project, BranchBuildPlan plan, Consumer result = generationBuildService.rebuild( + if (config.includePatterns() == null || config.excludePatterns() == null) { + throw new IllegalStateException("RAG include/exclude patterns are unavailable"); + } + admittedBuild = buildAdmissionService.admit( + project, + branch, + revision, + kind, + JobTriggerSource.UI, + lock.get(), + BranchIndexBuildAdmissionService.BuildOrigin.OPERATOR); + job = admittedBuild.job(); + Job admittedJob = job; + jobService.logToJob( + admittedJob, + JobLogLevel.INFO, + "branch_snapshot", + "Building exact RAG snapshot for branch: " + branch, + Map.of("branch", branch, "commit", revision)); + executionStarted = true; + Map result = generationBuildService.execute( project, connection, workspace, repository, branch, revision, - primary ? RagBranchIndexKind.PRIMARY : RagBranchIndexKind.DURABLE, + kind, config.includePatterns(), config.excludePatterns(), - job.getId(), event -> { + admittedBuild.preparedBuild(), event -> { Map forwarded = new LinkedHashMap<>(event); forwarded.put("type", "progress"); forwarded.put("branch", branch); @@ -209,43 +226,88 @@ private void rebuildOne(Project project, BranchBuildPlan plan, Consumer> events, + Map event) { + if (events == null) { + return; + } + try { + events.accept(event); + } catch (RuntimeException observerFailure) { + log.debug("RAG maintenance observer rejected event stage={}: {}", + event.get("stage"), observerFailure.getMessage()); + } + } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseService.java new file mode 100644 index 00000000..182d02c1 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseService.java @@ -0,0 +1,207 @@ +package org.rostilos.codecrow.ragengine.branch; + +import jakarta.annotation.PreDestroy; +import org.rostilos.codecrow.core.service.JobService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Maintains a database-backed producer lease for legacy shared-collection RAG + * updates. Exact-generation jobs use {@code RagIndexOperation} instead. + */ +@Service +public class LegacyRagJobLeaseService { + private static final Logger log = LoggerFactory.getLogger( + LegacyRagJobLeaseService.class); + + private final JobService jobService; + private final ScheduledExecutorService executor; + private final long leaseSeconds; + private final long heartbeatIntervalSeconds; + + @Autowired + public LegacyRagJobLeaseService( + JobService jobService, + @Value("${codecrow.rag.legacy-job.lease-seconds:120}") long leaseSeconds, + @Value("${codecrow.rag.legacy-job.heartbeat-interval-seconds:15}") + long heartbeatIntervalSeconds, + @Value("${codecrow.rag.legacy-job.heartbeat-threads:4}") + int heartbeatThreads) { + this( + jobService, + Executors.newScheduledThreadPool( + Math.max(2, heartbeatThreads), + new HeartbeatThreadFactory()), + leaseSeconds, + heartbeatIntervalSeconds); + } + + LegacyRagJobLeaseService( + JobService jobService, + ScheduledExecutorService executor, + long leaseSeconds, + long heartbeatIntervalSeconds) { + this.jobService = jobService; + this.executor = executor; + this.leaseSeconds = Math.max(30, leaseSeconds); + this.heartbeatIntervalSeconds = Math.max( + 1, + Math.min(heartbeatIntervalSeconds, this.leaseSeconds / 3)); + } + + /** + * Start supervising a RUNNING legacy RAG job. The first renewal is + * synchronous, so callers can refuse remote mutation without proven + * ownership. + */ + public JobLease start(long jobId) { + ActiveJobLease lease = new ActiveJobLease(jobId); + lease.renew(); + lease.schedule(); + return lease; + } + + public interface JobLease extends AutoCloseable { + boolean isOwnershipLost(); + + /** Earliest activity timestamp that is still owned by this lease. */ + OffsetDateTime validAfter(); + + /** Atomically prove ownership immediately before durable publication. */ + boolean confirmOwnership(); + + @Override + void close(); + } + + private final class ActiveJobLease implements JobLease { + private final long jobId; + private final AtomicBoolean ownershipLost = new AtomicBoolean(false); + private final AtomicBoolean closed = new AtomicBoolean(false); + private final AtomicBoolean transientFailureReported = new AtomicBoolean(false); + private final AtomicReference knownExpiresAt = + new AtomicReference<>(); + private volatile ScheduledFuture heartbeat; + + private ActiveJobLease(long jobId) { + this.jobId = jobId; + } + + private void schedule() { + if (ownershipLost.get()) { + return; + } + heartbeat = executor.scheduleWithFixedDelay( + this::renew, + heartbeatIntervalSeconds, + heartbeatIntervalSeconds, + TimeUnit.SECONDS); + } + + private boolean renew() { + if (closed.get() || ownershipLost.get()) { + return false; + } + OffsetDateTime renewalStartedAt = OffsetDateTime.now(); + try { + boolean renewed = jobService.renewLegacyRagJobLease( + jobId, + renewalStartedAt.minusSeconds(leaseSeconds), + renewalStartedAt); + if (!renewed) { + ownershipLost.set(true); + log.info("Legacy RAG job lease ownership was lost: job={}", jobId); + return false; + } + knownExpiresAt.set( + renewalStartedAt.plusSeconds(leaseSeconds).minusSeconds(1)); + if (transientFailureReported.compareAndSet(true, false)) { + log.info("Legacy RAG job heartbeat recovered: job={}", jobId); + } + return true; + } catch (RuntimeException renewalFailure) { + OffsetDateTime knownExpiry = knownExpiresAt.get(); + boolean previousLeaseStillActive = knownExpiry != null + && OffsetDateTime.now().isBefore(knownExpiry); + if (!previousLeaseStillActive) { + ownershipLost.set(true); + log.info( + "Legacy RAG job ownership could not be confirmed before its known expiry: " + + "job={}, detail={}", + jobId, + renewalFailure.getMessage()); + return false; + } + if (transientFailureReported.compareAndSet(false, true)) { + log.warn( + "Legacy RAG job heartbeat failed within the active lease; retrying: " + + "job={}, detail={}", + jobId, + renewalFailure.getMessage()); + } else { + log.debug( + "Legacy RAG job heartbeat remains degraded within the active lease: job={}", + jobId); + } + return true; + } + } + + @Override + public boolean isOwnershipLost() { + return ownershipLost.get(); + } + + @Override + public OffsetDateTime validAfter() { + return OffsetDateTime.now().minusSeconds(leaseSeconds); + } + + @Override + public boolean confirmOwnership() { + return renew(); + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + ScheduledFuture scheduled = heartbeat; + if (scheduled != null) { + scheduled.cancel(false); + } + } + } + + @PreDestroy + void close() { + executor.shutdownNow(); + } + + private static final class HeartbeatThreadFactory implements ThreadFactory { + private final AtomicInteger sequence = new AtomicInteger(); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread( + runnable, + "legacy-rag-job-heartbeat-" + sequence.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryService.java new file mode 100644 index 00000000..19dc22d0 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryService.java @@ -0,0 +1,148 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.service.JobService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Terminalizes legacy incremental RAG jobs whose database lease expired. The + * RAG lock is deliberately not released here: the legacy remote mutation has + * no fencing token, so its maintained lease must expire before a retry starts. + */ +@Service +public class LegacyRagJobRecoveryService { + private static final Logger log = LoggerFactory.getLogger( + LegacyRagJobRecoveryService.class); + + private final JobService jobService; + private final RagIndexTrackingService trackingService; + private final long staleAfterSeconds; + private final int batchSize; + private final AtomicBoolean recoveryDegraded = new AtomicBoolean(false); + + public LegacyRagJobRecoveryService( + JobService jobService, + RagIndexTrackingService trackingService, + @Value("${codecrow.rag.legacy-job.lease-seconds:120}") + long staleAfterSeconds, + @Value("${codecrow.rag.legacy-job.recovery-batch-size:100}") + int batchSize) { + this.jobService = jobService; + this.trackingService = trackingService; + this.staleAfterSeconds = Math.max(30, staleAfterSeconds); + this.batchSize = Math.max(1, batchSize); + } + + @Scheduled( + fixedDelayString = "${codecrow.rag.legacy-job.recovery-interval-ms:30000}", + initialDelayString = "${codecrow.rag.legacy-job.recovery-initial-delay-ms:30000}") + public void failAbandonedJobs() { + OffsetDateTime cutoff = OffsetDateTime.now().minusSeconds(staleAfterSeconds); + String diagnostic = "Legacy RAG update producer stopped heartbeating for " + + staleAfterSeconds + + " seconds; the last completed checkpoint was preserved"; + boolean passDegraded = false; + + try { + for (var coordinates : jobService.findAbandonedLegacyRagJobs( + cutoff, batchSize)) { + try { + if (!jobService.failAbandonedLegacyRagJob( + coordinates.getJobId(), cutoff, diagnostic)) { + continue; + } + log.warn( + "Failed abandoned legacy RAG job {} for project={}, branch={}, commit={}", + coordinates.getJobId(), + coordinates.getProjectId(), + coordinates.getBranchName(), + coordinates.getCommitHash()); + recordDurableFailure(coordinates.getJobId(), diagnostic); + if (!repairProjectStatus(coordinates, diagnostic)) { + passDegraded = true; + } + } catch (Exception failure) { + passDegraded = true; + reportDegraded( + "Could not terminalize abandoned legacy RAG job " + + coordinates.getJobId(), + failure); + } + } + + // If a process stopped after the job CAS but before repairing its + // project-level status, retry that projection independently. + for (var coordinates : jobService.findFailedLegacyRagJobsWithActiveStatus( + batchSize)) { + String priorDiagnostic = coordinates.getErrorMessage(); + if (!repairProjectStatus( + coordinates, + priorDiagnostic != null && !priorDiagnostic.isBlank() + ? priorDiagnostic + : diagnostic)) { + passDegraded = true; + } + } + } catch (Exception selectionFailure) { + passDegraded = true; + reportDegraded( + "Could not scan legacy RAG job recovery state", + selectionFailure); + } + + if (!passDegraded && recoveryDegraded.compareAndSet(true, false)) { + log.info("Legacy RAG job recovery scan recovered"); + } + } + + private void recordDurableFailure(long jobId, String diagnostic) { + try { + jobService.findById(jobId).ifPresent(job -> + jobService.recordExternallyFailedJob( + job, "rag_recovery", "Job failed: " + diagnostic)); + } catch (Exception failure) { + // The guarded CAS already made the job terminal. Projection repair + // is independent of an optional UI log/notification. + log.debug("Could not append optional recovery diagnostics to legacy RAG job {}: {}", + jobId, failure.getMessage()); + } + } + + private boolean repairProjectStatus( + JobRepository.LegacyRagJobRecoveryCoordinates coordinates, + String diagnostic) { + try { + // This is an incremental job, so retain the last usable checkpoint + // even if an older producer happened to label the live state INDEXING. + trackingService.recoverAbandonedIncrementalUpdate( + coordinates.getProjectId(), coordinates.getJobId(), diagnostic); + return true; + } catch (Exception failure) { + reportDegraded( + "Could not repair RAG status after legacy producer abandonment: project=" + + coordinates.getProjectId() + + ", job=" + coordinates.getJobId(), + failure); + return false; + } + } + + private void reportDegraded(String operation, Exception failure) { + String detail = failure.getMessage() != null + ? failure.getMessage() + : failure.getClass().getSimpleName(); + if (recoveryDegraded.compareAndSet(false, true)) { + log.warn("Legacy RAG job recovery degraded: {}: {}", operation, detail); + } else { + log.debug("Legacy RAG job recovery remains degraded: {}: {}", operation, detail); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionService.java new file mode 100644 index 00000000..ce5028fa --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionService.java @@ -0,0 +1,115 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.persistence.repository.analysis.RagIndexStatusRepository; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.interceptor.TransactionAspectSupport; + +import java.time.OffsetDateTime; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** Commits one legacy RAG job and all of its Java checkpoints atomically. */ +@Service +public class LegacyRagUpdateCompletionService { + private final JobRepository jobRepository; + private final RagIndexTrackingService trackingService; + private final RagIndexStatusRepository indexStatusRepository; + private final RagBranchIndexRepository branchIndexRepository; + + public LegacyRagUpdateCompletionService( + JobRepository jobRepository, + RagIndexTrackingService trackingService, + RagIndexStatusRepository indexStatusRepository, + RagBranchIndexRepository branchIndexRepository) { + this.jobRepository = jobRepository; + this.trackingService = trackingService; + this.indexStatusRepository = indexStatusRepository; + this.branchIndexRepository = branchIndexRepository; + } + + /** + * @return {@code false} when recovery or lease expiry won the job row first + */ + @Transactional + public boolean complete( + Project project, + String branchName, + String commitHash, + long jobId, + OffsetDateTime validAfter, + boolean tracksProjectStatus, + int addedFiles, + int deletedFiles, + Integer chunkCount, + Set deletedPaths) { + OffsetDateTime completedAt = OffsetDateTime.now(); + if (jobRepository.completeOwnedLegacyRagJob( + jobId, validAfter, completedAt) != 1) { + return false; + } + + if (tracksProjectStatus) { + var status = indexStatusRepository + .findByProjectIdForUpdate(project.getId()) + .orElseThrow(() -> new LegacyRagCompletionConflictException( + "RAG index status disappeared before legacy completion")); + if (status.getStatus() != RagIndexingStatus.UPDATING + || !Objects.equals(status.getActiveJobId(), jobId)) { + throw new LegacyRagCompletionConflictException( + "Legacy RAG job no longer owns the project index status"); + } + trackingService.markUpdatingCompleted( + project, + branchName, + commitHash, + addedFiles, + deletedFiles, + chunkCount, + jobId); + } + try { + updateBranchCheckpoint(project, branchName, commitHash, deletedPaths); + } catch (RuntimeException checkpointFailure) { + TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); + throw checkpointFailure; + } + return true; + } + + private void updateBranchCheckpoint( + Project project, + String branchName, + String commitHash, + Set deletedPaths) { + RagBranchIndex branchIndex = branchIndexRepository + .findByProjectIdAndBranchNameForUpdate( + project.getId(), branchName) + .orElseGet(() -> new RagBranchIndex(project, branchName)); + branchIndex.setCommitHash(commitHash); + branchIndex.setUpdatedAt(OffsetDateTime.now()); + if (deletedPaths != null && !deletedPaths.isEmpty()) { + Set accumulated = branchIndex.getDeletedFiles() != null + ? new HashSet<>(branchIndex.getDeletedFiles()) + : new HashSet<>(); + accumulated.addAll(deletedPaths); + branchIndex.setDeletedFiles(accumulated); + } + branchIndexRepository.save(branchIndex); + } + + /** Signals a fenced projection conflict; the producer must not fail newer state. */ + public static final class LegacyRagCompletionConflictException + extends RuntimeException { + public LegacyRagCompletionConflictException(String message) { + super(message); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java index b58b5ea5..3eda8437 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java @@ -1,6 +1,10 @@ package org.rostilos.codecrow.ragengine.branch; -import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; @@ -8,7 +12,6 @@ import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; /** * Restores human-readable Qdrant aliases for already active branch generations. @@ -21,11 +24,13 @@ */ @Service public class RagBranchOperatorAliasReconciliationService { + private static final int MAX_GENERATION_REFRESHES = 3; private static final Logger log = LoggerFactory.getLogger( RagBranchOperatorAliasReconciliationService.class); private final RagBranchIndexRepository branchIndexRepository; private final RagPipelineClient pipelineClient; + private ReconciliationState reconciliationState = ReconciliationState.HEALTHY; public RagBranchOperatorAliasReconciliationService( RagBranchIndexRepository branchIndexRepository, @@ -37,38 +42,141 @@ public RagBranchOperatorAliasReconciliationService( @Scheduled( fixedDelayString = "${codecrow.rag.operator-alias.reconcile-interval-ms:300000}", initialDelayString = "${codecrow.rag.operator-alias.reconcile-initial-delay-ms:15000}") - @Transactional(readOnly = true) - public void reconcileActiveGenerationAliases() { - for (RagBranchIndex branchIndex : branchIndexRepository.findAll()) { - if (!isPublishedOperatorBranch(branchIndex)) { - continue; - } - var generation = branchIndex.getActiveGeneration(); - var project = branchIndex.getProject(); + public synchronized void reconcileActiveGenerationAliases() { + List candidates; + try { + candidates = branchIndexRepository.findOperatorAliasCandidates(); + } catch (RuntimeException repositoryFailure) { + recordDegradedRun("Could not read active RAG aliases for reconciliation; " + + "retrying next run: " + repositoryFailure.getMessage()); + return; + } + + int candidateRejections = 0; + for (var candidate : candidates) { try { - pipelineClient.publishGenerationAliases( - project.getWorkspace().getName(), - project.getNamespace(), - branchIndex.getBranchName(), - generation.getRevision(), - generation.getCollectionName(), - true, - branchIndex.getIndexKind() == RagBranchIndexKind.PRIMARY); - } catch (Exception failure) { - log.warn( - "Could not reconcile readable RAG alias for project={} branch={}: {}", - project.getId(), - branchIndex.getBranchName(), - failure.getMessage()); + publishCurrentGeneration(candidate); + } catch (RagPipelineClient.RagApiException apiFailure) { + if (apiFailure.isServiceFailure()) { + recordDegradedRun("RAG alias reconciliation stopped after a service failure; " + + "remaining candidates will retry next run: " + apiFailure.getMessage()); + return; + } + candidateRejections++; + log.debug( + "RAG alias candidate was rejected for project={} branch={}; continuing: {}", + candidate.getProjectId(), + candidate.getBranchName(), + apiFailure.getMessage()); + } catch (IOException transportFailure) { + // One unavailable RAG service would make every remaining call + // fail and generate the same warning. Stop this bounded run; + // the scheduler's fixed delay is the retry backoff. + recordDegradedRun("RAG alias reconciliation stopped after a transport failure; " + + "remaining candidates will retry next run: " + transportFailure.getMessage()); + return; + } catch (CandidateRegistryReadException repositoryFailure) { + recordDegradedRun("RAG alias reconciliation stopped after a registry read failure; " + + "remaining candidates will retry next run: " + repositoryFailure.getMessage()); + return; + } catch (RuntimeException candidateFailure) { + candidateRejections++; + log.debug( + "RAG alias candidate failed for project={} branch={}; continuing: {}", + candidate.getProjectId(), + candidate.getBranchName(), + candidateFailure.getMessage()); + } + } + if (candidateRejections > 0) { + recordDegradedRun("RAG alias reconciliation completed with " + candidateRejections + + " rejected candidate(s); they will retry next run"); + } else { + recordHealthyRun(); + } + } + + private void publishCurrentGeneration( + RagBranchIndexRepository.OperatorAliasCandidate scheduledCandidate) throws IOException { + RagBranchIndexRepository.OperatorAliasCandidate current = refreshCandidate(scheduledCandidate) + .orElse(null); + if (current == null) { + return; + } + + for (int refresh = 0; refresh < MAX_GENERATION_REFRESHES; refresh++) { + publish(current); + Optional afterPublication = + refreshCandidate(current); + if (afterPublication.isEmpty()) { + return; } + if (sameGeneration(current, afterPublication.get())) { + return; + } + log.info("RAG active generation changed during alias reconciliation for " + + "project={} branch={}; publishing the current generation instead", + current.getProjectId(), current.getBranchName()); + current = afterPublication.get(); + } + + throw new IllegalStateException( + "RAG active generation kept changing during alias reconciliation for project=" + + current.getProjectId() + " branch=" + current.getBranchName()); + } + + private Optional refreshCandidate( + RagBranchIndexRepository.OperatorAliasCandidate candidate) { + try { + return branchIndexRepository.findOperatorAliasCandidateById(candidate.getBranchIndexId()); + } catch (RuntimeException repositoryFailure) { + throw new CandidateRegistryReadException(repositoryFailure); + } + } + + private void publish(RagBranchIndexRepository.OperatorAliasCandidate candidate) throws IOException { + pipelineClient.publishGenerationAliases( + candidate.getWorkspaceName(), + candidate.getProjectNamespace(), + candidate.getBranchName(), + candidate.getRevision(), + candidate.getCollectionName(), + candidate.getManifestDigest(), + true, + candidate.getIndexKind() == RagBranchIndexKind.PRIMARY); + } + + private static boolean sameGeneration( + RagBranchIndexRepository.OperatorAliasCandidate first, + RagBranchIndexRepository.OperatorAliasCandidate second) { + if (first.getGenerationId() != null || second.getGenerationId() != null) { + return Objects.equals(first.getGenerationId(), second.getGenerationId()); } + return Objects.equals(first.getCollectionName(), second.getCollectionName()) + && Objects.equals(first.getManifestDigest(), second.getManifestDigest()); } - private static boolean isPublishedOperatorBranch(RagBranchIndex branchIndex) { - if (branchIndex == null || branchIndex.getActiveGeneration() == null) { - return false; + private void recordDegradedRun(String message) { + if (reconciliationState == ReconciliationState.HEALTHY) { + reconciliationState = ReconciliationState.DEGRADED; + log.warn(message); + } else { + log.info(message); + } + } + + private void recordHealthyRun() { + if (reconciliationState == ReconciliationState.DEGRADED) { + reconciliationState = ReconciliationState.HEALTHY; + log.info("RAG alias reconciliation recovered"); + } + } + + private enum ReconciliationState { HEALTHY, DEGRADED } + + private static final class CandidateRegistryReadException extends RuntimeException { + private CandidateRegistryReadException(RuntimeException cause) { + super(cause.getMessage(), cause); } - return branchIndex.getIndexKind() == RagBranchIndexKind.PRIMARY - || branchIndex.getIndexKind() == RagBranchIndexKind.DURABLE; } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java new file mode 100644 index 00000000..220e8552 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatService.java @@ -0,0 +1,84 @@ +package org.rostilos.codecrow.ragengine.branch; + +import jakarta.annotation.PreDestroy; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** Keeps a live exact-generation operation owned while remote work is running. */ +@Service +public class RagIndexOperationHeartbeatService { + private static final long HEARTBEAT_INTERVAL_SECONDS = 15; + private static final int HEARTBEAT_THREADS = 4; + + private final RagBranchIndexRegistryService registryService; + private final ScheduledExecutorService executor; + private final long heartbeatIntervalSeconds; + + @Autowired + public RagIndexOperationHeartbeatService( + RagBranchIndexRegistryService registryService) { + this(registryService, + Executors.newScheduledThreadPool( + HEARTBEAT_THREADS, new HeartbeatThreadFactory()), + HEARTBEAT_INTERVAL_SECONDS); + } + + RagIndexOperationHeartbeatService( + RagBranchIndexRegistryService registryService, + ScheduledExecutorService executor, + long heartbeatIntervalSeconds) { + this.registryService = registryService; + this.executor = executor; + this.heartbeatIntervalSeconds = heartbeatIntervalSeconds; + } + + public HeartbeatScope start(long operationId) { + ScheduledFuture heartbeat = executor.scheduleAtFixedRate( + () -> heartbeat(operationId), + heartbeatIntervalSeconds, + heartbeatIntervalSeconds, + TimeUnit.SECONDS); + return () -> heartbeat.cancel(false); + } + + private void heartbeat(long operationId) { + try { + registryService.heartbeatBuild(operationId); + } catch (Exception ignored) { + // A later heartbeat may still succeed. If the producer stops, + // durable operation recovery owns the terminal transition. + } + } + + @PreDestroy + void close() { + executor.shutdownNow(); + } + + @FunctionalInterface + public interface HeartbeatScope extends AutoCloseable { + @Override + void close(); + } + + private static final class HeartbeatThreadFactory implements ThreadFactory { + private final AtomicInteger sequence = new AtomicInteger(); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread( + runnable, + "rag-generation-heartbeat-" + sequence.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java index 7c194528..4087421e 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java @@ -1,6 +1,14 @@ package org.rostilos.codecrow.ragengine.branch; +import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; +import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagIndexOperationRepository; +import org.rostilos.codecrow.core.service.JobService; import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -8,6 +16,7 @@ import org.springframework.stereotype.Service; import java.time.OffsetDateTime; +import java.util.concurrent.atomic.AtomicBoolean; /** * Terminates registry operations whose producer disappeared before publishing @@ -21,13 +30,29 @@ public class RagIndexOperationRecoveryService { RagIndexOperationRecoveryService.class); private final RagBranchIndexRegistryService registryService; + private final ProjectRepository projectRepository; + private final JobService jobService; + private final RagIndexTrackingService trackingService; + private final RagOperationsService ragOperationsService; + private final AnalysisLockService lockService; private final long staleAfterMinutes; + private final AtomicBoolean recoveryDegraded = new AtomicBoolean(false); public RagIndexOperationRecoveryService( RagBranchIndexRegistryService registryService, + ProjectRepository projectRepository, + JobService jobService, + RagIndexTrackingService trackingService, + RagOperationsService ragOperationsService, + AnalysisLockService lockService, @Value("${codecrow.rag.generation.stale-after-minutes:30}") long staleAfterMinutes) { this.registryService = registryService; + this.projectRepository = projectRepository; + this.jobService = jobService; + this.trackingService = trackingService; + this.ragOperationsService = ragOperationsService; + this.lockService = lockService; this.staleAfterMinutes = Math.max(5, staleAfterMinutes); } @@ -36,17 +61,233 @@ public RagIndexOperationRecoveryService( initialDelayString = "${codecrow.rag.generation.recovery-initial-delay-ms:60000}") public void failAbandonedOperations() { OffsetDateTime cutoff = OffsetDateTime.now().minusMinutes(staleAfterMinutes); - for (var operation : registryService.findRecoverableOperations(cutoff)) { - String diagnostic = "Exact RAG generation producer stopped heartbeating for " - + staleAfterMinutes + " minutes; the previous active generation was preserved"; + boolean passDegraded = false; + try { + for (var operation : registryService.findRecoverableOperations(cutoff)) { + String diagnostic = "Exact RAG generation producer stopped heartbeating for " + + staleAfterMinutes + " minutes; the previous active generation was preserved"; + try { + if (!registryService.failIfAbandoned( + operation.getOperationId(), cutoff, diagnostic)) { + continue; + } + log.warn("Failed abandoned RAG generation operation {} for branch {}", + operation.getOperationId(), operation.getBranchName()); + } catch (Exception failure) { + passDegraded = true; + reportDegraded( + "Could not terminalize abandoned RAG generation operation " + + operation.getOperationId(), + failure); + continue; + } + + if (!recoverProjections(operation, diagnostic)) { + passDegraded = true; + } + } + + for (var operation : registryService.findFailedOperationsWithActiveProjections()) { + String diagnostic = operation.getErrorMessage() != null + && !operation.getErrorMessage().isBlank() + ? operation.getErrorMessage() + : "RAG generation failed before its durable projections were terminalized"; + if (!recoverProjections(operation, diagnostic)) { + passDegraded = true; + } + } + + for (var operation : registryService.findSucceededOperationsWithActiveProjections()) { + if (!recoverPublishedProjections(operation)) { + passDegraded = true; + } + } + } catch (Exception selectionFailure) { + passDegraded = true; + reportDegraded( + "Could not scan exact RAG operation recovery state", + selectionFailure); + } + + if (!passDegraded && recoveryDegraded.compareAndSet(true, false)) { + log.info("Exact RAG operation recovery scan recovered"); + } + } + + private boolean recoverPublishedProjections( + RagIndexOperationRepository.SucceededOperationProjection operation) { + boolean recovered = true; + Long projectId = operation.getProjectId(); + String branchName = operation.getBranchName(); + Long jobId = operation.getJobId(); + try { + var project = projectRepository.findByIdWithFullDetails(projectId) + .orElse(null); + if (Boolean.TRUE.equals(operation.getActiveGeneration()) + && project != null + && branchName.equals(ragOperationsService.getBaseBranch(project))) { + trackingService.reconcilePublishedGeneration( + project, + branchName, + operation.getToRevision(), + operation.getFileCount(), + operation.getChunkCount(), + jobId); + } + } catch (Exception failure) { + recovered = false; + reportDegraded( + "Could not reconcile published RAG project status: project=" + + projectId + ", branch=" + branchName, + failure); + } + + if (jobId != null) { try { - registryService.fail(operation.getId(), diagnostic); - log.warn("Failed abandoned RAG generation operation {} for branch {}", - operation.getId(), operation.getBranchName()); + Job job = jobService.findById(jobId).orElse(null); + if (job != null && !job.isTerminal()) { + jobService.completeJob(job); + } } catch (Exception failure) { - log.error("Could not terminalize abandoned RAG generation operation {}", - operation.getId(), failure); + recovered = false; + reportDegraded( + "Could not complete durable job " + jobId + + " for published RAG generation", + failure); + } + } + + return releasePublishedLock(operation) && recovered; + } + + private boolean releasePublishedLock( + RagIndexOperationRepository.SucceededOperationProjection operation) { + String lockKey = operation.getAnalysisLockKey(); + if (lockKey == null || lockKey.isBlank()) { + return true; + } + try { + lockService.releaseLock(lockKey); + log.info( + "Released completed RAG indexing lock during projection recovery: " + + "project={}, branch={}, commit={}", + operation.getProjectId(), operation.getBranchName(), + operation.getToRevision()); + return true; + } catch (Exception failure) { + reportDegraded( + "Could not release completed RAG indexing lock: project=" + + operation.getProjectId() + ", branch=" + + operation.getBranchName(), + failure); + return false; + } + } + + private boolean recoverProjections( + RagIndexOperationRepository.RecoveryOperationProjection operation, + String diagnostic) { + boolean jobRecovered = failDurableJob(operation.getJobId(), diagnostic); + boolean statusRecovered = terminalizePrimaryStatus(operation, diagnostic); + boolean lockRecovered = releaseAbandonedLock(operation); + return jobRecovered && statusRecovered && lockRecovered; + } + + private boolean failDurableJob(Long jobId, String diagnostic) { + if (jobId == null) { + return true; + } + try { + Job job = jobService.findById(jobId).orElse(null); + if (job != null && !job.isTerminal()) { + jobService.failJob(job, diagnostic); + } + return true; + } catch (Exception failure) { + reportDegraded( + "Could not fail durable RAG job " + jobId + + " after producer abandonment", + failure); + return false; + } + } + + private boolean terminalizePrimaryStatus( + RagIndexOperationRepository.RecoveryOperationProjection operation, + String diagnostic) { + Long projectId = operation.getProjectId(); + String branchName = operation.getBranchName(); + Long jobId = operation.getJobId(); + try { + var project = projectRepository.findByIdWithFullDetails(projectId) + .orElse(null); + if (project == null + || !branchName.equals(ragOperationsService.getBaseBranch(project))) { + return true; } + var status = trackingService.getIndexStatus(project).orElse(null); + if (status == null) { + return true; + } + if (status.getActiveJobId() == null + || !status.getActiveJobId().equals(jobId)) { + log.info( + "Preserving RAG status owned by job {} while recovering abandoned job {}: " + + "project={}, branch={}", + status.getActiveJobId(), jobId, projectId, branchName); + return true; + } + if (status.getStatus() == RagIndexingStatus.INDEXING) { + trackingService.markIndexingFailed(project, diagnostic, jobId); + } else if (status.getStatus() == RagIndexingStatus.UPDATING) { + trackingService.markIncrementalUpdateFailed( + project, diagnostic, jobId); + } + return true; + } catch (Exception failure) { + reportDegraded( + "Could not terminalize RAG project status after producer abandonment: " + + "project=" + projectId + ", branch=" + branchName, + failure); + return false; + } + } + + private boolean releaseAbandonedLock( + RagIndexOperationRepository.RecoveryOperationProjection operation) { + Long projectId = operation.getProjectId(); + String branchName = operation.getBranchName(); + String lockKey = operation.getAnalysisLockKey(); + if (lockKey == null || lockKey.isBlank()) { + log.info( + "Cannot release abandoned RAG lock without its exact owner key; " + + "leaving it to expire: project={}, branch={}, commit={}", + projectId, branchName, operation.getToRevision()); + return true; + } + try { + lockService.releaseLock(lockKey); + log.info( + "Released abandoned RAG indexing lock for project={}, branch={}, commit={}", + projectId, branchName, operation.getToRevision()); + return true; + } catch (Exception failure) { + reportDegraded( + "Could not release abandoned RAG indexing lock: project=" + + projectId + ", branch=" + branchName, + failure); + return false; + } + } + + private void reportDegraded(String operation, Exception failure) { + String detail = failure.getMessage() != null + ? failure.getMessage() + : failure.getClass().getSimpleName(); + if (recoveryDegraded.compareAndSet(false, true)) { + log.warn("Exact RAG operation recovery degraded: {}: {}", operation, detail); + } else { + log.debug("Exact RAG operation recovery remains degraded: {}: {}", operation, detail); } } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java index 2aa28b55..fe060f86 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java @@ -1,7 +1,6 @@ package org.rostilos.codecrow.ragengine.branch; -import org.rostilos.codecrow.core.model.rag.RagBranchIndex; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; @@ -9,9 +8,11 @@ import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.time.OffsetDateTime; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; /** Removes only expired PR-target generations explicitly classified transient. */ @Service @@ -22,6 +23,8 @@ public class RagTransientBranchIndexCleanupService { private final RagBranchIndexRepository branchRepository; private final RagBranchIndexGenerationRepository generationRepository; private final RagPipelineClient pipelineClient; + private final String cleanupOwner = UUID.randomUUID().toString(); + private CleanupState cleanupState = CleanupState.HEALTHY; public RagTransientBranchIndexCleanupService( RagBranchIndexRepository branchRepository, @@ -35,14 +38,21 @@ public RagTransientBranchIndexCleanupService( @Scheduled( fixedDelayString = "${codecrow.rag.transient.cleanup-interval-ms:3600000}", initialDelayString = "${codecrow.rag.transient.cleanup-initial-delay-ms:300000}") - @Transactional - public void cleanupExpired() { + public synchronized void cleanupExpired() { OffsetDateTime now = OffsetDateTime.now(); - for (RagBranchIndex index : branchRepository.findByIndexKind( - RagBranchIndexKind.TRANSIENT)) { - var project = index.getProject(); - var config = project.getConfiguration() != null - ? project.getConfiguration().ragConfig() + List candidates; + try { + candidates = branchRepository.findTransientCleanupCandidates(); + } catch (RuntimeException repositoryFailure) { + recordDegradedRun("Could not read transient RAG cleanup candidates; retrying next run: " + + repositoryFailure.getMessage()); + return; + } + + int rejectedCandidates = 0; + for (var index : candidates) { + var config = index.getProjectConfiguration() != null + ? index.getProjectConfiguration().ragConfig() : null; int retentionDays = config != null ? config.getEffectiveBranchRetentionDays() @@ -54,24 +64,178 @@ public void cleanupExpired() { continue; } + OffsetDateTime cutoff = now.minusDays(retentionDays); + int claimed; + try { + claimed = branchRepository.claimExpiredTransientForDeletion( + index.getBranchIndexId(), cutoff, now.minusMinutes(10), + cleanupOwner, now); + } catch (RuntimeException repositoryFailure) { + recordDegradedRun("Transient RAG cleanup could not claim project=" + + index.getProjectId() + " branch=" + index.getBranchName() + + "; retrying next run: " + repositoryFailure.getMessage()); + return; + } + if (claimed == 0) { + // Access or a concurrent build won the atomic registry race. + continue; + } + boolean removed = true; - for (var generation : generationRepository - .findByBranchIndexIdOrderByCreatedAtDesc(index.getId())) { + boolean physicalDeletionStarted = false; + boolean deletionOutcomeUncertain = false; + List generations; + try { + generations = generationRepository.findCleanupCandidatesByBranchIndexId( + index.getBranchIndexId()); + } catch (RuntimeException repositoryFailure) { + cancelClaim(index.getBranchIndexId(), cleanupOwner); + recordDegradedRun("Transient RAG cleanup stopped after a registry read failure; " + + "remaining candidates will retry next run: " + repositoryFailure.getMessage()); + return; + } + // A readable active generation is deleted last. If an earlier + // target-specific rejection occurs, releasing the claim is safe as + // long as no physical target was removed; after a partial cleanup + // the durable claim stays in place until the next idempotent retry. + generations = generations.stream() + .sorted(Comparator.comparing(generation -> + generation.getStatus() == RagBranchIndexGenerationStatus.ACTIVE)) + .toList(); + for (var generation : generations) { + if (generation.getStatus() == RagBranchIndexGenerationStatus.ACTIVE + && !removed) { + // Keep the currently readable target intact if an older + // target could not be reconciled. If an older target was + // already removed, the claim below remains durable and + // prevents reads of that partial generation set. + break; + } + RagPipelineClient.BranchDeletionOutcome outcome; try { - removed &= pipelineClient.deleteBranch( - project.getWorkspace().getName(), project.getNamespace(), - index.getBranchName(), generation.getCollectionName()); - } catch (Exception failure) { + if (branchRepository.heartbeatTransientDeletionClaim( + index.getBranchIndexId(), cleanupOwner, OffsetDateTime.now()) == 0) { + recordDegradedRun("Transient RAG cleanup lost its durable claim for project=" + + index.getProjectId() + " branch=" + index.getBranchName() + + "; remaining targets will retry next run"); + return; + } + outcome = pipelineClient.deleteBranchWithOutcome( + index.getWorkspaceName(), index.getProjectNamespace(), + index.getBranchName(), generation.getCollectionName(), + generation.getRevision(), generation.getManifestDigest()); + } catch (RuntimeException unexpectedFailure) { + // The request may have reached RAG before the client threw. + // Keep the claim until an idempotent retry can reconcile it. + recordDegradedRun("Transient RAG cleanup stopped at target=" + + generation.getCollectionName() + + " after an unexpected client failure; remaining generations " + + "will retry next run: " + unexpectedFailure.getMessage()); + return; + } + if (outcome.successful()) { + physicalDeletionStarted = true; + } + if (!outcome.successful()) { + if (isRagDisabled(outcome)) { + // Deployment-level disablement is intentional. Retain + // the registry and avoid an hourly degraded signal. + cancelClaim(index.getBranchIndexId(), cleanupOwner); + recordHealthyRun(); + return; + } removed = false; - log.warn("Failed to clean transient RAG generation {}: {}", - generation.getId(), failure.getMessage()); + deletionOutcomeUncertain |= outcome.failure() + == RagPipelineClient.BranchDeletionFailure.TRANSPORT; + log.debug("Transient RAG generation cleanup rejected generation={} target={}: " + + "status={} detail={}", + generation.getGenerationId(), outcome.targetLabel(), + outcome.statusCode() != null ? outcome.statusCode() : outcome.failure(), + outcome.detail()); + if (outcome.shouldStopRemainingTargets()) { + if (!physicalDeletionStarted && !deletionOutcomeUncertain) { + cancelClaim(index.getBranchIndexId(), cleanupOwner); + } + recordDegradedRun("Transient RAG cleanup stopped at target=" + + outcome.targetLabel() + " after " + outcome.failure() + + " failure; remaining generations will retry next run: " + + outcome.detail()); + return; + } } } if (removed) { - branchRepository.delete(index); + int deleted; + try { + deleted = branchRepository.deleteClaimedTransientById( + index.getBranchIndexId(), cleanupOwner); + } catch (RuntimeException repositoryFailure) { + recordDegradedRun("Transient RAG cleanup could not finalize registry deletion " + + "for project=" + index.getProjectId() + " branch=" + + index.getBranchName() + "; retrying next run: " + + repositoryFailure.getMessage()); + return; + } + if (deleted == 0) { + // A claimed row cannot be made readable or rebuilt. A zero + // delete means registry finalization failed independently; + // delete leaves the durable token in place so destroyed data + // is never exposed. + recordDegradedRun("Transient RAG cleanup deleted physical targets but could not " + + "finalize its registry claim for project=" + index.getProjectId() + + " branch=" + index.getBranchName() + "; retrying next run"); + return; + } log.info("Removed expired transient RAG branch index project={}, branch={}", - project.getId(), index.getBranchName()); + index.getProjectId(), index.getBranchName()); + } else { + if (!physicalDeletionStarted && !deletionOutcomeUncertain) { + cancelClaim(index.getBranchIndexId(), cleanupOwner); + } + rejectedCandidates++; } } + if (rejectedCandidates > 0) { + recordDegradedRun("Transient RAG cleanup completed with " + rejectedCandidates + + " rejected candidate(s); they will retry next run"); + } else { + recordHealthyRun(); + } + } + + private void cancelClaim(long branchIndexId, String claimToken) { + try { + branchRepository.cancelTransientDeletion(branchIndexId, claimToken); + } catch (RuntimeException cancellationFailure) { + // Keeping the token is fail-safe: no reader will receive a target + // whose cleanup outcome cannot be reconciled. + log.debug("Could not release transient cleanup claim branchIndex={}: {}", + branchIndexId, cancellationFailure.getMessage()); + } } + + private static boolean isRagDisabled( + RagPipelineClient.BranchDeletionOutcome outcome) { + return outcome.statusCode() == null + && outcome.failure() == RagPipelineClient.BranchDeletionFailure.TARGET + && "RAG disabled".equals(outcome.detail()); + } + + private void recordDegradedRun(String message) { + if (cleanupState == CleanupState.HEALTHY) { + cleanupState = CleanupState.DEGRADED; + log.warn(message); + } else { + log.info(message); + } + } + + private void recordHealthyRun() { + if (cleanupState == CleanupState.DEGRADED) { + cleanupState = CleanupState.HEALTHY; + log.info("Transient RAG cleanup recovered"); + } + } + + private enum CleanupState { HEALTHY, DEGRADED } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java index c6c0473d..2ee37601 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java @@ -28,6 +28,28 @@ public class RagPipelineClient { private final boolean ragEnabled; private final String serviceSecret; + /** HTTP response failure with enough structure for bounded retry decisions. */ + public static final class RagApiException extends IOException { + private final int statusCode; + + public RagApiException(int statusCode, String detail) { + super("RAG API error: " + statusCode + " — " + detail); + this.statusCode = statusCode; + } + + public int getStatusCode() { + return statusCode; + } + + public boolean isServiceFailure() { + return statusCode == 401 + || statusCode == 403 + || statusCode == 408 + || statusCode == 429 + || statusCode >= 500; + } + } + public RagPipelineClient( @Value("${codecrow.rag.api.url:http://rag-pipeline:8001}") String ragApiUrl, @Value("${codecrow.rag.api.enabled:true}") boolean ragEnabled, @@ -110,6 +132,25 @@ public Map indexRepository( String collectionTarget, boolean publishBranchAlias, boolean publishLegacyProjectAlias + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, collectionTarget, + publishBranchAlias, publishLegacyProjectAlias, (String) null); + } + + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias, + String reuseCollectionTarget ) throws IOException { if (!ragEnabled) { log.debug("RAG indexing disabled, skipping repository indexing"); @@ -125,6 +166,9 @@ public Map indexRepository( if (collectionTarget != null && !collectionTarget.isBlank()) { payload.put("collection_target", collectionTarget); } + if (reuseCollectionTarget != null && !reuseCollectionTarget.isBlank()) { + payload.put("reuse_collection_target", reuseCollectionTarget); + } if (publishBranchAlias) { payload.put("publish_branch_alias", true); } @@ -182,6 +226,58 @@ public Map indexRepository( boolean publishBranchAlias, boolean publishLegacyProjectAlias, Consumer> progressConsumer + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, collectionTarget, + publishBranchAlias, publishLegacyProjectAlias, false, null, + progressConsumer); + } + + /** + * Streaming exact-generation indexing with an explicit shared-snapshot + * ownership handoff. The RAG service atomically moves the snapshot before + * it emits admission, so a lost stream cannot expose its active worker to + * caller-side deletion of the original path. + */ + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias, + boolean transferRepositoryOwnership, + Runnable ownershipAdmissionConsumer, + Consumer> progressConsumer + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, collectionTarget, + publishBranchAlias, publishLegacyProjectAlias, + transferRepositoryOwnership, null, ownershipAdmissionConsumer, + progressConsumer); + } + + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias, + boolean transferRepositoryOwnership, + String reuseCollectionTarget, + Runnable ownershipAdmissionConsumer, + Consumer> progressConsumer ) throws IOException { if (!ragEnabled) { log.debug("RAG indexing disabled, skipping repository indexing"); @@ -197,12 +293,18 @@ public Map indexRepository( if (collectionTarget != null && !collectionTarget.isBlank()) { payload.put("collection_target", collectionTarget); } + if (reuseCollectionTarget != null && !reuseCollectionTarget.isBlank()) { + payload.put("reuse_collection_target", reuseCollectionTarget); + } if (publishBranchAlias) { payload.put("publish_branch_alias", true); } if (publishLegacyProjectAlias) { payload.put("publish_legacy_project_alias", true); } + if (transferRepositoryOwnership) { + payload.put("transfer_repo_ownership", true); + } payload.put("source_tree_sha256", RepositorySourceTreeIdentity.sha256(Path.of(repoPath))); if (includePatterns != null && !includePatterns.isEmpty()) { payload.put("include_patterns", includePatterns); @@ -211,7 +313,8 @@ public Map indexRepository( payload.put("exclude_patterns", excludePatterns); } return postLongRunningSse( - ragApiUrl + "/index/repository/stream", payload, progressConsumer); + ragApiUrl + "/index/repository/stream", payload, + ownershipAdmissionConsumer, progressConsumer); } public Map updateFiles( @@ -374,6 +477,7 @@ public void publishGenerationAliases( String branch, String commit, String collectionTarget, + String generationManifestSha256, boolean publishBranchAlias, boolean publishLegacyProjectAlias) throws IOException { if (!ragEnabled || !publishBranchAlias) { @@ -385,6 +489,7 @@ public void publishGenerationAliases( payload.put("branch", branch); payload.put("commit", commit); payload.put("collection_target", collectionTarget); + payload.put("generation_manifest_sha256", generationManifestSha256); payload.put("publish_branch_alias", true); if (publishLegacyProjectAlias) { payload.put("publish_legacy_project_alias", true); @@ -515,31 +620,122 @@ public void deleteIndex(String workspace, String project, String branch) throws * @return true if points were deleted or already absent, false on error */ public boolean deletePrFiles(String workspace, String project, int prNumber) { + return deletePrFiles(workspace, project, prNumber, null); + } + + /** + * Deletes PR overlay points from one immutable physical generation. + * Passing no target retains the legacy project-alias behavior for projects + * that predate the generation registry. + */ + public boolean deletePrFiles( + String workspace, + String project, + int prNumber, + String collectionTarget) { + PrFilesDeletionOutcome outcome = deletePrFilesWithOutcome( + workspace, project, prNumber, collectionTarget); + if (!outcome.successful()) { + log.warn("Failed to delete PR #{} files from {}/{} target={}: status={} detail={}", + prNumber, workspace, project, outcome.targetLabel(), + outcome.statusCode() != null ? outcome.statusCode() : "transport", + outcome.detail()); + } + return outcome.successful(); + } + + /** + * Performs one PR-overlay deletion without logging. Multi-generation callers + * use the structured result to emit one contextual diagnostic and stop after + * a service-wide failure instead of repeating the same timeout per target. + */ + public PrFilesDeletionOutcome deletePrFilesWithOutcome( + String workspace, + String project, + int prNumber, + String collectionTarget) { + String targetLabel = collectionTarget != null && !collectionTarget.isBlank() + ? collectionTarget + : "legacy-alias"; if (!ragEnabled) { log.debug("RAG disabled, skipping PR files deletion"); - return true; + return PrFilesDeletionOutcome.success(targetLabel); } - String url = String.format("%s/index/pr-files/%s/%s/%d", ragApiUrl, workspace, project, prNumber); + HttpUrl.Builder urlBuilder = HttpUrl.get(String.format( + "%s/index/pr-files/%s/%s/%d", ragApiUrl, workspace, project, prNumber)).newBuilder(); + if (collectionTarget != null && !collectionTarget.isBlank()) { + urlBuilder.addQueryParameter("collection_target", collectionTarget); + } Request.Builder builder = new Request.Builder() - .url(url) + .url(urlBuilder.build()) .delete(); addAuthHeader(builder); Request request = builder.build(); try (Response response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { - log.info("Deleted PR #{} indexed data from {}/{}", prNumber, workspace, project); - return true; + log.info("Deleted PR #{} indexed data from {}/{} target={}", + prNumber, workspace, project, targetLabel); + return PrFilesDeletionOutcome.success(targetLabel); } else { - log.warn("Failed to delete PR #{} files: {} - {}", prNumber, response.code(), - response.body() != null ? response.body().string() : "no body"); - return false; + int statusCode = response.code(); + String detail = response.body() != null ? response.body().string() : "no body"; + boolean serviceFailure = statusCode == 401 + || statusCode == 403 + || statusCode == 408 + || statusCode == 409 + || statusCode == 429 + || statusCode >= 500; + return PrFilesDeletionOutcome.failure( + targetLabel, + serviceFailure + ? PrFilesDeletionFailure.SERVICE + : PrFilesDeletionFailure.TARGET, + statusCode, + truncateDetail(detail)); } } catch (IOException e) { - log.warn("Error deleting PR #{} files from {}/{}: {}", prNumber, workspace, project, e.getMessage()); - return false; + return PrFilesDeletionOutcome.failure( + targetLabel, + PrFilesDeletionFailure.TRANSPORT, + null, + e.getMessage()); + } + } + + public enum PrFilesDeletionFailure { + NONE, + TARGET, + SERVICE, + TRANSPORT + } + + public record PrFilesDeletionOutcome( + String targetLabel, + boolean successful, + PrFilesDeletionFailure failure, + Integer statusCode, + String detail) { + + public static PrFilesDeletionOutcome success(String targetLabel) { + return new PrFilesDeletionOutcome( + targetLabel, true, PrFilesDeletionFailure.NONE, null, null); + } + + public static PrFilesDeletionOutcome failure( + String targetLabel, + PrFilesDeletionFailure failure, + Integer statusCode, + String detail) { + return new PrFilesDeletionOutcome( + targetLabel, false, failure, statusCode, detail); + } + + public boolean shouldStopRemainingTargets() { + return failure == PrFilesDeletionFailure.SERVICE + || failure == PrFilesDeletionFailure.TRANSPORT; } } @@ -563,8 +759,62 @@ public boolean deleteBranch( String branch, String collectionTarget ) throws IOException { + return deleteBranch( + workspace, project, branch, collectionTarget, null, null); + } + + public boolean deleteBranch( + String workspace, + String project, + String branch, + String collectionTarget, + String generationRevision, + String generationManifestSha256 + ) throws IOException { + BranchDeletionOutcome outcome = deleteBranchWithOutcome( + workspace, project, branch, collectionTarget, + generationRevision, generationManifestSha256); + if (outcome.failure() == BranchDeletionFailure.TRANSPORT) { + throw new IOException(outcome.detail()); + } + if (!outcome.successful() + && !(outcome.statusCode() == null && "RAG disabled".equals(outcome.detail()))) { + log.warn("Failed to delete branch data target={}: status={} detail={}", + outcome.targetLabel(), outcome.statusCode(), outcome.detail()); + } + return outcome.successful(); + } + + /** Structured, non-logging branch deletion for multi-generation cleanup. */ + public BranchDeletionOutcome deleteBranchWithOutcome( + String workspace, + String project, + String branch, + String collectionTarget + ) { + return deleteBranchWithOutcome( + workspace, project, branch, collectionTarget, null, null); + } + + /** + * Deletes one exact generation using its registry-owned revision and + * manifest digest as an O(1) ownership proof. The RAG service retrieves the + * deterministic manifest point; it does not scan every collection member. + */ + public BranchDeletionOutcome deleteBranchWithOutcome( + String workspace, + String project, + String branch, + String collectionTarget, + String generationRevision, + String generationManifestSha256 + ) { + String targetLabel = collectionTarget != null && !collectionTarget.isBlank() + ? collectionTarget + : "legacy-alias"; if (!ragEnabled) { - return false; + return BranchDeletionOutcome.failure( + targetLabel, BranchDeletionFailure.TARGET, null, "RAG disabled"); } // URL-encode branch name to handle slashes (e.g., feature/xyz -> feature%2Fxyz) @@ -573,6 +823,14 @@ public boolean deleteBranch( "%s/index/%s/%s/branch/%s", ragApiUrl, workspace, project, encodedBranch)).newBuilder(); if (collectionTarget != null && !collectionTarget.isBlank()) { urlBuilder.addQueryParameter("collection_target", collectionTarget); + if (generationRevision != null && !generationRevision.isBlank()) { + urlBuilder.addQueryParameter("generation_revision", generationRevision); + } + if (generationManifestSha256 != null + && !generationManifestSha256.isBlank()) { + urlBuilder.addQueryParameter( + "generation_manifest_sha256", generationManifestSha256); + } } Request.Builder builder = new Request.Builder() @@ -583,13 +841,68 @@ public boolean deleteBranch( try (Response response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { - log.info("Deleted branch data for {}/{}/{}", workspace, project, branch); - return true; + log.info("Deleted branch data for {}/{}/{} target={}", + workspace, project, branch, targetLabel); + return BranchDeletionOutcome.success(targetLabel); } else { - log.warn("Failed to delete branch data: {} - {}", response.code(), - response.body() != null ? response.body().string() : "no body"); - return false; + int statusCode = response.code(); + String detail = response.body() != null + ? response.body().string() + : "no body"; + boolean serviceFailure = statusCode == 401 + || statusCode == 403 + || statusCode == 408 + || statusCode == 409 + || statusCode == 429 + || statusCode >= 500; + return BranchDeletionOutcome.failure( + targetLabel, + serviceFailure + ? BranchDeletionFailure.SERVICE + : BranchDeletionFailure.TARGET, + statusCode, + truncateDetail(detail)); } + } catch (IOException transportFailure) { + return BranchDeletionOutcome.failure( + targetLabel, + BranchDeletionFailure.TRANSPORT, + null, + transportFailure.getMessage()); + } + } + + public enum BranchDeletionFailure { + NONE, + TARGET, + SERVICE, + TRANSPORT + } + + public record BranchDeletionOutcome( + String targetLabel, + boolean successful, + BranchDeletionFailure failure, + Integer statusCode, + String detail) { + + public static BranchDeletionOutcome success(String targetLabel) { + return new BranchDeletionOutcome( + targetLabel, true, BranchDeletionFailure.NONE, null, null); + } + + public static BranchDeletionOutcome failure( + String targetLabel, + BranchDeletionFailure failure, + Integer statusCode, + String detail) { + return new BranchDeletionOutcome( + targetLabel, false, failure, statusCode, detail); + } + + public boolean shouldStopRemainingTargets() { + return failure == BranchDeletionFailure.SERVICE + || failure == BranchDeletionFailure.TRANSPORT; } } @@ -761,6 +1074,7 @@ private Map postLongRunning(String url, Map payl private Map postLongRunningSse( String url, Map payload, + Runnable ownershipAdmissionConsumer, Consumer> progressConsumer ) throws IOException { RequestBody body = RequestBody.create(objectMapper.writeValueAsString(payload), JSON); @@ -773,7 +1087,7 @@ private Map postLongRunningSse( try (Response response = longRunningHttpClient.newCall(builder.build()).execute()) { if (!response.isSuccessful()) { String detail = response.body() != null ? response.body().string() : "{}"; - throw new IOException("RAG API error: " + response.code() + " — " + detail); + throw new RagApiException(response.code(), detail); } if (response.body() == null) { throw new IOException("RAG progress stream returned no body"); @@ -789,6 +1103,15 @@ private Map postLongRunningSse( } Map event = objectMapper.readValue(json, Map.class); String type = String.valueOf(event.get("type")); + if ("admitted".equals(type)) { + if (Boolean.TRUE.equals( + event.get("repositoryOwnershipTransferred")) + && ownershipAdmissionConsumer != null) { + ownershipAdmissionConsumer.run(); + ownershipAdmissionConsumer = null; + } + continue; + } if ("progress".equals(type)) { if (progressConsumer != null) { progressConsumer.accept(new LinkedHashMap<>(event)); @@ -810,6 +1133,13 @@ private Map postLongRunningSse( throw new IOException("RAG progress stream ended without a terminal result"); } + private static String truncateDetail(String detail) { + if (detail == null) { + return "no detail"; + } + return detail.length() > 500 ? detail.substring(0, 500) + "..." : detail; + } + /** * Adds the x-service-secret header to the request if a secret is configured. */ @@ -834,12 +1164,14 @@ private Map doRequest(String url, Map payload, O String responseBody = response.body() != null ? response.body().string() : "{}"; if (!response.isSuccessful()) { - log.error("RAG API request failed: {} - {}", response.code(), responseBody); + // Callers own contextual, rate-bounded diagnostics. Keep the + // complete detail on the exception without logging it twice. + log.debug("RAG API request failed: {} - {}", response.code(), responseBody); // Include truncated response body in exception so callers can see the actual error String detail = responseBody.length() > 500 ? responseBody.substring(0, 500) + "..." : responseBody; - throw new IOException("RAG API error: " + response.code() + " — " + detail); + throw new RagApiException(response.code(), detail); } return objectMapper.readValue(responseBody, Map.class); diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java index 5b2a3ab8..b8e04c5a 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java @@ -14,6 +14,7 @@ import java.time.OffsetDateTime; import java.util.HexFormat; import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -41,7 +42,16 @@ public record BuildRegistration( RagBranchIndex branchIndex, RagBranchIndexGeneration generation, RagIndexOperation operation, - boolean existingOperation) { + boolean existingOperation, + String sourceCollectionTarget) { + + public BuildRegistration( + RagBranchIndex branchIndex, + RagBranchIndexGeneration generation, + RagIndexOperation operation, + boolean existingOperation) { + this(branchIndex, generation, operation, existingOperation, null); + } } @Transactional @@ -63,16 +73,27 @@ public BuildRegistration registerBuild( .findByProjectIdAndOperationKey(project.getId(), operationKey); if (existing.isPresent()) { RagIndexOperation operation = existing.get(); + Long branchIndexId = operation.getGeneration().getBranchIndex().getId(); + RagBranchIndex branchIndex = branchIndexRepository + .findByIdForPublication(branchIndexId) + .orElseThrow(() -> new IllegalStateException( + "RAG branch index not found: " + branchIndexId)); + rejectCleanupClaim(branchIndex); + branchIndex.markAccessed(); + branchIndexRepository.save(branchIndex); + operation.getGeneration().setBranchIndex(branchIndex); return new BuildRegistration( - operation.getGeneration().getBranchIndex(), + branchIndex, operation.getGeneration(), operation, - true); + true, + sourceCollectionTarget(operation.getGeneration())); } RagBranchIndex branchIndex = branchIndexRepository .findByProjectIdAndBranchNameForUpdate(project.getId(), branch) .orElseGet(() -> new RagBranchIndex(project, branch, kind)); + rejectCleanupClaim(branchIndex); if (branchIndex.getIndexKind() == RagBranchIndexKind.LEGACY || kind == RagBranchIndexKind.PRIMARY || (branchIndex.getIndexKind() == RagBranchIndexKind.TRANSIENT @@ -98,20 +119,45 @@ public BuildRegistration registerBuild( operation.setGeneration(generation); operation = operationRepository.save(operation); - return new BuildRegistration(branchIndex, generation, operation, false); + return new BuildRegistration( + branchIndex, + generation, + operation, + false, + sourceCollectionTarget(generation)); + } + + private static String sourceCollectionTarget( + RagBranchIndexGeneration generation) { + RagBranchIndexGeneration parent = generation.getParentGeneration(); + return parent != null ? parent.getCollectionName() : null; } @Transactional public void startBuild(long operationId, Long jobId) { - RagIndexOperation operation = requireOperation(operationId); + startBuild(operationId, jobId, null); + } + + @Transactional + public void startBuild(long operationId, Long jobId, String analysisLockKey) { + RagIndexOperation operation = requireOperationForUpdate(operationId); if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { return; } + String normalizedLockKey = normalizeOptional(analysisLockKey); + if (operation.getStatus() == RagIndexOperationStatus.RUNNING + && Objects.equals(operation.getJobId(), jobId) + && Objects.equals(operation.getAnalysisLockKey(), normalizedLockKey)) { + operation.heartbeat(); + operationRepository.save(operation); + return; + } if (operation.getStatus() == RagIndexOperationStatus.FAILED) { operation.getGeneration().retry(); generationRepository.save(operation.getGeneration()); } operation.setJobId(jobId); + operation.setAnalysisLockKey(normalizedLockKey); operation.start(); operationRepository.save(operation); } @@ -122,7 +168,7 @@ public RagBranchIndexGeneration publish( String manifestDigest, int fileCount, int chunkCount) { - RagIndexOperation operation = requireOperation(operationId); + RagIndexOperation operation = requireOperationForUpdate(operationId); RagBranchIndexGeneration generation = operation.getGeneration(); if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { return generation; @@ -137,6 +183,7 @@ public RagBranchIndexGeneration publish( .orElseThrow(() -> new IllegalStateException( "RAG branch index not found: " + branchIndexId)); generation.setBranchIndex(branchIndex); + rejectCleanupClaim(branchIndex); String digest = requireText(manifestDigest, "manifestDigest"); if (!generation.getRevision().equals(branchIndex.getDesiredCommitHash())) { @@ -166,10 +213,33 @@ public RagBranchIndexGeneration publish( } @Transactional - public void fail(long operationId, String errorMessage) { - RagIndexOperation operation = requireOperation(operationId); - if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { - return; + public boolean fail(long operationId, String errorMessage) { + return failOperation(requireOperationForUpdate(operationId), errorMessage); + } + + /** + * Claims an abandoned operation only after serializing with heartbeat and + * publication updates and re-checking the cutoff inside that transaction. + */ + @Transactional + public boolean failIfAbandoned( + long operationId, + OffsetDateTime updatedBefore, + String errorMessage) { + RagIndexOperation operation = requireOperationForUpdate(operationId); + if ((operation.getStatus() != RagIndexOperationStatus.PENDING + && operation.getStatus() != RagIndexOperationStatus.RUNNING) + || operation.getUpdatedAt() == null + || !operation.getUpdatedAt().isBefore(updatedBefore)) { + return false; + } + return failOperation(operation, errorMessage); + } + + private boolean failOperation(RagIndexOperation operation, String errorMessage) { + if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED + || operation.getStatus() == RagIndexOperationStatus.FAILED) { + return false; } String failure = requireText(errorMessage, "errorMessage"); RagBranchIndexGeneration generation = operation.getGeneration(); @@ -187,6 +257,7 @@ public void fail(long operationId, String errorMessage) { } operation.fail(failure); operationRepository.save(operation); + return true; } @Transactional @@ -195,11 +266,17 @@ public Optional findAvailableGeneration( String branchName, String revision) { Optional branchIndex = branchIndexRepository - .findByProjectIdAndBranchName(projectId, requireText(branchName, "branchName")); + .findByProjectIdAndBranchNameForUpdate( + projectId, requireText(branchName, "branchName")); if (branchIndex.isEmpty()) { return Optional.empty(); } RagBranchIndex index = branchIndex.get(); + if (index.getCleanupClaimToken() != null) { + // Cleanup has a durable claim. Returning a generation here would + // expose a physical target that may already have been deleted. + return Optional.empty(); + } index.markAccessed(); branchIndexRepository.save(index); if (index.getActiveGeneration() != null @@ -215,17 +292,35 @@ public Optional findAvailableGeneration( @Transactional public void heartbeatBuild(long operationId) { - RagIndexOperation operation = requireOperation(operationId); + RagIndexOperation operation = requireOperationForUpdate(operationId); operation.heartbeat(); operationRepository.save(operation); } - public List findRecoverableOperations(OffsetDateTime updatedBefore) { - return operationRepository.findByStatusInAndUpdatedAtBefore( + public List + findRecoverableOperations(OffsetDateTime updatedBefore) { + return operationRepository.findRecoverableOperationProjections( List.of(RagIndexOperationStatus.PENDING, RagIndexOperationStatus.RUNNING), updatedBefore); } + public List + findFailedOperationsWithActiveProjections() { + return operationRepository.findFailedOperationsWithActiveProjections(); + } + + public List + findSucceededOperationsWithActiveProjections() { + return operationRepository.findSucceededOperationsWithActiveProjections(); + } + + public boolean hasLiveOperation(long projectId, String branchName) { + return operationRepository.existsByProjectIdAndBranchNameAndStatusIn( + projectId, + requireText(branchName, "branchName"), + List.of(RagIndexOperationStatus.PENDING, RagIndexOperationStatus.RUNNING)); + } + static String physicalCollectionName( Project project, String branchName, @@ -254,8 +349,8 @@ static String operationKey( + nullToEmpty(representationFingerprint)); } - private RagIndexOperation requireOperation(long operationId) { - return operationRepository.findById(operationId) + private RagIndexOperation requireOperationForUpdate(long operationId) { + return operationRepository.findByIdForUpdate(operationId) .orElseThrow(() -> new IllegalArgumentException("RAG index operation not found: " + operationId)); } @@ -272,6 +367,17 @@ private static String requireText(String value, String field) { return value.trim(); } + private static String normalizeOptional(String value) { + return value == null || value.isBlank() ? null : value.trim(); + } + + private static void rejectCleanupClaim(RagBranchIndex branchIndex) { + if (branchIndex.getCleanupClaimToken() != null) { + throw new IllegalStateException( + "RAG branch index cleanup owns this transient branch; retry later"); + } + } + private static String nullToEmpty(String value) { return value == null ? "" : value; } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java index 6e27a9d6..49b7e03a 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java @@ -10,6 +10,7 @@ import org.springframework.transaction.annotation.Transactional; import java.time.OffsetDateTime; +import java.util.Objects; import java.util.Optional; @Service @@ -34,8 +35,13 @@ public Optional getIndexStatus(Project project) { } @Transactional - public RagIndexStatus markIndexingStarted(Project project, String branchName, String commitHash) { - Optional existingOpt = ragIndexStatusRepository.findByProjectId(project.getId()); + public RagIndexStatus markIndexingStarted( + Project project, + String branchName, + String commitHash, + Long activeJobId) { + Optional existingOpt = + ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()); RagIndexStatus status; if (existingOpt.isPresent()) { @@ -44,6 +50,7 @@ public RagIndexStatus markIndexingStarted(Project project, String branchName, St status.setIndexedBranch(branchName); status.setIndexedCommitHash(commitHash); status.setErrorMessage(null); + status.setActiveJobId(activeJobId); } else { status = new RagIndexStatus(); status.setProject(project); @@ -53,6 +60,7 @@ public RagIndexStatus markIndexingStarted(Project project, String branchName, St status.setIndexedBranch(branchName); status.setIndexedCommitHash(commitHash); status.setCollectionName(generateCollectionName(project)); + status.setActiveJobId(activeJobId); } status = ragIndexStatusRepository.save(status); @@ -63,10 +71,26 @@ public RagIndexStatus markIndexingStarted(Project project, String branchName, St @Transactional public RagIndexStatus markIndexingCompleted(Project project, String branchName, String commitHash, Integer filesIndexed, Integer chunkCount) { - RagIndexStatus status = ragIndexStatusRepository.findByProjectId(project.getId()) + return markIndexingCompleted( + project, branchName, commitHash, filesIndexed, chunkCount, null); + } + + @Transactional + public RagIndexStatus markIndexingCompleted( + Project project, + String branchName, + String commitHash, + Integer filesIndexed, + Integer chunkCount, + Long expectedActiveJobId) { + RagIndexStatus status = ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()) .orElseThrow( () -> new IllegalStateException("RAG index status not found for project: " + project.getId())); + if (!ownsStatus(status, expectedActiveJobId, "complete full indexing")) { + return status; + } + status.setStatus(RagIndexingStatus.INDEXED); status.setIndexedBranch(branchName); status.setIndexedCommitHash(commitHash); @@ -76,6 +100,7 @@ public RagIndexStatus markIndexingCompleted(Project project, String branchName, } status.setLastIndexedAt(OffsetDateTime.now()); status.setErrorMessage(null); + status.setActiveJobId(null); // Reset failed incremental count on successful full index status.resetFailedIncrementalCount(); @@ -87,13 +112,26 @@ public RagIndexStatus markIndexingCompleted(Project project, String branchName, @Transactional public RagIndexStatus markIndexingFailed(Project project, String errorMessage) { - Optional existingOpt = ragIndexStatusRepository.findByProjectId(project.getId()); + return markIndexingFailed(project, errorMessage, null); + } + + @Transactional + public RagIndexStatus markIndexingFailed( + Project project, + String errorMessage, + Long expectedActiveJobId) { + Optional existingOpt = + ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()); RagIndexStatus status; if (existingOpt.isPresent()) { status = existingOpt.get(); + if (!ownsStatus(status, expectedActiveJobId, "fail full indexing")) { + return status; + } status.setStatus(RagIndexingStatus.FAILED); status.setErrorMessage(errorMessage); + status.setActiveJobId(null); } else { status = new RagIndexStatus(); status.setProject(project); @@ -102,6 +140,7 @@ public RagIndexStatus markIndexingFailed(Project project, String errorMessage) { status.setStatus(RagIndexingStatus.FAILED); status.setErrorMessage(errorMessage); status.setCollectionName(generateCollectionName(project)); + status.setActiveJobId(null); } status = ragIndexStatusRepository.save(status); @@ -118,14 +157,22 @@ public RagIndexStatus markIndexingFailed(Project project, String errorMessage) { */ @Transactional public boolean markIndexingHeartbeat(Project project) { + return markIndexingHeartbeat(project, null); + } + + @Transactional + public boolean markIndexingHeartbeat(Project project, Long expectedActiveJobId) { Optional statusOpt = - ragIndexStatusRepository.findByProjectId(project.getId()); + ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()); if (statusOpt.isEmpty()) { log.warn("Ignoring RAG heartbeat without index status for project {}", project.getId()); return false; } RagIndexStatus status = statusOpt.get(); + if (!ownsStatus(status, expectedActiveJobId, "record indexing heartbeat")) { + return false; + } if (status.getStatus() != RagIndexingStatus.INDEXING && status.getStatus() != RagIndexingStatus.UPDATING) { log.debug( @@ -141,12 +188,17 @@ public boolean markIndexingHeartbeat(Project project) { } @Transactional - public RagIndexStatus markUpdatingStarted(Project project, String branchName, String commitHash) { - RagIndexStatus status = ragIndexStatusRepository.findByProjectId(project.getId()) + public RagIndexStatus markUpdatingStarted( + Project project, + String branchName, + String commitHash, + Long activeJobId) { + RagIndexStatus status = ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()) .orElseThrow(() -> new IllegalStateException("Cannot update non-indexed project: " + project.getId())); status.setStatus(RagIndexingStatus.UPDATING); status.setErrorMessage(null); + status.setActiveJobId(activeJobId); status = ragIndexStatusRepository.save(status); log.info("Marked RAG indexing as UPDATING for project {} toward branch {} commit {}; " @@ -155,6 +207,122 @@ public RagIndexStatus markUpdatingStarted(Project project, String branchName, St return status; } + /** + * Atomically aligns the project checkpoint with an already-published exact + * generation. This is used by same-revision no-ops and post-publication + * recovery, neither of which needs a synthetic RUNNING status transition. + * A newer job owner is never overwritten. + */ + @Transactional + public boolean reconcilePublishedGeneration( + Project project, + String branchName, + String commitHash, + Integer fileCount, + Integer chunkCount) { + return reconcilePublishedGeneration( + project, branchName, commitHash, fileCount, chunkCount, null); + } + + @Transactional + public boolean reconcilePublishedGeneration( + Project project, + String branchName, + String commitHash, + Integer fileCount, + Integer chunkCount, + Long expectedActiveJobId) { + Optional existing = + ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()); + RagIndexStatus status; + if (existing.isPresent()) { + status = existing.get(); + if (expectedActiveJobId != null + && !Objects.equals(status.getActiveJobId(), expectedActiveJobId)) { + log.info( + "Preserving RAG status owned by job {} while job {} reconciles " + + "a published generation for project {}", + status.getActiveJobId(), expectedActiveJobId, project.getId()); + return false; + } + if (expectedActiveJobId == null && status.getActiveJobId() != null) { + log.info( + "Preserving RAG status owned by live job {} while reconciling " + + "published generation for project {}", + status.getActiveJobId(), project.getId()); + return false; + } + } else { + if (expectedActiveJobId != null) { + log.info( + "Published generation for job {} has no owned RAG status to reconcile " + + "for project {}", + expectedActiveJobId, project.getId()); + return false; + } + status = new RagIndexStatus(); + status.setProject(project); + status.setWorkspaceName(project.getWorkspace().getName()); + status.setProjectName(project.getName()); + status.setCollectionName(generateCollectionName(project)); + } + + status.setStatus(RagIndexingStatus.INDEXED); + status.setIndexedBranch(branchName); + status.setIndexedCommitHash(commitHash); + if (fileCount != null) { + status.setTotalFilesIndexed(fileCount); + } + if (chunkCount != null) { + status.setChunkCount(chunkCount); + } + status.setLastIndexedAt(OffsetDateTime.now()); + status.setErrorMessage(null); + status.setActiveJobId(null); + status.resetFailedIncrementalCount(); + ragIndexStatusRepository.save(status); + return true; + } + + /** + * Restores the active exact generation as the completed checkpoint while + * the caller owns the branch RAG lock. The caller immediately admits the + * replacement job in the same transaction, so an older status owner + * cannot later publish through the job-id ownership checks. + */ + @Transactional + public void preparePublishedGenerationForUpdate( + Project project, + String branchName, + String commitHash, + Integer fileCount, + Integer chunkCount) { + RagIndexStatus status = ragIndexStatusRepository + .findByProjectIdForUpdate(project.getId()) + .orElseGet(() -> { + RagIndexStatus created = new RagIndexStatus(); + created.setProject(project); + created.setWorkspaceName(project.getWorkspace().getName()); + created.setProjectName(project.getName()); + created.setCollectionName(generateCollectionName(project)); + return created; + }); + status.setStatus(RagIndexingStatus.INDEXED); + status.setIndexedBranch(branchName); + status.setIndexedCommitHash(commitHash); + if (fileCount != null) { + status.setTotalFilesIndexed(fileCount); + } + if (chunkCount != null) { + status.setChunkCount(chunkCount); + } + status.setLastIndexedAt(OffsetDateTime.now()); + status.setErrorMessage(null); + status.setActiveJobId(null); + status.resetFailedIncrementalCount(); + ragIndexStatusRepository.save(status); + } + /** * Marks an incremental update as completed. * Updates totalFilesIndexed by adding addedFiles count and subtracting @@ -164,7 +332,22 @@ public RagIndexStatus markUpdatingStarted(Project project, String branchName, St public RagIndexStatus markUpdatingCompleted(Project project, String branchName, String commitHash, Integer addedFilesCount, Integer deletedFilesCount, Integer chunkCount) { return markUpdatingCompleted( - project, branchName, commitHash, addedFilesCount, deletedFilesCount, chunkCount, true); + project, branchName, commitHash, addedFilesCount, deletedFilesCount, + chunkCount, true, null); + } + + @Transactional + public RagIndexStatus markUpdatingCompleted( + Project project, + String branchName, + String commitHash, + Integer addedFilesCount, + Integer deletedFilesCount, + Integer chunkCount, + Long expectedActiveJobId) { + return markUpdatingCompleted( + project, branchName, commitHash, addedFilesCount, deletedFilesCount, + chunkCount, true, expectedActiveJobId); } /** @@ -176,10 +359,29 @@ public RagIndexStatus markUpdatingCompleted(Project project, String branchName, public RagIndexStatus markUpdatingCompleted(Project project, String branchName, String commitHash, Integer addedFilesCount, Integer deletedFilesCount, Integer chunkCount, boolean advanceProjectCheckpoint) { - RagIndexStatus status = ragIndexStatusRepository.findByProjectId(project.getId()) + return markUpdatingCompleted( + project, branchName, commitHash, addedFilesCount, deletedFilesCount, + chunkCount, advanceProjectCheckpoint, null); + } + + @Transactional + public RagIndexStatus markUpdatingCompleted( + Project project, + String branchName, + String commitHash, + Integer addedFilesCount, + Integer deletedFilesCount, + Integer chunkCount, + boolean advanceProjectCheckpoint, + Long expectedActiveJobId) { + RagIndexStatus status = ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()) .orElseThrow( () -> new IllegalStateException("RAG index status not found for project: " + project.getId())); + if (!ownsStatus(status, expectedActiveJobId, "complete incremental indexing")) { + return status; + } + status.setStatus(RagIndexingStatus.INDEXED); if (advanceProjectCheckpoint) { status.setIndexedBranch(branchName); @@ -197,6 +399,7 @@ public RagIndexStatus markUpdatingCompleted(Project project, String branchName, status.setLastIndexedAt(OffsetDateTime.now()); status.setErrorMessage(null); + status.setActiveJobId(null); // Reset failed incremental count on successful update status.resetFailedIncrementalCount(); @@ -214,15 +417,28 @@ public RagIndexStatus markUpdatingCompleted(Project project, String branchName, */ @Transactional public RagIndexStatus markIncrementalUpdateFailed(Project project, String errorMessage) { - RagIndexStatus status = ragIndexStatusRepository.findByProjectId(project.getId()) + return markIncrementalUpdateFailed(project, errorMessage, null); + } + + @Transactional + public RagIndexStatus markIncrementalUpdateFailed( + Project project, + String errorMessage, + Long expectedActiveJobId) { + RagIndexStatus status = ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()) .orElseThrow( () -> new IllegalStateException("RAG index status not found for project: " + project.getId())); + if (!ownsStatus(status, expectedActiveJobId, "fail incremental indexing")) { + return status; + } + // Restore the usable terminal state and retain the last completed // branch/commit checkpoint. The attempted commit is never published. status.setStatus(RagIndexingStatus.INDEXED); status.setErrorMessage("Incremental update failed: " + errorMessage); status.incrementFailedIncrementalCount(); + status.setActiveJobId(null); status = ragIndexStatusRepository.save(status); log.warn("Marked RAG incremental update as FAILED for project {} (failure count: {}): {}", @@ -230,6 +446,20 @@ public RagIndexStatus markIncrementalUpdateFailed(Project project, String errorM return status; } + /** + * Quiet, owner-guarded repair used by the periodic legacy-job recovery + * scan. The scheduler owns degraded/recovered log transitions, preventing + * the same database outage from producing one warning per scan and job. + */ + @Transactional + public boolean recoverAbandonedIncrementalUpdate( + Long projectId, + Long expectedActiveJobId, + String errorMessage) { + return ragIndexStatusRepository.recoverAbandonedIncrementalUpdate( + projectId, expectedActiveJobId, errorMessage) == 1; + } + @Transactional(readOnly = true) public boolean canStartIndexing(Project project) { Optional statusOpt = ragIndexStatusRepository.findByProjectId(project.getId()); @@ -248,4 +478,21 @@ private String generateCollectionName(Project project) { String projectName = project.getName().replaceAll("[^a-zA-Z0-9_-]", "_"); return String.format("%s_%s", workspace, projectName).toLowerCase(); } + + private boolean ownsStatus( + RagIndexStatus status, + Long expectedActiveJobId, + String transition) { + if (Objects.equals(status.getActiveJobId(), expectedActiveJobId)) { + return true; + } + log.info( + "Ignoring stale RAG status transition '{}' for project {}: " + + "expected owner {}, current owner {}", + transition, + status.getProject() != null ? status.getProject().getId() : null, + expectedActiveJobId, + status.getActiveJobId()); + return false; + } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java index e37f2e84..6cad6d3f 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java @@ -18,6 +18,9 @@ import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; +import org.rostilos.codecrow.ragengine.branch.BranchIndexBuildAdmissionService; +import org.rostilos.codecrow.ragengine.branch.LegacyRagJobLeaseService; +import org.rostilos.codecrow.ragengine.branch.LegacyRagUpdateCompletionService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.slf4j.Logger; @@ -30,6 +33,7 @@ import java.io.IOException; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -58,6 +62,9 @@ public class RagOperationsServiceImpl implements RagOperationsService { private final RagPipelineClient ragPipelineClient; private final RagBranchIndexRegistryService branchIndexRegistryService; private final BranchIndexGenerationBuildService branchGenerationBuildService; + private final BranchIndexBuildAdmissionService branchIndexBuildAdmissionService; + private final LegacyRagJobLeaseService legacyRagJobLeaseService; + private final LegacyRagUpdateCompletionService legacyRagUpdateCompletionService; @Autowired(required = false) private RagBranchIndexGenerationRepository branchGenerationRepository; @@ -65,6 +72,9 @@ public class RagOperationsServiceImpl implements RagOperationsService { @Value("${codecrow.rag.api.enabled:true}") private boolean ragApiEnabled; + @Value("${analysis.lock.rag.timeout.minutes:360}") + private int legacyRagLockLeaseMinutes = 360; + public RagOperationsServiceImpl( RagIndexTrackingService ragIndexTrackingService, IncrementalRagUpdateService incrementalRagUpdateService, @@ -76,7 +86,7 @@ public RagOperationsServiceImpl( this(ragIndexTrackingService, incrementalRagUpdateService, analysisLockService, analysisJobService, ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, null, null); + ragPipelineClient, null, null, null, null, null); } public RagOperationsServiceImpl( @@ -91,10 +101,9 @@ public RagOperationsServiceImpl( this(ragIndexTrackingService, incrementalRagUpdateService, analysisLockService, analysisJobService, ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, branchIndexRegistryService, null); + ragPipelineClient, branchIndexRegistryService, null, null, null, null); } - @Autowired public RagOperationsServiceImpl( RagIndexTrackingService ragIndexTrackingService, IncrementalRagUpdateService incrementalRagUpdateService, @@ -105,6 +114,46 @@ public RagOperationsServiceImpl( RagPipelineClient ragPipelineClient, RagBranchIndexRegistryService branchIndexRegistryService, BranchIndexGenerationBuildService branchGenerationBuildService) { + this(ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, branchIndexRegistryService, + branchGenerationBuildService, null, null, null); + } + + public RagOperationsServiceImpl( + RagIndexTrackingService ragIndexTrackingService, + IncrementalRagUpdateService incrementalRagUpdateService, + AnalysisLockService analysisLockService, + AnalysisJobService analysisJobService, + RagBranchIndexRepository ragBranchIndexRepository, + VcsClientProvider vcsClientProvider, + RagPipelineClient ragPipelineClient, + RagBranchIndexRegistryService branchIndexRegistryService, + BranchIndexGenerationBuildService branchGenerationBuildService, + BranchIndexBuildAdmissionService branchIndexBuildAdmissionService) { + this(ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, branchIndexRegistryService, + branchGenerationBuildService, branchIndexBuildAdmissionService, + null, null); + } + + @Autowired + public RagOperationsServiceImpl( + RagIndexTrackingService ragIndexTrackingService, + IncrementalRagUpdateService incrementalRagUpdateService, + AnalysisLockService analysisLockService, + AnalysisJobService analysisJobService, + RagBranchIndexRepository ragBranchIndexRepository, + VcsClientProvider vcsClientProvider, + RagPipelineClient ragPipelineClient, + RagBranchIndexRegistryService branchIndexRegistryService, + BranchIndexGenerationBuildService branchGenerationBuildService, + BranchIndexBuildAdmissionService branchIndexBuildAdmissionService, + LegacyRagJobLeaseService legacyRagJobLeaseService, + LegacyRagUpdateCompletionService legacyRagUpdateCompletionService) { this.ragIndexTrackingService = ragIndexTrackingService; this.incrementalRagUpdateService = incrementalRagUpdateService; this.analysisLockService = analysisLockService; @@ -114,6 +163,9 @@ public RagOperationsServiceImpl( this.ragPipelineClient = ragPipelineClient; this.branchIndexRegistryService = branchIndexRegistryService; this.branchGenerationBuildService = branchGenerationBuildService; + this.branchIndexBuildAdmissionService = branchIndexBuildAdmissionService; + this.legacyRagJobLeaseService = legacyRagJobLeaseService; + this.legacyRagUpdateCompletionService = legacyRagUpdateCompletionService; } @Override @@ -136,6 +188,19 @@ public boolean isRagIndexReady(Project project) { if (!isRagEnabled(project)) { return false; } + if (usesExactGenerations(project)) { + String primaryBranch = getBaseBranch(project); + boolean primaryRegistered = ragBranchIndexRepository + .existsByProjectIdAndBranchName(project.getId(), primaryBranch); + if (primaryRegistered) { + if (ragBranchIndexRepository.markAccessedIfUnclaimed( + project.getId(), primaryBranch, OffsetDateTime.now()) == 0) { + return false; + } + return ragBranchIndexRepository.findActiveGenerationCoordinates( + project.getId(), primaryBranch).isPresent(); + } + } return ragIndexTrackingService.isProjectIndexed(project); } @@ -149,13 +214,81 @@ public boolean deletePrFiles(Project project, int prNumber) { try { String workspace = project.getWorkspace().getName(); String namespace = project.getNamespace(); - return ragPipelineClient.deletePrFiles(workspace, namespace, prNumber); + Set collectionTargets = new LinkedHashSet<>(); + if (branchGenerationRepository != null) { + List registeredTargets = branchGenerationRepository + .findCollectionNamesByProjectIdAndStatusIn( + project.getId(), + List.of( + RagBranchIndexGenerationStatus.ACTIVE, + RagBranchIndexGenerationStatus.SUPERSEDED)); + if (registeredTargets != null) { + registeredTargets.stream() + .filter(target -> target != null && !target.isBlank()) + .map(String::trim) + .forEach(collectionTargets::add); + } + } + + if (collectionTargets.isEmpty()) { + // Legacy projects have no physical-generation registry. Their + // project alias remains the only cleanup target. + RagPipelineClient.PrFilesDeletionOutcome outcome; + try { + outcome = ragPipelineClient.deletePrFilesWithOutcome( + workspace, namespace, prNumber, null); + } catch (RuntimeException unexpectedFailure) { + log.warn("Failed to delete PR #{} files for project={} target=legacy-alias: " + + "status=unexpected detail={}", + prNumber, project.getId(), unexpectedFailure.getMessage()); + return false; + } + if (!outcome.successful()) { + logPrCleanupFailure(project, prNumber, outcome); + } + return outcome.successful(); + } + + boolean allTargetsCleaned = true; + for (String collectionTarget : collectionTargets) { + RagPipelineClient.PrFilesDeletionOutcome outcome; + try { + outcome = ragPipelineClient.deletePrFilesWithOutcome( + workspace, namespace, prNumber, collectionTarget); + } catch (RuntimeException unexpectedFailure) { + log.warn("Failed to delete PR #{} files for project={} target={}: " + + "status=unexpected detail={}", + prNumber, project.getId(), collectionTarget, + unexpectedFailure.getMessage()); + return false; + } + if (!outcome.successful()) { + allTargetsCleaned = false; + logPrCleanupFailure(project, prNumber, outcome); + if (outcome.shouldStopRemainingTargets()) { + break; + } + } + } + return allTargetsCleaned; } catch (Exception e) { log.warn("Failed to delete PR #{} files for project {}: {}", prNumber, project.getId(), e.getMessage()); return false; } } + private static void logPrCleanupFailure( + Project project, + int prNumber, + RagPipelineClient.PrFilesDeletionOutcome outcome) { + log.warn("Failed to delete PR #{} files for project={} target={}: status={} detail={}", + prNumber, + project.getId(), + outcome.targetLabel(), + outcome.statusCode() != null ? outcome.statusCode() : outcome.failure(), + outcome.detail()); + } + @Override public boolean triggerIncrementalUpdate( Project project, @@ -167,63 +300,64 @@ public boolean triggerIncrementalUpdate( log.info("triggerIncrementalUpdate called for project={}, branch={}, commit={}, diffLength={}", project.getId(), branchName, commitHash, rawDiff != null ? rawDiff.length() : 0); try { - if (!incrementalRagUpdateService.shouldPerformIncrementalUpdate(project)) { + boolean exactGenerationMode = usesExactGenerations(project); + boolean normalIncrementalReady = + incrementalRagUpdateService.shouldPerformIncrementalUpdate(project); + boolean exactRecoveryCandidate = exactGenerationMode + && isRagEnabled(project) + && ragBranchIndexRepository.existsByProjectIdAndBranchName( + project.getId(), branchName); + if (!normalIncrementalReady && !exactRecoveryCandidate) { log.info( "Skipping RAG incremental update for project={}, branch={} - RAG not enabled or main branch not yet indexed", project.getId(), branchName); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "message", "Skipping RAG update - main branch must be indexed first")); return false; } - boolean exactGenerationMode = usesExactGenerations(project); + boolean tracksProjectStatus = branchName.equals(getBaseBranch(project)); RagBranchIndexKind exactGenerationKind = exactGenerationMode ? indexKind(project, branchName) : null; - boolean publishBranchAlias = exactGenerationKind == RagBranchIndexKind.PRIMARY - || exactGenerationKind == RagBranchIndexKind.DURABLE; - boolean publishLegacyProjectAlias = exactGenerationKind - == RagBranchIndexKind.PRIMARY; - RagBranchIndexGeneration initialSourceGeneration = exactGenerationMode - ? ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), branchName) - .map(RagBranchIndex::getActiveGeneration) - .orElse(null) - : null; - boolean fullExactSnapshotRequired = exactGenerationMode - && initialSourceGeneration == null; - - String effectiveRawDiff = fullExactSnapshotRequired - ? "" - : resolveDiffFromCompletedCheckpoint( - project, branchName, commitHash, rawDiff); - log.info("RAG checkpoint reconciliation complete; parsing effective diff..."); - - // Parse the diff to find changed files - IncrementalRagUpdateService.DiffResult diffResult = - incrementalRagUpdateService.parseDiffForRag(effectiveRawDiff); - Set addedFiles = diffResult.added(); - Set modifiedFiles = diffResult.modified(); - Set deletedFiles = diffResult.deleted(); - - int addedOrModifiedSize = addedFiles.size() + modifiedFiles.size(); - - log.info("Diff parsed: added={}, modified={}, deleted={}", addedFiles, modifiedFiles, deletedFiles); - - if (addedOrModifiedSize == 0 && deletedFiles.isEmpty() - && !exactGenerationMode) { - log.info("Skipping RAG incremental update - no files changed in diff"); - return true; + Set addedFiles = Set.of(); + Set modifiedFiles = Set.of(); + Set deletedFiles = Set.of(); + int addedOrModifiedSize = 0; + + // Preserve the legacy checkpoint flow, including its no-op return + // before a job or lock is created. Exact generations are resolved + // below only after this update owns the branch RAG lock. + if (!exactGenerationMode) { + String effectiveRawDiff = resolveDiffFromCompletedCheckpoint( + project, branchName, commitHash, rawDiff); + log.info("RAG checkpoint reconciliation complete; parsing effective diff..."); + IncrementalRagUpdateService.DiffResult diffResult = + incrementalRagUpdateService.parseDiffForRag(effectiveRawDiff); + addedFiles = diffResult.added(); + modifiedFiles = diffResult.modified(); + deletedFiles = diffResult.deleted(); + addedOrModifiedSize = addedFiles.size() + modifiedFiles.size(); + + log.info("Diff parsed: added={}, modified={}, deleted={}", + addedFiles, modifiedFiles, deletedFiles); + if (addedOrModifiedSize == 0 && deletedFiles.isEmpty()) { + log.info("Skipping RAG incremental update - no files changed in diff"); + return true; + } + log.info("RAG incremental update: {} files to add/update, {} files to delete", + addedOrModifiedSize, deletedFiles.size()); } - log.info("RAG incremental update: {} files to add/update, {} files to delete", - addedOrModifiedSize, deletedFiles.size()); - - job = analysisJobService.createRagIndexJob(project, false, JobTriggerSource.WEBHOOK); - analysisJobService.info(job, "rag_init", - String.format( - "Starting incremental RAG update for branch '%s' (commit: %s) - %d files to update, %d to delete", - branchName, commitHash, addedOrModifiedSize, deletedFiles.size())); + if (!exactGenerationMode) { + job = analysisJobService.createRagIndexJob( + project, false, JobTriggerSource.WEBHOOK, branchName, commitHash); + analysisJobService.startJob(job); + analysisJobService.info(job, "rag_init", + String.format( + "Starting incremental RAG update for branch '%s' (commit: %s) - %d files to update, %d to delete", + branchName, commitHash, addedOrModifiedSize, deletedFiles.size())); + } Optional ragLockKey = analysisLockService.acquireLock( project, @@ -233,21 +367,137 @@ public boolean triggerIncrementalUpdate( null); if (ragLockKey.isEmpty()) { - log.warn("RAG update already in progress for project={}, branch={}", + log.info("RAG update already in progress for project={}, branch={}; " + + "deferring this revision to a later trigger", project.getId(), branchName); - analysisJobService.warn(job, "rag_skip", "RAG update already in progress - skipping"); - analysisJobService.failJob(job, "RAG update already in progress"); + String reason = "RAG update already in progress; this revision was skipped and " + + "the previous checkpoint is retained for the next trigger"; + if (job != null) { + analysisJobService.info(job, "rag_skip", reason); + analysisJobService.skipJob(job, reason); + } + emitEvent(eventConsumer, Map.of( + "type", "info", + "state", "rag_skip", + "message", reason)); return false; } + BranchIndexBuildAdmissionService.AdmittedBuild admittedBuild = null; + boolean exactExecutionStarted = false; + boolean exactPublicationCompleted = false; + LegacyRagJobLeaseService.JobLease legacyJobLease = null; + AnalysisLockService.LockLease legacyLockLease = null; try { - eventConsumer.accept(Map.of( - "type", "status", - "state", "rag_update", - "message", - "Updating RAG index with " + (addedOrModifiedSize + deletedFiles.size()) + " changed files")); + if (!exactGenerationMode) { + if (legacyRagJobLeaseService == null + || job == null + || job.getId() == null) { + throw new IllegalStateException( + "Legacy RAG job lease service is unavailable"); + } + legacyJobLease = legacyRagJobLeaseService.start(job.getId()); + legacyLockLease = analysisLockService.maintainLockLease( + ragLockKey.get(), Math.max(1, legacyRagLockLeaseMinutes)); + requireLegacyOwnership(legacyJobLease, legacyLockLease); + } + RagBranchIndexRepository.ActiveGenerationCoordinates sourceGeneration = null; + if (exactGenerationMode) { + boolean existingExactIndex = ragBranchIndexRepository + .existsByProjectIdAndBranchName(project.getId(), branchName); + if (existingExactIndex) { + int accessed = ragBranchIndexRepository.markAccessedIfUnclaimed( + project.getId(), branchName, OffsetDateTime.now()); + if (accessed == 0) { + log.info("Skipping exact RAG update while transient cleanup owns " + + "project={}, branch={}", + project.getId(), branchName); + String reason = "Temporary branch index cleanup is in progress; " + + "the update will retry later"; + emitEvent(eventConsumer, Map.of( + "type", "info", + "state", "rag_skipped", + "message", reason)); + return false; + } + sourceGeneration = ragBranchIndexRepository + .findActiveGenerationCoordinates(project.getId(), branchName) + .orElse(null); + } - ragIndexTrackingService.markUpdatingStarted(project, branchName, commitHash); + if (sourceGeneration != null + && sourceGeneration.getRevision().equals(commitHash)) { + String message = String.format( + "RAG generation for branch '%s' already represents commit %s", + branchName, commitHash); + if (tracksProjectStatus) { + // Publication is authoritative. Repair a project + // checkpoint left stale by a crash in one atomic + // terminal transition, without creating an + // ownerless UPDATING window. + ragIndexTrackingService.preparePublishedGenerationForUpdate( + project, branchName, commitHash, + sourceGeneration.getFileCount(), + sourceGeneration.getChunkCount()); + } + emitEvent(eventConsumer, Map.of( + "type", "info", + "state", "rag_complete", + "message", message)); + return true; + } + + if (branchIndexBuildAdmissionService == null) { + throw new IllegalStateException( + "Exact RAG build admission service is unavailable"); + } + VcsRepoBinding exactBinding = project.getVcsRepoBinding(); + if (exactBinding == null || exactBinding.getVcsConnection() == null + || exactBinding.getExternalNamespace() == null + || exactBinding.getExternalNamespace().isBlank() + || exactBinding.getExternalRepoSlug() == null + || exactBinding.getExternalRepoSlug().isBlank()) { + throw new IllegalStateException( + "Project has no complete VcsRepoBinding configured"); + } + if (project.getConfiguration() == null + || project.getConfiguration().ragConfig() == null) { + throw new IllegalStateException( + "Project has no RAG configuration"); + } + admittedBuild = branchIndexBuildAdmissionService.admit( + project, + branchName, + commitHash, + exactGenerationKind, + JobTriggerSource.WEBHOOK, + ragLockKey.get(), + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + job = admittedBuild.job(); + analysisJobService.info(job, "rag_init", + String.format( + "Starting exact RAG generation rebuild for branch '%s' (commit: %s)", + branchName, commitHash)); + } + + emitEvent(eventConsumer, exactGenerationMode + ? Map.of( + "type", "status", + "state", "rag_update", + "message", String.format( + "Building exact RAG snapshot for branch '%s' at commit %s", + branchName, commitHash)) + : Map.of( + "type", "status", + "state", "rag_update", + "message", "Updating RAG index with " + + (addedOrModifiedSize + deletedFiles.size()) + + " changed files")); + + if (tracksProjectStatus && !exactGenerationMode) { + ragIndexTrackingService.markUpdatingStarted( + project, branchName, commitHash, job != null ? job.getId() : null); + } log.info("Performing RAG incremental update for project={}, branch={}, commit={}", project.getId(), branchName, commitHash); @@ -262,99 +512,55 @@ public boolean triggerIncrementalUpdate( String workspaceSlug = vcsRepoBinding.getExternalNamespace(); String repoSlug = vcsRepoBinding.getExternalRepoSlug(); - RagBranchIndexGeneration sourceGeneration = null; - RagBranchIndexRegistryService.BuildRegistration branchBuild = null; + Map result; if (exactGenerationMode) { - sourceGeneration = initialSourceGeneration; - if (sourceGeneration != null) { - branchBuild = branchIndexRegistryService.registerBuild( - project, - branchName, - exactGenerationKind, - sourceGeneration.getRevision(), - commitHash, - sourceGeneration.getRepresentationFingerprint()); - branchIndexRegistryService.startBuild( - branchBuild.operation().getId(), - job != null ? job.getId() : null); + if (branchGenerationBuildService == null) { + throw new IllegalStateException( + "Exact RAG generation build service is unavailable"); } - } - - Map result; - try { - if (fullExactSnapshotRequired) { - var ragConfig = project.getConfiguration().ragConfig(); - result = branchGenerationBuildService.build( + var ragConfig = project.getConfiguration().ragConfig(); + // A same-revision active generation returned above. Any + // remaining mismatch must create and activate a fresh + // snapshot, including A -> B -> A where an older A + // generation already succeeded but is no longer active. + exactExecutionStarted = true; + result = branchGenerationBuildService.execute( + project, + vcsConnection, + workspaceSlug, + repoSlug, + branchName, + commitHash, + exactGenerationKind, + ragConfig.includePatterns(), + ragConfig.excludePatterns(), + admittedBuild.preparedBuild(), + null); + exactPublicationCompleted = true; + } else { + result = incrementalRagUpdateService.performIncrementalUpdate( project, vcsConnection, workspaceSlug, repoSlug, branchName, commitHash, - exactGenerationKind, - ragConfig.includePatterns(), - ragConfig.excludePatterns(), - job != null ? job.getId() : null); - } else if (exactGenerationMode) { - result = incrementalRagUpdateService.performIncrementalUpdate( - project, - vcsConnection, - workspaceSlug, - repoSlug, - branchName, - commitHash, - addedFiles, - modifiedFiles, - deletedFiles, - sourceGeneration.getRevision(), - sourceGeneration.getCollectionName(), - branchBuild.generation().getCollectionName(), - false, - false); - } else { - result = incrementalRagUpdateService.performIncrementalUpdate( - project, - vcsConnection, - workspaceSlug, - repoSlug, - branchName, - commitHash, - addedFiles, - modifiedFiles, - deletedFiles); - } - if (exactGenerationMode && !fullExactSnapshotRequired) { - Object digest = result.get("generation_manifest_sha256"); - if (!(digest instanceof String manifestDigest) - || manifestDigest.isBlank()) { - throw new IllegalStateException( - "Advanced RAG generation has no manifest digest"); - } - RagBranchIndexGeneration published = branchIndexRegistryService.publish( - branchBuild.operation().getId(), - manifestDigest, - ((Number) result.getOrDefault("document_count", 0)).intValue(), - ((Number) result.getOrDefault("chunk_count", 0)).intValue()); - publishReadableAliasesIfActive( - project, branchName, commitHash, - branchBuild.generation().getCollectionName(), - published, publishBranchAlias, - publishLegacyProjectAlias); - } - } catch (Exception generationFailure) { - if (branchBuild != null) { - branchIndexRegistryService.fail( - branchBuild.operation().getId(), - generationFailure.getMessage() != null - ? generationFailure.getMessage() - : generationFailure.getClass().getSimpleName()); - } - throw generationFailure; + addedFiles, + modifiedFiles, + deletedFiles); + } + + if (!exactGenerationMode) { + confirmLegacyOwnership(legacyJobLease, legacyLockLease); } - int filesUpdated = (Integer) result.getOrDefault("updatedFiles", 0); - int filesDeleted = (Integer) result.getOrDefault("deletedFiles", 0); - int filesSkipped = (Integer) result.getOrDefault("skippedFiles", 0); + int documentCount = result.get("document_count") instanceof Number number + ? number.intValue() : 0; + int filesUpdated = exactGenerationMode + ? documentCount + : ((Number) result.getOrDefault("updatedFiles", 0)).intValue(); + int filesDeleted = ((Number) result.getOrDefault("deletedFiles", 0)).intValue(); + int filesSkipped = ((Number) result.getOrDefault("skippedFiles", 0)).intValue(); Integer newlyAddedFilesCount = (Integer) result.get("addedFilesCount"); Integer chunkCount = null; @@ -362,63 +568,160 @@ public boolean triggerIncrementalUpdate( chunkCount = ((Number) result.get("chunk_count")).intValue(); } - ragIndexTrackingService.markUpdatingCompleted( - project, - branchName, - commitHash, - newlyAddedFilesCount != null ? newlyAddedFilesCount : 0, - filesDeleted, - chunkCount, - branchName.equals(getBaseBranch(project))); + if (tracksProjectStatus && exactGenerationMode) { + ragIndexTrackingService.reconcilePublishedGeneration( + project, + branchName, + commitHash, + documentCount, + chunkCount, + job != null ? job.getId() : null); + } - // Track branch index for deleted files - trackBranchIndex(project, branchName, commitHash, deletedFiles); + if (!exactGenerationMode) { + if (legacyRagUpdateCompletionService == null + || job == null + || job.getId() == null + || !legacyRagUpdateCompletionService.complete( + project, + branchName, + commitHash, + job.getId(), + legacyJobLease.validAfter(), + tracksProjectStatus, + newlyAddedFilesCount != null + ? newlyAddedFilesCount : 0, + filesDeleted, + chunkCount, + deletedFiles)) { + throw new LegacyRagOwnershipLostException( + "Legacy RAG update lost durable ownership before publication"); + } + } - eventConsumer.accept(Map.of( + String completionMessage = exactGenerationMode + ? String.format( + "Exact RAG snapshot activated: %d documents, %d chunks", + documentCount, chunkCount != null ? chunkCount : 0) + : String.format( + "RAG index updated: %d files updated, %d deleted, %d non-text files skipped", + filesUpdated, filesDeleted, filesSkipped); + emitEvent(eventConsumer, Map.of( "type", "status", "state", "rag_complete", - "message", - String.format( - "RAG index updated: %d files updated, %d deleted, %d non-text files skipped", - filesUpdated, filesDeleted, filesSkipped))); + "message", completionMessage)); log.info("RAG incremental update completed for project={}: {} files updated, {} deleted, " + "{} non-text files skipped", project.getId(), filesUpdated, filesDeleted, filesSkipped); - analysisJobService.info(job, "rag_complete", - String.format( - "RAG incremental update completed: %d files updated, %d deleted, " - + "%d non-text files skipped", - filesUpdated, filesDeleted, filesSkipped)); - analysisJobService.completeJob(job, null); + if (exactGenerationMode) { + analysisJobService.info(job, "rag_complete", completionMessage); + analysisJobService.completeJob(job, null); + } else { + try { + analysisJobService.recordExternallyCompletedJob( + job, "rag_complete", completionMessage); + } catch (Exception notificationFailure) { + // The job and checkpoints already committed atomically. + // A local observer failure cannot reverse that outcome. + log.warn( + "Could not announce completed legacy RAG job {}: {}", + job != null ? job.getId() : null, + notificationFailure.getMessage()); + } + } return true; } catch (Exception e) { - // Use markIncrementalUpdateFailed (keeps status INDEXED, increments failure counter) - // NOT markIndexingFailed which would set status to FAILED and permanently block - // all future incremental updates even though the base index is still valid. - ragIndexTrackingService.markIncrementalUpdateFailed(project, e.getMessage()); - log.error("RAG incremental update failed", e); - if (job != null) { - analysisJobService.error(job, "rag_error", "RAG incremental update failed: " + e.getMessage()); - analysisJobService.failJob(job, "RAG incremental update failed: " + e.getMessage()); + // execute owns operation failure after it starts. Fail the + // admitted operation here only when a local hand-off step + // between admission and execute raised first. + if (admittedBuild != null && !exactExecutionStarted) { + try { + branchIndexBuildAdmissionService.abortOperation( + admittedBuild, + e.getMessage() != null + ? e.getMessage() + : e.getClass().getSimpleName()); + } catch (Exception abortFailure) { + log.error("Could not terminalize admitted exact RAG build {}", + admittedBuild.preparedBuild().operationId(), abortFailure); + } } - eventConsumer.accept(Map.of( + boolean legacyOwnershipLost = + e instanceof LegacyRagOwnershipLostException + || e instanceof LegacyRagUpdateCompletionService + .LegacyRagCompletionConflictException; + if (exactPublicationCompleted) { + // execute returns only after the registry operation is + // SUCCEEDED and the generation is active. A later status, + // job-log, or job-completion failure is projection drift, + // not a failed build. Preserve its RUNNING ownership for + // RagIndexOperationRecoveryService to finish idempotently. + log.error( + "Exact RAG generation was published but its projections could not be finalized: " + + "project={}, branch={}, job={}", + project.getId(), branchName, + job != null ? job.getId() : null, e); + } else if (legacyOwnershipLost) { + // Recovery may already own the job/status terminal + // transition. Never overwrite that durable outcome from a + // producer that can no longer prove ownership. + log.info("Legacy RAG update stopped after ownership loss: {}", + e.getMessage()); + } else { + if (tracksProjectStatus) { + if (exactGenerationMode && admittedBuild != null + && admittedBuild.statusAdmission() + == BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING) { + ragIndexTrackingService.markIndexingFailed( + project, e.getMessage(), + job != null ? job.getId() : null); + } else { + // Incremental failure preserves the preceding + // completed checkpoint and usable index. + ragIndexTrackingService.markIncrementalUpdateFailed( + project, e.getMessage(), + job != null ? job.getId() : null); + } + } + log.error("RAG incremental update failed", e); + if (job != null) { + analysisJobService.failJob( + job, "RAG incremental update failed: " + e.getMessage()); + } + } + emitEvent(eventConsumer, Map.of( "type", "warning", "state", "rag_error", "message", "RAG incremental update failed: " + e.getMessage())); return false; } finally { - analysisLockService.releaseLock(ragLockKey.get()); + if (legacyJobLease != null) { + legacyJobLease.close(); + } + if (legacyLockLease != null) { + legacyLockLease.close(); + } + try { + analysisLockService.releaseLock(ragLockKey.get()); + } catch (RuntimeException releaseFailure) { + // Publication/job terminalization is authoritative. A lock + // cleanup outage must not reverse a completed operation; + // exact recovery or the lock TTL will remove the row. + log.info( + "RAG indexing lock could not be released after processing; " + + "leaving it to recovery/expiry: project={}, branch={}, detail={}", + project.getId(), branchName, releaseFailure.getMessage()); + } } } catch (Exception e) { log.warn("RAG incremental update failed (non-critical): {}", e.getMessage()); if (job != null) { - analysisJobService.error(job, "rag_error", - "RAG incremental update failed (non-critical): " + e.getMessage()); - analysisJobService.failJob(job, e.getMessage()); + analysisJobService.failJob( + job, "RAG incremental update failed: " + e.getMessage()); } - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "state", "rag_error", "message", "RAG incremental update failed: " + e.getMessage())); @@ -426,27 +729,28 @@ public boolean triggerIncrementalUpdate( } } - private void publishReadableAliasesIfActive( - Project project, - String branch, - String revision, - String collectionTarget, - RagBranchIndexGeneration published, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias) { - if (published == null - || published.getStatus() != RagBranchIndexGenerationStatus.ACTIVE - || !publishBranchAlias) { - return; + private static void requireLegacyOwnership( + LegacyRagJobLeaseService.JobLease jobLease, + AnalysisLockService.LockLease lockLease) { + if (jobLease.isOwnershipLost() || lockLease.isOwnershipLost()) { + throw new LegacyRagOwnershipLostException( + "Legacy RAG update lost durable ownership before remote mutation"); } - try { - ragPipelineClient.publishGenerationAliases( - project.getWorkspace().getName(), project.getNamespace(), - branch, revision, collectionTarget, - true, publishLegacyProjectAlias); - } catch (IOException aliasFailure) { - log.warn("Readable alias publication failed for active RAG generation {}: {}", - published.getId(), aliasFailure.getMessage()); + } + + private static void confirmLegacyOwnership( + LegacyRagJobLeaseService.JobLease jobLease, + AnalysisLockService.LockLease lockLease) { + if (!jobLease.confirmOwnership() || !lockLease.confirmOwnership()) { + throw new LegacyRagOwnershipLostException( + "Legacy RAG update lost durable ownership before publication"); + } + } + + private static final class LegacyRagOwnershipLostException + extends IllegalStateException { + private LegacyRagOwnershipLostException(String message) { + super(message); } } @@ -559,7 +863,6 @@ public boolean isBranchIndexReady(Project project, String branchName) { } @Override - @Transactional public void createOrUpdateBranchIndex( Project project, String branchName, @@ -582,7 +885,7 @@ public boolean updateBranchIndex( return false; } - if (!isRagIndexReady(project)) { + if (!isRagIndexReady(project) && !usesExactGenerations(project)) { log.warn("Cannot update branch index - base RAG index not ready for project={}", project.getId()); return false; } @@ -591,7 +894,7 @@ public boolean updateBranchIndex( && !shouldHaveBranchIndex(project, targetBranch)) { log.info("Skipping branch index update for non-retained branch: project={}, branch={}", project.getId(), targetBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "state", "rag_skipped", "message", "Branch is not configured as a retained RAG branch")); @@ -620,6 +923,20 @@ public boolean updateBranchIndex( VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); String targetCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); + if (usesExactGenerations(project)) { + if (isExactGenerationCurrent( + project, targetBranch, targetCommit, false)) { + log.info("Exact branch generation already represents project={}, branch={}, commit={}", + project.getId(), targetBranch, targetCommit); + return true; + } + log.info("Binding exact branch update after acquiring its RAG lock: " + + "project={}, branch={}, target={}", + project.getId(), targetBranch, targetCommit); + return triggerIncrementalUpdate( + project, targetBranch, targetCommit, "", eventConsumer); + } + Optional completedBranchIndex = ragBranchIndexRepository .findByProjectIdAndBranchName(project.getId(), targetBranch) .filter(index -> index.getCommitHash() != null && !index.getCommitHash().isBlank()); @@ -643,7 +960,7 @@ public boolean updateBranchIndex( log.info("Seeding legacy branch index for project={}, branch={} (diff vs {})", project.getId(), targetBranch, baseBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "branch_index", "message", String.format("Calculating diff between '%s' and '%s'", baseBranch, targetBranch))); @@ -654,7 +971,7 @@ public boolean updateBranchIndex( if (rawDiff == null || rawDiff.isEmpty()) { log.info("No diff between {} and {} - branch has same content as base", baseBranch, targetBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "message", String.format("Branch '%s' has same content as '%s'", targetBranch, baseBranch))); if (usesExactGenerations(project)) { @@ -677,7 +994,7 @@ public boolean updateBranchIndex( } catch (Exception e) { log.error("Failed to update branch index for project={}, branch={}", project.getId(), targetBranch, e); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "error", "message", "Failed to update branch index: " + e.getMessage())); return false; @@ -702,7 +1019,7 @@ public boolean ensureBranchIndexForPrTarget( // Check if base index is ready if (!isRagIndexReady(project)) { log.warn("Cannot ensure branch index - base RAG index not ready for project={}", project.getId()); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "message", "Base RAG index not ready")); return false; @@ -759,7 +1076,7 @@ public boolean ensureBranchIndexForPrTarget( log.info("Fetching diff between base branch '{}' and target branch '{}' for project={}", baseBranch, targetBranch, project.getId()); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "branch_index", "message", String.format("Indexing branch '%s' (diff vs '%s')", targetBranch, baseBranch))); @@ -774,7 +1091,7 @@ public boolean ensureBranchIndexForPrTarget( if (rawDiff == null || rawDiff.isEmpty()) { log.info("No diff between '{}' and '{}' - branch has same content as base, using main index", baseBranch, targetBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "message", String.format("No changes between %s and %s - using main branch index", baseBranch, targetBranch))); @@ -794,7 +1111,7 @@ public boolean ensureBranchIndexForPrTarget( } catch (Exception e) { log.error("Failed to index branch data for project={}, branch={}: {}", project.getId(), targetBranch, e.getMessage(), e); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "state", "branch_error", "message", "Failed to index branch: " + e.getMessage())); @@ -807,24 +1124,31 @@ public boolean deleteBranchIndex( Project project, String branchName, Consumer> eventConsumer) { + return deleteBranchIndexWithOutcome(project, branchName, eventConsumer).successful(); + } + + private BranchIndexDeletionResult deleteBranchIndexWithOutcome( + Project project, + String branchName, + Consumer> eventConsumer) { if (!isRagEnabled(project)) { log.debug("RAG not enabled for project={}", project.getId()); - return false; + return new BranchIndexDeletionResult(false, false); } String baseBranch = getBaseBranch(project); if (branchName.equals(baseBranch)) { log.warn("Cannot delete main branch index for project={}", project.getId()); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "message", "Cannot delete main branch index")); - return false; + return new BranchIndexDeletionResult(false, false); } VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); if (vcsRepoBinding == null) { log.error("Project has no VcsRepoBinding configured"); - return false; + return new BranchIndexDeletionResult(false, false); } String workspaceSlug = vcsRepoBinding.getExternalNamespace(); @@ -833,12 +1157,13 @@ public boolean deleteBranchIndex( try { log.info("Deleting branch index for project={}, branch={}", project.getId(), branchName); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "branch_delete", "message", String.format("Deleting RAG index for branch '%s'", branchName))); boolean success; + boolean stopRemainingBranches = false; Optional trackedIndex = ragBranchIndexRepository .findByProjectIdAndBranchName(project.getId(), branchName); List generations = trackedIndex.isPresent() @@ -847,16 +1172,49 @@ public boolean deleteBranchIndex( trackedIndex.get().getId()) : List.of(); if (!generations.isEmpty()) { + generations = generations.stream() + .sorted(Comparator.comparing(generation -> + generation.getStatus() + == RagBranchIndexGenerationStatus.ACTIVE)) + .toList(); success = true; for (RagBranchIndexGeneration generation : generations) { - success &= ragPipelineClient.deleteBranch( - project.getWorkspace().getName(), project.getNamespace(), - branchName, generation.getCollectionName()); + if (generation.getStatus() == RagBranchIndexGenerationStatus.ACTIVE + && !success) { + // A failed older-target deletion must not leave the + // registry pointing at a destroyed active target. + break; + } + RagPipelineClient.BranchDeletionOutcome outcome = + ragPipelineClient.deleteBranchWithOutcome( + project.getWorkspace().getName(), project.getNamespace(), + branchName, generation.getCollectionName(), + generation.getRevision(), generation.getManifestDigest()); + if (!outcome.successful()) { + success = false; + if (isRagDisabled(outcome)) { + return new BranchIndexDeletionResult(false, true); + } + logBranchCleanupFailure(project, branchName, outcome); + if (outcome.shouldStopRemainingTargets()) { + stopRemainingBranches = true; + break; + } + } } } else { // Backward-compatible cleanup for the legacy shared collection. - success = ragPipelineClient.deleteBranch( - workspaceSlug, projectSlug, branchName); + RagPipelineClient.BranchDeletionOutcome outcome = + ragPipelineClient.deleteBranchWithOutcome( + workspaceSlug, projectSlug, branchName, null); + success = outcome.successful(); + if (!success) { + if (isRagDisabled(outcome)) { + return new BranchIndexDeletionResult(false, true); + } + logBranchCleanupFailure(project, branchName, outcome); + stopRemainingBranches = outcome.shouldStopRemainingTargets(); + } } if (success) { @@ -864,26 +1222,47 @@ public boolean deleteBranchIndex( ragBranchIndexRepository.deleteByProjectIdAndBranchName(project.getId(), branchName); log.info("Successfully deleted branch index for project={}, branch={}", project.getId(), branchName); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "success", "message", String.format("Deleted RAG index for branch '%s'", branchName))); - return true; + return new BranchIndexDeletionResult(true, false); } else { - log.warn("Failed to delete branch index from RAG pipeline for project={}, branch={}", - project.getId(), branchName); - return false; + return new BranchIndexDeletionResult(false, stopRemainingBranches); } } catch (Exception e) { log.error("Failed to delete branch index for project={}, branch={}", project.getId(), branchName, e); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "error", "message", "Failed to delete branch index: " + e.getMessage())); - return false; + return new BranchIndexDeletionResult(false, true); } } + private record BranchIndexDeletionResult( + boolean successful, + boolean shouldStopRemainingBranches) { + } + + private static void logBranchCleanupFailure( + Project project, + String branchName, + RagPipelineClient.BranchDeletionOutcome outcome) { + log.warn("Failed to delete branch RAG generation for project={}, branch={}, target={}: " + + "status={} detail={}", + project.getId(), branchName, outcome.targetLabel(), + outcome.statusCode() != null ? outcome.statusCode() : outcome.failure(), + outcome.detail()); + } + + private static boolean isRagDisabled( + RagPipelineClient.BranchDeletionOutcome outcome) { + return outcome.statusCode() == null + && outcome.failure() == RagPipelineClient.BranchDeletionFailure.TARGET + && "RAG disabled".equals(outcome.detail()); + } + @Override public Map cleanupStaleBranches( Project project, @@ -931,7 +1310,7 @@ public Map cleanupStaleBranches( log.info("Cleaning up {} stale branches for project={}: {}", staleBranches.size(), project.getId(), staleBranches); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "cleanup", "message", String.format("Cleaning up %d stale branches", staleBranches.size()))); @@ -941,11 +1320,20 @@ public Map cleanupStaleBranches( for (String branch : staleBranches) { try { - boolean success = deleteBranchIndex(project, branch, eventConsumer); - if (success) { + BranchIndexDeletionResult deletion = + deleteBranchIndexWithOutcome(project, branch, eventConsumer); + if (deletion.successful()) { deletedBranches.add(branch); } else { failedBranches.add(branch); + if (deletion.shouldStopRemainingBranches()) { + log.info("Stopping stale branch cleanup after a service-wide deletion " + + "failure; {} remaining branch(es) will retry next run", + staleBranches.size() + - deletedBranches.size() + - failedBranches.size()); + break; + } } } catch (Exception e) { log.warn("Failed to delete stale branch {} for project={}: {}", @@ -957,7 +1345,7 @@ public Map cleanupStaleBranches( log.info("Cleanup complete for project={}: deleted={}, failed={}", project.getId(), deletedBranches.size(), failedBranches.size()); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "success", "message", String.format("Cleaned up %d stale branches", deletedBranches.size()))); @@ -1017,7 +1405,7 @@ public boolean ensureRagIndexUpToDate( && !shouldCreateTransientBranchIndex(project, targetBranch)) { log.info("Skipping RAG preparation for non-indexed PR target: project={}, branch={}", project.getId(), targetBranch); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "info", "state", "rag_skipped", "message", "PR target is not configured for retained or temporary RAG indexing")); @@ -1039,7 +1427,7 @@ public boolean ensureRagIndexUpToDate( } catch (Exception e) { log.error("Failed to ensure RAG index up-to-date for project={}, targetBranch={}", project.getId(), targetBranch, e); - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "warning", "state", "rag_error", "message", "Failed to update RAG index: " + e.getMessage())); @@ -1058,7 +1446,7 @@ private boolean ensureMainIndexUpToDate( String workspaceSlug, String repoSlug, Consumer> eventConsumer) throws IOException { - if (!isRagIndexReady(project)) { + if (!isRagIndexReady(project) && !usesExactGenerations(project)) { log.debug("Main RAG index not ready for project={}", project.getId()); return false; } @@ -1066,6 +1454,21 @@ private boolean ensureMainIndexUpToDate( // Get current commit on branch String currentCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, branchName); + if (usesExactGenerations(project)) { + if (isExactGenerationCurrent( + project, branchName, currentCommit, true)) { + log.debug("Exact main RAG generation and project checkpoint are up-to-date " + + "for project={}, commit={}", + project.getId(), currentCommit); + return true; + } + log.info("Exact main RAG generation requires reconciliation for project={}, " + + "branch={}, target={}", + project.getId(), branchName, currentCommit); + return triggerIncrementalUpdate( + project, branchName, currentCommit, "", eventConsumer); + } + // Get indexed commit from tracking service Optional indexStatus = ragIndexTrackingService.getIndexStatus(project); if (indexStatus.isEmpty()) { @@ -1075,17 +1478,6 @@ private boolean ensureMainIndexUpToDate( String indexedCommit = indexStatus.get().getIndexedCommitHash(); - if (usesExactGenerations(project) - && ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), branchName) - .map(RagBranchIndex::getActiveGeneration) - .isEmpty()) { - log.info("Creating first exact primary generation for project={}, branch={}, commit={}", - project.getId(), branchName, currentCommit); - return triggerIncrementalUpdate( - project, branchName, currentCommit, "", eventConsumer); - } - // If commits match, index is up to date if (currentCommit.equals(indexedCommit)) { log.debug("Main RAG index is up-to-date for project={}, commit={}", project.getId(), currentCommit); @@ -1104,7 +1496,7 @@ private boolean ensureMainIndexUpToDate( return true; } - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "rag_update", "message", String.format("Updating RAG index from %s to %s", @@ -1134,6 +1526,20 @@ private boolean ensureBranchIndexUpToDate( String currentCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); log.info("Current commit on branch '{}': {}", targetBranch, currentCommit); + if (usesExactGenerations(project)) { + if (isExactGenerationCurrent( + project, targetBranch, currentCommit, false)) { + log.info("Exact branch generation is up-to-date for project={}, branch={}, commit={}", + project.getId(), targetBranch, currentCommit); + return true; + } + log.info("Exact branch generation requires reconciliation for project={}, " + + "branch={}, target={}", + project.getId(), targetBranch, currentCommit); + return triggerIncrementalUpdate( + project, targetBranch, currentCommit, "", eventConsumer); + } + // Check if we have branch index tracking Optional branchIndexOpt = ragBranchIndexRepository .findByProjectIdAndBranchName(project.getId(), targetBranch); @@ -1155,11 +1561,6 @@ private boolean ensureBranchIndexUpToDate( log.info("Existing RagBranchIndex for project={}, branch={}: indexedCommit={}", project.getId(), targetBranch, indexedCommit); - if (usesExactGenerations(project) && branchIndex.getActiveGeneration() == null) { - return triggerIncrementalUpdate( - project, targetBranch, currentCommit, "", eventConsumer); - } - // If commits match, index is up to date if (currentCommit.equals(indexedCommit)) { log.info("Branch index is up-to-date for project={}, branch={}, commit={}", @@ -1189,7 +1590,7 @@ private boolean ensureBranchIndexUpToDate( return true; } - eventConsumer.accept(Map.of( + emitEvent(eventConsumer, Map.of( "type", "status", "state", "branch_update", "message", @@ -1202,6 +1603,36 @@ private boolean ensureBranchIndexUpToDate( project, targetBranch, currentCommit, rawDiff, eventConsumer); } + /** + * A cheap exact-generation readiness hint that is safe against transient + * cleanup. The atomic touch must win before the scalar projection is read; + * otherwise the locked trigger owns reconciliation. Primary callers also + * require the independently persisted project checkpoint to be current so + * a publication/status crash is repaired by the trigger's locked no-op. + */ + private boolean isExactGenerationCurrent( + Project project, + String branchName, + String targetRevision, + boolean requireProjectCheckpoint) { + if (ragBranchIndexRepository.markAccessedIfUnclaimed( + project.getId(), branchName, OffsetDateTime.now()) == 0) { + return false; + } + boolean generationCurrent = ragBranchIndexRepository + .findActiveGenerationCoordinates(project.getId(), branchName) + .map(RagBranchIndexRepository.ActiveGenerationCoordinates::getRevision) + .filter(targetRevision::equals) + .isPresent(); + if (!generationCurrent || !requireProjectCheckpoint) { + return generationCurrent; + } + return ragIndexTrackingService.getIndexStatus(project) + .map(RagIndexStatus::getIndexedCommitHash) + .filter(targetRevision::equals) + .isPresent(); + } + private boolean usesExactGenerations(Project project) { return branchIndexRegistryService != null && branchGenerationBuildService != null @@ -1220,4 +1651,19 @@ private RagBranchIndexKind indexKind(Project project, String branchName) { : RagBranchIndexKind.TRANSIENT; } + /** Observer delivery is never part of the durable RAG operation outcome. */ + private static void emitEvent( + Consumer> eventConsumer, + Map event) { + if (eventConsumer == null) { + return; + } + try { + eventConsumer.accept(event); + } catch (RuntimeException observerFailure) { + log.debug("RAG progress observer rejected event state={}: {}", + event.get("state"), observerFailure.getMessage()); + } + } + } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java index 92be1916..c3496ff6 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java @@ -200,7 +200,8 @@ private Map performIndexing( return Map.of("status", "error", "message", errorMsg); } - ragIndexTrackingService.markIndexingStarted(project, branch, commitHash); + ragIndexTrackingService.markIndexingStarted( + project, branch, commitHash, job != null ? job.getId() : null); String downloadMsg = "Downloading repository archive..."; messageConsumer.accept(Map.of( @@ -316,7 +317,8 @@ private Map performIndexing( } catch (Exception e) { log.error("RAG indexing failed for project {}", project.getName(), e); - ragIndexTrackingService.markIndexingFailed(project, e.getMessage()); + ragIndexTrackingService.markIndexingFailed( + project, e.getMessage(), job != null ? job.getId() : null); analysisLockService.releaseLock(lockKey); messageConsumer.accept(Map.of( @@ -473,7 +475,8 @@ public void pollRagIndexingJobAsync( + jobId; break; } - ragIndexTrackingService.markIndexingHeartbeat(project); + ragIndexTrackingService.markIndexingHeartbeat( + project, job != null ? job.getId() : null); continue; } errorMessage = "RAG indexing produced no worker heartbeat for " @@ -500,7 +503,8 @@ public void pollRagIndexingJobAsync( // Redis activity keeps the lease alive; persist the same // activity on the existing status row so long-running, // healthy indexes do not look stalled to operators. - ragIndexTrackingService.markIndexingHeartbeat(project); + ragIndexTrackingService.markIndexingHeartbeat( + project, job != null ? job.getId() : null); String state = String.valueOf(event.getOrDefault("stage", event.getOrDefault("state", "indexing"))); @@ -551,7 +555,13 @@ public void pollRagIndexingJobAsync( // Update Job and Project Tracking if (success) { - ragIndexTrackingService.markIndexingCompleted(project, branch, commitHash, filesIndexed, chunkCount); + ragIndexTrackingService.markIndexingCompleted( + project, + branch, + commitHash, + filesIndexed, + chunkCount, + job != null ? job.getId() : null); String completeMsg = "RAG indexing completed successfully. Files indexed: " + (filesIndexed != null ? filesIndexed : 0); if (job != null) { @@ -561,8 +571,10 @@ public void pollRagIndexingJobAsync( log.info("RAG indexing completed for project {} branch {}: {} files", project.getName(), branch, filesIndexed); } else { - ragIndexTrackingService.markIndexingFailed(project, - errorMessage != null ? errorMessage : "Unknown Error"); + ragIndexTrackingService.markIndexingFailed( + project, + errorMessage != null ? errorMessage : "Unknown Error", + job != null ? job.getId() : null); if (job != null) { jobService.logToJob(job, JobLogLevel.ERROR, "error", "RAG indexing failed: " + errorMessage); jobService.failJob(job, "RAG indexing failed: " + errorMessage); diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java new file mode 100644 index 00000000..67779ef6 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java @@ -0,0 +1,155 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagIndexOperation; +import org.rostilos.codecrow.core.service.AnalysisJobService; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class BranchIndexBuildAdmissionServiceTest { + + @Mock private RagBranchIndexRegistryService registryService; + @Mock private AnalysisJobService jobService; + @Mock private RagIndexTrackingService trackingService; + + private BranchIndexBuildAdmissionService service; + private Project project; + private RagBranchIndexRegistryService.BuildRegistration registration; + + @BeforeEach + void setUp() { + service = new BranchIndexBuildAdmissionService( + registryService, jobService, trackingService); + project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + RagBranchIndex branchIndex = new RagBranchIndex( + project, "main", RagBranchIndexKind.PRIMARY); + branchIndex.setId(10L); + RagBranchIndexGeneration source = new RagBranchIndexGeneration( + branchIndex, "revision-a", "source-target", null, null, null); + source.setId(19L); + source.activate("source-manifest", 120, 240); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, "revision-b", "physical-target", source, null, null); + generation.setId(20L); + RagIndexOperation operation = new RagIndexOperation( + project, "main", null, "revision-b", "operation-key"); + operation.setId(30L); + operation.setGeneration(generation); + registration = new RagBranchIndexRegistryService.BuildRegistration( + branchIndex, generation, operation, false, "source-target"); + } + + @Test + void registersThenAtomicallyLinksAndStartsJobAndOperation() { + when(registryService.registerBuild( + eq(project), eq("main"), eq(RagBranchIndexKind.PRIMARY), + isNull(), eq("revision-b"), startsWith("exact-full-snapshot:automatic:"))) + .thenReturn(registration); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + when(jobService.createRagIndexJob( + project, false, JobTriggerSource.WEBHOOK, "main", "revision-b")) + .thenReturn(job); + + var admitted = service.admit( + project, "main", "revision-b", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "lock-owner-123", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + + assertThat(admitted.job()).isSameAs(job); + assertThat(admitted.preparedBuild().operationId()).isEqualTo(30L); + assertThat(admitted.preparedBuild().collectionTarget()) + .isEqualTo("physical-target"); + assertThat(admitted.preparedBuild().analysisLockKey()) + .isEqualTo("lock-owner-123"); + assertThat(admitted.preparedBuild().sourceCollectionTarget()) + .isEqualTo("source-target"); + assertThat(admitted.statusAdmission()).isEqualTo( + BranchIndexBuildAdmissionService.ProjectStatusAdmission.UPDATING); + InOrder order = inOrder(registryService, jobService, trackingService); + order.verify(registryService).registerBuild( + eq(project), eq("main"), eq(RagBranchIndexKind.PRIMARY), + isNull(), eq("revision-b"), startsWith("exact-full-snapshot:automatic:")); + order.verify(trackingService).preparePublishedGenerationForUpdate( + project, "main", "revision-a", 120, 240); + order.verify(jobService).createRagIndexJob( + project, false, JobTriggerSource.WEBHOOK, "main", "revision-b"); + order.verify(registryService).startBuild(30L, 77L, "lock-owner-123"); + order.verify(jobService).startJob(job); + order.verify(trackingService).markUpdatingStarted( + project, "main", "revision-b", 77L); + } + + @Test + void rejectsPreviouslyCommittedAdmissionWithoutCreatingAnotherJob() { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), anyString())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + registration.branchIndex(), registration.generation(), + registration.operation(), true)); + + assertThatThrownBy(() -> service.admit( + project, "main", "revision-b", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "same-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already admitted"); + + verifyNoInteractions(jobService); + verify(registryService, never()).startBuild(anyLong(), any(), anyString()); + } + + @Test + void initialPrimaryAdmissionStartsIndexingWithoutInventingASourceCheckpoint() { + RagBranchIndexGeneration initial = new RagBranchIndexGeneration( + registration.branchIndex(), "revision-first", "initial-target", + null, null, null); + initial.setId(21L); + RagIndexOperation initialOperation = new RagIndexOperation( + project, "main", null, "revision-first", "initial-operation"); + initialOperation.setId(31L); + initialOperation.setGeneration(initial); + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + eq("revision-first"), anyString())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + registration.branchIndex(), initial, initialOperation, false)); + Job job = mock(Job.class); + when(job.getId()).thenReturn(78L); + when(jobService.createRagIndexJob( + project, true, JobTriggerSource.WEBHOOK, "main", "revision-first")) + .thenReturn(job); + + var admitted = service.admit( + project, "main", "revision-first", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "initial-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + + assertThat(admitted.statusAdmission()).isEqualTo( + BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING); + verify(trackingService).markIndexingStarted( + project, "main", "revision-first", 78L); + verify(trackingService, never()).preparePublishedGenerationForUpdate( + any(), anyString(), anyString(), any(), any()); + verify(trackingService, never()).markUpdatingStarted( + any(), anyString(), anyString(), any()); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java index e9e14f20..7c4fad65 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java @@ -7,6 +7,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; @@ -34,6 +35,10 @@ class BranchIndexGenerationBuildServiceTest { @Mock private BranchArchiveService archiveService; @Mock private RagPipelineClient pipelineClient; @Mock private RagBranchIndexRegistryService registryService; + @Mock private RagIndexOperationHeartbeatService heartbeatService; + @Mock private RagIndexOperationHeartbeatService.HeartbeatScope heartbeatScope; + @Mock private AnalysisLockService analysisLockService; + @Mock private AnalysisLockService.LockLease lockLease; private BranchIndexGenerationBuildService service; private Project project; @@ -43,7 +48,8 @@ class BranchIndexGenerationBuildServiceTest { @BeforeEach void setUp() { service = new BranchIndexGenerationBuildService( - archiveService, pipelineClient, registryService); + archiveService, pipelineClient, registryService, heartbeatService); + lenient().when(heartbeatService.start(anyLong())).thenReturn(heartbeatScope); project = new Project(); ReflectionTestUtils.setField(project, "id", 42L); RagBranchIndex branchIndex = new RagBranchIndex( @@ -65,12 +71,13 @@ void buildsPinnedSnapshotPublishesManifestAndRemovesTemporaryTree() throws Excep project, "develop", RagBranchIndexKind.DURABLE, null, "develop-400", null)) .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( - generation.getBranchIndex(), generation, operation, false)); + generation.getBranchIndex(), generation, operation, false, + "opaque-generation-source")); when(pipelineClient.indexRepository( anyString(), eq("workspace"), eq("namespace"), eq("develop"), eq("develop-400"), eq(List.of("src/**")), eq(List.of("vendor/**")), eq("opaque-generation-target"), - eq(false), eq(false))) + eq(false), eq(false), eq("opaque-generation-source"))) .thenReturn(Map.of( "generation_manifest_sha256", "manifest-400", "document_count", 231, @@ -92,11 +99,18 @@ project, new VcsConnection(), "provider-workspace", "repo", assertThat(result).containsEntry( "generation_manifest_sha256", "manifest-400"); - verify(registryService).startBuild(30L, 77L); + verify(registryService).startBuild(30L, 77L, null); + verify(heartbeatService).start(30L); + verify(heartbeatScope).close(); verify(registryService).publish(30L, "manifest-400", 231, 400); + verify(pipelineClient).indexRepository( + anyString(), eq("workspace"), eq("namespace"), + eq("develop"), eq("develop-400"), eq(List.of("src/**")), + eq(List.of("vendor/**")), eq("opaque-generation-target"), + eq(false), eq(false), eq("opaque-generation-source")); verify(pipelineClient).publishGenerationAliases( "workspace", "namespace", "develop", "develop-400", - "opaque-generation-target", true, false); + "opaque-generation-target", "manifest-400", true, false); ArgumentCaptor snapshot = ArgumentCaptor.forClass(Path.class); verify(archiveService).downloadAndExtractSnapshotToDirectory( any(), eq("provider-workspace"), eq("repo"), @@ -104,6 +118,44 @@ project, new VcsConnection(), "provider-workspace", "repo", assertThat(Files.exists(snapshot.getValue())).isFalse(); } + @Test + void runtimeAliasFailureCannotReclassifyAPublishedGenerationAsFailed() + throws Exception { + when(pipelineClient.indexRepository( + anyString(), eq("workspace"), eq("namespace"), + eq("develop"), eq("develop-400"), eq(List.of()), + eq(List.of()), eq("opaque-generation-target"), + eq(false), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400)); + when(registryService.publish(30L, "manifest-400", 231, 400)) + .thenAnswer(ignored -> { + generation.activate("manifest-400", 231, 400); + return generation; + }); + doThrow(new IllegalStateException("observer alias adapter failed")) + .when(pipelineClient).publishGenerationAliases( + anyString(), anyString(), anyString(), anyString(), + anyString(), anyString(), anyBoolean(), anyBoolean()); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 30L, "opaque-generation-target", false, null, null); + + Map result = service.execute( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), prepared, null); + + assertThat(result).containsEntry("generation_manifest_sha256", "manifest-400"); + verify(registryService).publish(30L, "manifest-400", 231, 400); + verify(registryService, never()).fail(anyLong(), anyString()); + } + @Test void missingManifestFailsOperationAndDoesNotPublish() throws Exception { when(registryService.registerBuild(any(), anyString(), any(), isNull(), @@ -128,6 +180,8 @@ project, new VcsConnection(), "provider-workspace", "repo", verify(registryService).fail(30L, "RAG full branch generation has no manifest digest"); + verify(heartbeatService).start(30L); + verify(heartbeatScope).close(); verify(registryService, never()).publish(anyLong(), anyString(), anyInt(), anyInt()); } @@ -155,13 +209,15 @@ project, new VcsConnection(), "provider-workspace", "repo", void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws Exception { when(registryService.registerBuild( eq(project), eq("develop"), eq(RagBranchIndexKind.DURABLE), - isNull(), eq("develop-400"), eq("operator-refresh:77"))) + isNull(), eq("develop-400"), startsWith("full-snapshot:job:77:"))) .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( - generation.getBranchIndex(), generation, operation, false)); + generation.getBranchIndex(), generation, operation, false, + "opaque-generation-source")); when(pipelineClient.indexRepository( anyString(), anyString(), anyString(), eq("develop"), eq("develop-400"), anyList(), anyList(), eq("opaque-generation-target"), - eq(false), eq(false), any())) + eq(false), eq(false), eq(true), eq("opaque-generation-source"), + any(Runnable.class), any())) .thenReturn(Map.of( "generation_manifest_sha256", "fresh-manifest", "document_count", 231, @@ -183,9 +239,83 @@ void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws verify(archiveService).downloadAndExtractSnapshotToDirectory( any(), eq("provider-workspace"), eq("repo"), eq("develop-400"), isNull(), any()); verify(registryService).publish(30L, "fresh-manifest", 231, 400); + verify(pipelineClient).indexRepository( + anyString(), eq("workspace"), eq("namespace"), eq("develop"), + eq("develop-400"), anyList(), anyList(), + eq("opaque-generation-target"), eq(false), eq(false), eq(true), + eq("opaque-generation-source"), any(Runnable.class), any()); verify(pipelineClient).publishGenerationAliases( "workspace", "namespace", "develop", "develop-400", - "opaque-generation-target", true, false); + "opaque-generation-target", "fresh-manifest", true, false); + } + + @Test + void streamFailureBeforeOwnershipAdmissionLeavesCleanupWithJava() throws Exception { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), startsWith("full-snapshot:job:77:"))) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + ArgumentCaptor snapshot = ArgumentCaptor.forClass(String.class); + when(pipelineClient.indexRepository( + snapshot.capture(), anyString(), anyString(), anyString(), anyString(), + anyList(), anyList(), anyString(), anyBoolean(), anyBoolean(), + eq(true), any(Runnable.class), any())) + .thenThrow(new IOException("connection refused before admission")); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + assertThatThrownBy(() -> service.rebuild( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), 77L, ignored -> { })) + .isInstanceOf(IOException.class) + .hasMessageContaining("before admission"); + + assertThat(Files.exists(Path.of(snapshot.getValue()))).isFalse(); + verify(registryService).fail(30L, "connection refused before admission"); + } + + @Test + void streamAdmissionRelinquishesJavaSnapshotCleanup() throws Exception { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), startsWith("full-snapshot:job:77:"))) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + ArgumentCaptor snapshot = ArgumentCaptor.forClass(String.class); + when(pipelineClient.indexRepository( + snapshot.capture(), anyString(), anyString(), anyString(), anyString(), + anyList(), anyList(), anyString(), anyBoolean(), anyBoolean(), + eq(true), any(Runnable.class), any())) + .thenAnswer(invocation -> { + ((Runnable) invocation.getArgument(11)).run(); + throw new IOException("stream disconnected after admission"); + }); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + assertThatThrownBy(() -> service.rebuild( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), 77L, ignored -> { })) + .isInstanceOf(IOException.class) + .hasMessageContaining("after admission"); + + Path retainedSnapshot = Path.of(snapshot.getValue()); + assertThat(retainedSnapshot).exists(); + try (var paths = Files.walk(retainedSnapshot)) { + paths.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException cleanupFailure) { + throw new RuntimeException(cleanupFailure); + } + }); + } + verify(registryService).fail(30L, "stream disconnected after admission"); } @Test @@ -215,7 +345,49 @@ void staleCompletedGenerationDoesNotPublishReadableAliases() throws Exception { List.of(), List.of()); verify(pipelineClient, never()).publishGenerationAliases( - anyString(), anyString(), anyString(), anyString(), anyString(), + anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyBoolean(), anyBoolean()); } + + @Test + void admittedBuildMaintainsAndConfirmsBranchLockBeforePublication() throws Exception { + service = new BranchIndexGenerationBuildService( + archiveService, pipelineClient, registryService, heartbeatService, + analysisLockService, 12); + when(analysisLockService.maintainLockLease("rag-lock", 12)) + .thenReturn(lockLease); + when(lockLease.confirmOwnership()).thenReturn(true); + when(pipelineClient.indexRepository( + anyString(), anyString(), anyString(), eq("develop"), eq("develop-400"), + anyList(), anyList(), eq("opaque-generation-target"), + eq(false), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400)); + when(registryService.publish(30L, "manifest-400", 231, 400)) + .thenAnswer(ignored -> { + generation.activate("manifest-400", 231, 400); + return generation; + }); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 30L, "opaque-generation-target", false, null, "rag-lock"); + + service.execute( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), prepared, null); + + var order = inOrder(pipelineClient, lockLease, registryService); + order.verify(pipelineClient).indexRepository( + anyString(), anyString(), anyString(), anyString(), anyString(), + anyList(), anyList(), anyString(), anyBoolean(), anyBoolean()); + order.verify(lockLease).confirmOwnership(); + order.verify(registryService).publish(30L, "manifest-400", 231, 400); + verify(lockLease).close(); + } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java new file mode 100644 index 00000000..512d545a --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java @@ -0,0 +1,103 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; +import org.rostilos.codecrow.core.model.project.config.RagConfig; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; +import org.rostilos.codecrow.core.service.AnalysisJobService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.VcsClientProvider; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class BranchIndexMaintenanceServiceTest { + + @Test + void observerAndLockCleanupFailuresCannotReverseAPublishedBuild() throws Exception { + RagOperationsService ragOperations = mock(RagOperationsService.class); + VcsClientProvider vcsClients = mock(VcsClientProvider.class); + BranchIndexGenerationBuildService builds = mock( + BranchIndexGenerationBuildService.class); + BranchIndexBuildAdmissionService admissions = mock( + BranchIndexBuildAdmissionService.class); + RagIndexTrackingService tracking = mock(RagIndexTrackingService.class); + AnalysisLockService locks = mock(AnalysisLockService.class); + AnalysisJobService jobs = mock(AnalysisJobService.class); + BranchIndexMaintenanceService service = new BranchIndexMaintenanceService( + ragOperations, vcsClients, builds, admissions, tracking, locks, jobs, + Runnable::run, 1); + + Project project = mock(Project.class); + when(project.getId()).thenReturn(42L); + when(project.getConfiguration()).thenReturn(new ProjectConfig( + false, "main", null, + new RagConfig(true, "main", List.of(), List.of()))); + VcsRepoBinding binding = mock(VcsRepoBinding.class); + VcsConnection connection = new VcsConnection(); + when(project.getVcsRepoBinding()).thenReturn(binding); + when(binding.getVcsConnection()).thenReturn(connection); + when(binding.getExternalNamespace()).thenReturn("workspace"); + when(binding.getExternalRepoSlug()).thenReturn("repository"); + VcsClient vcs = mock(VcsClient.class); + when(vcsClients.getClient(connection)).thenReturn(vcs); + when(vcs.getLatestCommitHash("workspace", "repository", "main")) + .thenReturn("revision-a"); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(locks.acquireLock(eq(project), eq("main"), any(), eq("revision-a"), isNull())) + .thenReturn(Optional.of("rag-lock")); + + Job job = mock(Job.class); + when(job.getId()).thenReturn(91L); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 81L, "physical-generation", false, null, "rag-lock"); + when(admissions.admit( + eq(project), eq("main"), eq("revision-a"), + eq(RagBranchIndexKind.PRIMARY), any(), eq("rag-lock"), any())) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); + when(builds.execute( + eq(project), eq(connection), eq("workspace"), eq("repository"), + eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), + eq(List.of()), eq(List.of()), eq(prepared), any())) + .thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + Consumer> progress = invocation.getArgument(10); + progress.accept(Map.of("stage", "indexing", "message", "halfway")); + return Map.of("document_count", 12, "chunk_count", 34); + }); + doThrow(new IllegalStateException("lock database unavailable")) + .when(locks).releaseLock("rag-lock"); + + Map outcome = service.rebuild( + project, + "main", + false, + ignored -> { + throw new IllegalStateException("observer disconnected"); + }); + + assertThat(outcome.get("branches")).isEqualTo(List.of("main")); + assertThat(outcome.get("failedBranches")).isEqualTo(Map.of()); + verify(tracking).reconcilePublishedGeneration( + project, "main", "revision-a", 12, 34, 91L); + verify(jobs).completeJob(job, Map.of("branch", "main", "revision", "revision-a")); + verify(jobs, never()).failJob(any(), anyString()); + verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); + verify(locks).releaseLock("rag-lock"); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseServiceTest.java new file mode 100644 index 00000000..fef214ab --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseServiceTest.java @@ -0,0 +1,105 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.service.JobService; + +import java.time.OffsetDateTime; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +class LegacyRagJobLeaseServiceTest { + + @Test + void renewsSynchronouslyThenHeartbeatsUntilClosed() { + JobService jobs = mock(JobService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + ScheduledFuture scheduled = mock(ScheduledFuture.class); + when(jobs.renewLegacyRagJobLease(eq(91L), any(), any())) + .thenReturn(true); + doReturn(scheduled).when(executor).scheduleWithFixedDelay( + any(Runnable.class), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); + LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( + jobs, executor, 60, 15); + + LegacyRagJobLeaseService.JobLease lease = service.start(91L); + + var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(executor).scheduleWithFixedDelay( + heartbeat.capture(), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); + verify(jobs, times(1)).renewLegacyRagJobLease(eq(91L), any(), any()); + heartbeat.getValue().run(); + verify(jobs, times(2)).renewLegacyRagJobLease(eq(91L), any(), any()); + assertThat(lease.confirmOwnership()).isTrue(); + verify(jobs, times(3)).renewLegacyRagJobLease(eq(91L), any(), any()); + + lease.close(); + verify(scheduled).cancel(false); + } + + @Test + void refusesWorkWhenTheInitialLeaseCannotBeProven() { + JobService jobs = mock(JobService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + when(jobs.renewLegacyRagJobLease(eq(92L), any(), any())) + .thenReturn(false); + LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( + jobs, executor, 60, 15); + + LegacyRagJobLeaseService.JobLease lease = service.start(92L); + + assertThat(lease.isOwnershipLost()).isTrue(); + assertThat(lease.confirmOwnership()).isFalse(); + verifyNoInteractions(executor); + } + + @Test + void transientHeartbeatFailureDoesNotDiscardAStillValidLease() { + JobService jobs = mock(JobService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + ScheduledFuture scheduled = mock(ScheduledFuture.class); + when(jobs.renewLegacyRagJobLease(eq(93L), any(), any())) + .thenReturn(true) + .thenThrow(new IllegalStateException("temporary database outage")) + .thenReturn(true); + doReturn(scheduled).when(executor).scheduleWithFixedDelay( + any(Runnable.class), anyLong(), anyLong(), any()); + LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( + jobs, executor, 60, 15); + + LegacyRagJobLeaseService.JobLease lease = service.start(93L); + var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(executor).scheduleWithFixedDelay( + heartbeat.capture(), anyLong(), anyLong(), any()); + + heartbeat.getValue().run(); + assertThat(lease.isOwnershipLost()).isFalse(); + assertThat(lease.confirmOwnership()).isTrue(); + } + + @Test + void renewalUsesARealLeaseWindow() { + JobService jobs = mock(JobService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + when(jobs.renewLegacyRagJobLease(eq(94L), any(), any())) + .thenReturn(true); + doReturn(mock(ScheduledFuture.class)).when(executor).scheduleWithFixedDelay( + any(Runnable.class), anyLong(), anyLong(), any()); + LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( + jobs, executor, 60, 15); + + service.start(94L); + + var validAfter = org.mockito.ArgumentCaptor.forClass(OffsetDateTime.class); + var renewedAt = org.mockito.ArgumentCaptor.forClass(OffsetDateTime.class); + verify(jobs).renewLegacyRagJobLease( + eq(94L), validAfter.capture(), renewedAt.capture()); + assertThat(validAfter.getValue()).isEqualTo( + renewedAt.getValue().minusSeconds(60)); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryServiceTest.java new file mode 100644 index 00000000..89397664 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryServiceTest.java @@ -0,0 +1,151 @@ +package org.rostilos.codecrow.ragengine.branch; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.service.JobService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class LegacyRagJobRecoveryServiceTest { + private JobService jobs; + private RagIndexTrackingService tracking; + private LegacyRagJobRecoveryService recovery; + private JobRepository.LegacyRagJobRecoveryCoordinates coordinates; + + @BeforeEach + void setUp() { + jobs = mock(JobService.class); + tracking = mock(RagIndexTrackingService.class); + recovery = new LegacyRagJobRecoveryService( + jobs, tracking, 120, 100); + coordinates = mock( + JobRepository.LegacyRagJobRecoveryCoordinates.class); + when(coordinates.getJobId()).thenReturn(91L); + when(coordinates.getProjectId()).thenReturn(42L); + when(coordinates.getBranchName()).thenReturn("main"); + when(coordinates.getCommitHash()).thenReturn("commit-a"); + } + + @Test + void staleProducerIsAtomicallyFailedAndItsOwnedStatusIsRestored() { + Job job = mock(Job.class); + when(jobs.findAbandonedLegacyRagJobs(any(OffsetDateTime.class), eq(100))) + .thenReturn(List.of(coordinates)); + when(jobs.failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), anyString())) + .thenReturn(true); + when(jobs.findById(91L)).thenReturn(Optional.of(job)); + + recovery.failAbandonedJobs(); + + verify(jobs).failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), contains("stopped heartbeating")); + verify(jobs).recordExternallyFailedJob( + eq(job), eq("rag_recovery"), + contains("last completed checkpoint was preserved")); + verify(tracking).recoverAbandonedIncrementalUpdate( + eq(42L), eq(91L), contains("stopped heartbeating")); + } + + @Test + void heartbeatWinningTheCasPreservesTheLiveProducer() { + when(jobs.findAbandonedLegacyRagJobs(any(OffsetDateTime.class), eq(100))) + .thenReturn(List.of(coordinates)); + when(jobs.failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), anyString())) + .thenReturn(false); + + recovery.failAbandonedJobs(); + + verify(jobs, never()).findById(anyLong()); + verifyNoInteractions(tracking); + } + + @Test + void statusOwnedByANewerJobIsPreserved() { + when(jobs.findAbandonedLegacyRagJobs(any(OffsetDateTime.class), eq(100))) + .thenReturn(List.of(coordinates)); + when(jobs.failAbandonedLegacyRagJob( + eq(91L), any(OffsetDateTime.class), anyString())) + .thenReturn(true); + when(tracking.recoverAbandonedIncrementalUpdate( + eq(42L), eq(91L), anyString())).thenReturn(false); + + recovery.failAbandonedJobs(); + + verify(tracking).recoverAbandonedIncrementalUpdate( + eq(42L), eq(91L), anyString()); + } + + @Test + void failedJobProjectionDriftIsRetriedOnLaterScan() { + when(coordinates.getErrorMessage()).thenReturn("prior producer failure"); + when(jobs.findFailedLegacyRagJobsWithActiveStatus(100)) + .thenReturn(List.of(coordinates)); + + recovery.failAbandonedJobs(); + + verify(tracking).recoverAbandonedIncrementalUpdate( + 42L, 91L, "prior producer failure"); + verify(jobs, never()).failAbandonedLegacyRagJob( + anyLong(), any(), anyString()); + } + + @Test + void scanOutageLogsOneTransitionThenDebugAndOneRecovery() { + Logger logger = (Logger) org.slf4j.LoggerFactory.getLogger( + LegacyRagJobRecoveryService.class); + Level previousLevel = logger.getLevel(); + logger.setLevel(Level.DEBUG); + ListAppender logs = new ListAppender<>(); + logs.start(); + logger.addAppender(logs); + try { + when(jobs.findAbandonedLegacyRagJobs( + any(OffsetDateTime.class), eq(100))) + .thenThrow(new IllegalStateException("database unavailable")) + .thenThrow(new IllegalStateException("database unavailable")) + .thenReturn(List.of()); + + recovery.failAbandonedJobs(); + recovery.failAbandonedJobs(); + recovery.failAbandonedJobs(); + + long warnings = logs.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .filter(event -> event.getFormattedMessage() + .contains("recovery degraded")) + .count(); + long repeats = logs.list.stream() + .filter(event -> event.getLevel() == Level.DEBUG) + .filter(event -> event.getFormattedMessage() + .contains("remains degraded")) + .count(); + long recoveries = logs.list.stream() + .filter(event -> event.getLevel() == Level.INFO) + .filter(event -> event.getFormattedMessage() + .contains("recovery scan recovered")) + .count(); + assertThat(warnings).isEqualTo(1); + assertThat(repeats).isEqualTo(1); + assertThat(recoveries).isEqualTo(1); + } finally { + logger.detachAppender(logs); + logs.stop(); + logger.setLevel(previousLevel); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionServiceTest.java new file mode 100644 index 00000000..d5ae45f6 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionServiceTest.java @@ -0,0 +1,123 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; +import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.persistence.repository.analysis.RagIndexStatusRepository; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.OffsetDateTime; +import java.util.Optional; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class LegacyRagUpdateCompletionServiceTest { + private JobRepository jobs; + private RagIndexTrackingService tracking; + private RagIndexStatusRepository statuses; + private RagBranchIndexRepository branches; + private LegacyRagUpdateCompletionService completion; + private Project project; + + @BeforeEach + void setUp() { + jobs = mock(JobRepository.class); + tracking = mock(RagIndexTrackingService.class); + statuses = mock(RagIndexStatusRepository.class); + branches = mock(RagBranchIndexRepository.class); + completion = new LegacyRagUpdateCompletionService( + jobs, tracking, statuses, branches); + project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + } + + @Test + void recoveryWinningTheJobCasPreventsEveryCheckpointWrite() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) + .thenReturn(0); + + boolean completed = completion.complete( + project, "main", "commit-b", 91L, validAfter, + true, 2, 1, 30, Set.of("old.java")); + + assertThat(completed).isFalse(); + verifyNoInteractions(tracking, branches); + } + + @Test + void ownedJobAndAllCheckpointsAdvanceTogether() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + RagBranchIndex branch = new RagBranchIndex(project, "main"); + branch.setCommitHash("commit-a"); + branch.setDeletedFiles(new java.util.HashSet<>(Set.of("older.java"))); + when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) + .thenReturn(1); + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.UPDATING); + status.setActiveJobId(91L); + when(statuses.findByProjectIdForUpdate(42L)) + .thenReturn(Optional.of(status)); + when(branches.findByProjectIdAndBranchNameForUpdate(42L, "main")) + .thenReturn(Optional.of(branch)); + + boolean completed = completion.complete( + project, "main", "commit-b", 91L, validAfter, + true, 2, 1, 30, Set.of("old.java")); + + assertThat(completed).isTrue(); + verify(tracking).markUpdatingCompleted( + project, "main", "commit-b", 2, 1, 30, 91L); + verify(branches).save(argThat(saved -> + "commit-b".equals(saved.getCommitHash()) + && saved.getDeletedFiles().equals( + Set.of("older.java", "old.java")))); + } + + @Test + void newerProjectStatusOwnerFencesEveryCheckpointWrite() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.UPDATING); + status.setActiveJobId(92L); + when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) + .thenReturn(1); + when(statuses.findByProjectIdForUpdate(42L)) + .thenReturn(Optional.of(status)); + + org.assertj.core.api.Assertions.assertThatThrownBy(() -> completion.complete( + project, "main", "commit-b", 91L, validAfter, + true, 2, 1, 30, Set.of("old.java"))) + .isInstanceOf(LegacyRagUpdateCompletionService + .LegacyRagCompletionConflictException.class); + + verifyNoInteractions(tracking, branches); + } + + @Test + void nonPrimaryBranchDoesNotTouchProjectStatus() { + OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); + when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) + .thenReturn(1); + when(branches.findByProjectIdAndBranchNameForUpdate(42L, "feature")) + .thenReturn(Optional.empty()); + + assertThat(completion.complete( + project, "feature", "commit-b", 91L, validAfter, + false, 2, 1, null, Set.of())).isTrue(); + + verifyNoInteractions(tracking); + verify(branches).save(argThat(saved -> + "feature".equals(saved.getBranchName()) + && "commit-b".equals(saved.getCommitHash()))); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java index 96896140..865b935d 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java @@ -1,16 +1,20 @@ package org.rostilos.codecrow.ragengine.branch; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.rag.RagBranchIndex; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.model.workspace.Workspace; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.List; +import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.*; class RagBranchOperatorAliasReconciliationServiceTest { @@ -22,38 +26,228 @@ void restoresReadableAliasesForDurableAndPrimaryGenerationsOnly() throws Excepti RagBranchOperatorAliasReconciliationService service = new RagBranchOperatorAliasReconciliationService(repository, client); - RagBranchIndex primary = index("main", RagBranchIndexKind.PRIMARY, "main-target"); - RagBranchIndex durable = index("develop", RagBranchIndexKind.DURABLE, "develop-target"); - RagBranchIndex transientIndex = index("release", RagBranchIndexKind.TRANSIENT, "release-target"); - when(repository.findAll()).thenReturn(List.of(primary, durable, transientIndex)); + var primary = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + var durable = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(primary, durable)); + stubCurrent(repository, primary, durable); service.reconcileActiveGenerationAliases(); verify(client).publishGenerationAliases( - "workspace", "project", "main", "revision", "main-target", true, true); + "workspace", "project", "main", "revision", "main-target", "manifest-main", true, true); verify(client).publishGenerationAliases( - "workspace", "project", "develop", "revision", "develop-target", true, false); + "workspace", "project", "develop", "revision", "develop-target", "manifest-develop", true, false); verifyNoMoreInteractions(client); } - private static RagBranchIndex index( + @Test + void transportFailureStopsRemainingCandidatesUntilNextScheduledRun() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var first = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + var second = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(first, second)); + stubCurrent(repository, first, second); + doThrow(new IOException("timeout")).when(client).publishGenerationAliases( + eq("workspace"), eq("project"), eq("main"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + + service.reconcileActiveGenerationAliases(); + + verify(client).publishGenerationAliases( + "workspace", "project", "main", "revision", "main-target", "manifest-main", true, true); + verify(client, never()).publishGenerationAliases( + eq("workspace"), eq("project"), eq("develop"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + } + + @Test + void rateLimitStopsRemainingCandidatesUntilNextScheduledRun() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var first = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + var second = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(first, second)); + stubCurrent(repository, first, second); + doThrow(new RagPipelineClient.RagApiException(429, "rate limited")) + .when(client).publishGenerationAliases( + eq("workspace"), eq("project"), eq("main"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + + service.reconcileActiveGenerationAliases(); + + verify(client, never()).publishGenerationAliases( + eq("workspace"), eq("project"), eq("develop"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + } + + @Test + void generationValidationRejectionDoesNotBlockOtherCandidates() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var first = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + var second = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(first, second)); + stubCurrent(repository, first, second); + doThrow(new RagPipelineClient.RagApiException(409, "manifest mismatch")) + .when(client).publishGenerationAliases( + eq("workspace"), eq("project"), eq("main"), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + + service.reconcileActiveGenerationAliases(); + + verify(client).publishGenerationAliases( + "workspace", "project", "develop", "revision", "develop-target", "manifest-develop", true, false); + } + + @Test + void repairsAliasWithNewGenerationWhenRegistryChangesDuringPublication() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var generationA = candidate( + "main", RagBranchIndexKind.PRIMARY, "generation-a", 10L, 100L); + var generationB = candidate( + "main", RagBranchIndexKind.PRIMARY, "generation-b", 10L, 101L); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(generationA)); + when(repository.findOperatorAliasCandidateById(10L)).thenReturn( + Optional.of(generationA), Optional.of(generationB), Optional.of(generationB)); + + service.reconcileActiveGenerationAliases(); + + var inOrder = inOrder(client); + inOrder.verify(client).publishGenerationAliases( + "workspace", "project", "main", "revision", "generation-a", "manifest-main", true, true); + inOrder.verify(client).publishGenerationAliases( + "workspace", "project", "main", "revision", "generation-b", "manifest-main", true, true); + } + + @Test + void persistentOutageWarnsOnceAndLogsRecoveryOnce() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var candidate = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(candidate)); + stubCurrent(repository, candidate); + doThrow(new IOException("timeout")) + .doThrow(new IOException("timeout")) + .doNothing() + .when(client).publishGenerationAliases( + anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + Logger logger = (Logger) LoggerFactory.getLogger( + RagBranchOperatorAliasReconciliationService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + try { + service.reconcileActiveGenerationAliases(); + service.reconcileActiveGenerationAliases(); + service.reconcileActiveGenerationAliases(); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage)) + .containsExactly("RAG alias reconciliation stopped after a transport failure; " + + "remaining candidates will retry next run: timeout"); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.INFO) + .map(ILoggingEvent::getFormattedMessage)) + .anyMatch(message -> message.contains("retry next run: timeout")) + .contains("RAG alias reconciliation recovered"); + } + + @Test + void persistentCandidateRejectionWarnsOnlyOnDegradedTransition() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + var candidate = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); + when(repository.findOperatorAliasCandidates()).thenReturn(List.of(candidate)); + stubCurrent(repository, candidate); + doThrow(new RagPipelineClient.RagApiException(409, "manifest mismatch")) + .doThrow(new RagPipelineClient.RagApiException(409, "manifest mismatch")) + .doNothing() + .when(client).publishGenerationAliases( + anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + Logger logger = (Logger) LoggerFactory.getLogger( + RagBranchOperatorAliasReconciliationService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + try { + service.reconcileActiveGenerationAliases(); + service.reconcileActiveGenerationAliases(); + service.reconcileActiveGenerationAliases(); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage)) + .containsExactly("RAG alias reconciliation completed with 1 rejected candidate(s); " + + "they will retry next run"); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.INFO) + .map(ILoggingEvent::getFormattedMessage)) + .contains("RAG alias reconciliation recovered"); + } + + private static void stubCurrent( + RagBranchIndexRepository repository, + RagBranchIndexRepository.OperatorAliasCandidate... candidates) { + for (var candidate : candidates) { + when(repository.findOperatorAliasCandidateById(candidate.getBranchIndexId())) + .thenReturn(Optional.of(candidate)); + } + } + + private static RagBranchIndexRepository.OperatorAliasCandidate candidate( String branch, RagBranchIndexKind kind, String target) { - Workspace workspace = mock(Workspace.class); - when(workspace.getName()).thenReturn("workspace"); - Project project = mock(Project.class); - when(project.getWorkspace()).thenReturn(workspace); - when(project.getNamespace()).thenReturn("project"); - when(project.getId()).thenReturn(1L); - RagBranchIndexGeneration generation = mock(RagBranchIndexGeneration.class); - when(generation.getRevision()).thenReturn("revision"); - when(generation.getCollectionName()).thenReturn(target); - RagBranchIndex index = mock(RagBranchIndex.class); - when(index.getProject()).thenReturn(project); - when(index.getBranchName()).thenReturn(branch); - when(index.getIndexKind()).thenReturn(kind); - when(index.getActiveGeneration()).thenReturn(generation); - return index; + var candidate = mock(RagBranchIndexRepository.OperatorAliasCandidate.class); + when(candidate.getBranchIndexId()).thenReturn( + "main".equals(branch) ? 10L : 20L); + when(candidate.getGenerationId()).thenReturn( + "main".equals(branch) ? 100L : 200L); + when(candidate.getProjectId()).thenReturn(1L); + when(candidate.getWorkspaceName()).thenReturn("workspace"); + when(candidate.getProjectNamespace()).thenReturn("project"); + when(candidate.getBranchName()).thenReturn(branch); + when(candidate.getRevision()).thenReturn("revision"); + when(candidate.getCollectionName()).thenReturn(target); + when(candidate.getManifestDigest()).thenReturn("manifest-" + branch); + when(candidate.getIndexKind()).thenReturn(kind); + return candidate; + } + + private static RagBranchIndexRepository.OperatorAliasCandidate candidate( + String branch, + RagBranchIndexKind kind, + String target, + Long branchIndexId, + Long generationId) { + var candidate = candidate(branch, kind, target); + when(candidate.getBranchIndexId()).thenReturn(branchIndexId); + when(candidate.getGenerationId()).thenReturn(generationId); + return candidate; } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatServiceTest.java new file mode 100644 index 00000000..cc081c31 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationHeartbeatServiceTest.java @@ -0,0 +1,62 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +class RagIndexOperationHeartbeatServiceTest { + + @Test + void scopeSchedulesHeartbeatsAndCancelsOnClose() { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + @SuppressWarnings("unchecked") + ScheduledFuture scheduled = mock(ScheduledFuture.class); + doReturn(scheduled).when(executor).scheduleAtFixedRate( + any(Runnable.class), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); + RagIndexOperationHeartbeatService service = + new RagIndexOperationHeartbeatService(registry, executor, 15L); + + RagIndexOperationHeartbeatService.HeartbeatScope scope = service.start(91L); + + var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(executor).scheduleAtFixedRate( + heartbeat.capture(), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); + heartbeat.getValue().run(); + verify(registry).heartbeatBuild(91L); + + scope.close(); + verify(scheduled).cancel(false); + service.close(); + verify(executor).shutdownNow(); + } + + @Test + void transientHeartbeatFailureDoesNotEscapeSchedulerTask() { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + @SuppressWarnings("unchecked") + ScheduledFuture scheduled = mock(ScheduledFuture.class); + doReturn(scheduled).when(executor).scheduleAtFixedRate( + any(Runnable.class), anyLong(), anyLong(), any()); + doThrow(new IllegalStateException("database unavailable")) + .when(registry).heartbeatBuild(92L); + RagIndexOperationHeartbeatService service = + new RagIndexOperationHeartbeatService(registry, executor, 15L); + + service.start(92L); + + var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(executor).scheduleAtFixedRate( + heartbeat.capture(), anyLong(), anyLong(), any()); + heartbeat.getValue().run(); + verify(registry).heartbeatBuild(92L); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java index 404706b4..19af8a79 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java @@ -1,31 +1,305 @@ package org.rostilos.codecrow.ragengine.branch; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.model.rag.RagIndexOperation; +import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; +import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; +import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobStatus; +import org.rostilos.codecrow.core.model.job.JobType; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagIndexOperationRepository; +import org.rostilos.codecrow.core.service.JobService; import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.slf4j.LoggerFactory; import java.time.OffsetDateTime; import java.util.List; +import java.util.Optional; -import static org.mockito.ArgumentMatchers.any; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; class RagIndexOperationRecoveryServiceTest { + private RagBranchIndexRegistryService registry; + private ProjectRepository projects; + private JobService jobs; + private RagIndexTrackingService tracking; + private RagOperationsService ragOperations; + private AnalysisLockService locks; + private RagIndexOperationRecoveryService recovery; + private RagIndexOperationRepository.RecoveryOperationProjection operation; + private Project project; + + @BeforeEach + void setUp() { + registry = mock(RagBranchIndexRegistryService.class); + projects = mock(ProjectRepository.class); + jobs = mock(JobService.class); + tracking = mock(RagIndexTrackingService.class); + ragOperations = mock(RagOperationsService.class); + locks = mock(AnalysisLockService.class); + recovery = new RagIndexOperationRecoveryService( + registry, projects, jobs, tracking, ragOperations, locks, 30); + + project = mock(Project.class); + when(project.getId()).thenReturn(42L); + operation = mock( + RagIndexOperationRepository.RecoveryOperationProjection.class); + when(operation.getOperationId()).thenReturn(81L); + when(operation.getProjectId()).thenReturn(42L); + when(operation.getBranchName()).thenReturn("main"); + when(operation.getToRevision()).thenReturn("commit-a"); + when(operation.getJobId()).thenReturn(91L); + when(operation.getAnalysisLockKey()).thenReturn("rag-lock-owner-91"); + when(registry.findFailedOperationsWithActiveProjections()) + .thenReturn(List.of()); + when(registry.findSucceededOperationsWithActiveProjections()) + .thenReturn(List.of()); + } + @Test - void abandonedPersistedOperationBecomesTerminalWithUsefulDiagnostic() { - RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); - RagIndexOperation operation = new RagIndexOperation(); - operation.setId(81L); - operation.setBranchName("develop"); + void abandonedOperationTerminalizesJobPrimaryStatusAndLock() { + Job job = new Job(); + job.setJobType(JobType.RAG_INCREMENTAL_INDEX); + job.setStatus(JobStatus.RUNNING); + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.INDEXING); + status.setActiveJobId(91L); when(registry.findRecoverableOperations(any(OffsetDateTime.class))) .thenReturn(List.of(operation)); + when(registry.failIfAbandoned(eq(81L), any(OffsetDateTime.class), anyString())) + .thenReturn(true); + when(jobs.findById(91L)).thenReturn(Optional.of(job)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(tracking.getIndexStatus(project)).thenReturn(Optional.of(status)); + recovery.failAbandonedOperations(); - new RagIndexOperationRecoveryService(registry, 30) - .failAbandonedOperations(); - - verify(registry).fail(eq(81L), argThat(message -> + verify(registry).failIfAbandoned(eq(81L), any(OffsetDateTime.class), argThat(message -> message.contains("stopped heartbeating") && message.contains("previous active generation was preserved"))); + verify(jobs).failJob(eq(job), contains("stopped heartbeating")); + verify(tracking).markIndexingFailed( + eq(project), contains("stopped heartbeating"), eq(91L)); + verify(locks).releaseLock("rag-lock-owner-91"); + } + + @Test + void abandonedIncrementalOperationRestoresTheUsableIndexedStatus() { + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.UPDATING); + status.setActiveJobId(91L); + when(registry.findRecoverableOperations(any(OffsetDateTime.class))) + .thenReturn(List.of(operation)); + when(registry.failIfAbandoned(eq(81L), any(OffsetDateTime.class), anyString())) + .thenReturn(true); + when(jobs.findById(91L)).thenReturn(Optional.empty()); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(tracking.getIndexStatus(project)).thenReturn(Optional.of(status)); + + recovery.failAbandonedOperations(); + + verify(tracking).markIncrementalUpdateFailed( + eq(project), contains("stopped heartbeating"), eq(91L)); + verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); + } + + @Test + void newerLiveOperationPreservesStatusWhileOldExactLockIsReleased() { + Job job = new Job(); + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.INDEXING); + status.setActiveJobId(92L); + when(registry.findRecoverableOperations(any(OffsetDateTime.class))) + .thenReturn(List.of(operation)); + when(registry.failIfAbandoned(eq(81L), any(OffsetDateTime.class), anyString())) + .thenReturn(true); + when(jobs.findById(91L)).thenReturn(Optional.of(job)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(tracking.getIndexStatus(project)).thenReturn(Optional.of(status)); + + recovery.failAbandonedOperations(); + + verify(jobs).failJob(eq(job), contains("stopped heartbeating")); + verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); + verify(locks).releaseLock("rag-lock-owner-91"); + } + + @Test + void alreadyFailedOperationRepairsOlderProjectionDrift() { + Job job = new Job(); + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.INDEXING); + status.setActiveJobId(91L); + when(operation.getErrorMessage()).thenReturn("producer was abandoned"); + when(registry.findRecoverableOperations(any(OffsetDateTime.class))) + .thenReturn(List.of()); + when(registry.findFailedOperationsWithActiveProjections()) + .thenReturn(List.of(operation)); + when(jobs.findById(91L)).thenReturn(Optional.of(job)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(tracking.getIndexStatus(project)).thenReturn(Optional.of(status)); + + recovery.failAbandonedOperations(); + + verify(registry, never()).failIfAbandoned(anyLong(), any(), anyString()); + verify(jobs).failJob(job, "producer was abandoned"); + verify(tracking).markIndexingFailed(project, "producer was abandoned", 91L); + verify(locks).releaseLock("rag-lock-owner-91"); + } + + @Test + void alreadyFailedOperationPreservesStatusOwnedByANewerJob() { + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.INDEXING); + status.setActiveJobId(92L); + when(registry.findRecoverableOperations(any(OffsetDateTime.class))) + .thenReturn(List.of()); + when(registry.findFailedOperationsWithActiveProjections()) + .thenReturn(List.of(operation)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(tracking.getIndexStatus(project)).thenReturn(Optional.of(status)); + + recovery.failAbandonedOperations(); + + verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); + verify(tracking, never()).markIncrementalUpdateFailed(any(), anyString(), any()); + verify(locks).releaseLock("rag-lock-owner-91"); + } + + @Test + void failedOperationNeverGuessesOwnershipOfAnOwnerlessStatus() { + RagIndexStatus status = new RagIndexStatus(); + status.setStatus(RagIndexingStatus.UPDATING); + when(registry.findRecoverableOperations(any(OffsetDateTime.class))) + .thenReturn(List.of()); + when(registry.findFailedOperationsWithActiveProjections()) + .thenReturn(List.of(operation)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(tracking.getIndexStatus(project)).thenReturn(Optional.of(status)); + + recovery.failAbandonedOperations(); + + verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); + verify(tracking, never()).markIncrementalUpdateFailed(any(), anyString(), any()); + verify(locks).releaseLock("rag-lock-owner-91"); + } + + @Test + void publishedInitialOperationCompletesJobRepairsPrimaryStatusAndReleasesLock() { + RagIndexOperationRepository.SucceededOperationProjection published = + mock(RagIndexOperationRepository.SucceededOperationProjection.class); + when(published.getOperationId()).thenReturn(82L); + when(published.getProjectId()).thenReturn(42L); + when(published.getBranchName()).thenReturn("main"); + when(published.getToRevision()).thenReturn("commit-published"); + when(published.getJobId()).thenReturn(92L); + when(published.getAnalysisLockKey()).thenReturn("published-lock-owner"); + when(published.getFileCount()).thenReturn(214); + when(published.getChunkCount()).thenReturn(642); + when(published.getActiveGeneration()).thenReturn(true); + Job job = new Job(); + job.setJobType(JobType.RAG_INITIAL_INDEX); + job.setStatus(JobStatus.RUNNING); + when(registry.findRecoverableOperations(any())).thenReturn(List.of()); + when(registry.findSucceededOperationsWithActiveProjections()) + .thenReturn(List.of(published)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(jobs.findById(92L)).thenReturn(Optional.of(job)); + + recovery.failAbandonedOperations(); + + verify(tracking).reconcilePublishedGeneration( + project, "main", "commit-published", 214, 642, 92L); + verify(jobs).completeJob(job); + verify(jobs, never()).failJob(any(), anyString()); + verify(locks).releaseLock("published-lock-owner"); + } + + @Test + void publishedOperationWithOwnerlessStatusIsNotGuessedOrRegressed() { + RagIndexOperationRepository.SucceededOperationProjection published = + mock(RagIndexOperationRepository.SucceededOperationProjection.class); + when(published.getProjectId()).thenReturn(42L); + when(published.getBranchName()).thenReturn("main"); + when(published.getToRevision()).thenReturn("old-commit"); + when(published.getJobId()).thenReturn(92L); + when(published.getActiveGeneration()).thenReturn(true); + when(registry.findRecoverableOperations(any())).thenReturn(List.of()); + when(registry.findSucceededOperationsWithActiveProjections()) + .thenReturn(List.of(published)); + when(projects.findByIdWithFullDetails(42L)).thenReturn(Optional.of(project)); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(tracking.reconcilePublishedGeneration( + project, "main", "old-commit", 0, 0, 92L)) + .thenReturn(false); + + recovery.failAbandonedOperations(); + + verify(tracking).reconcilePublishedGeneration( + project, "main", "old-commit", 0, 0, 92L); + verifyNoMoreInteractions(tracking); + } + + @Test + void persistentProjectionFailuresEmitOneWarningThenDebugUntilRecovery() { + when(registry.findRecoverableOperations(any())).thenReturn(List.of()); + when(registry.findFailedOperationsWithActiveProjections()) + .thenReturn(List.of(operation), List.of(operation), List.of()); + when(jobs.findById(91L)).thenThrow(new IllegalStateException("database down")); + when(projects.findByIdWithFullDetails(42L)) + .thenThrow(new IllegalStateException("database down")); + doThrow(new IllegalStateException("database down")) + .when(locks).releaseLock("rag-lock-owner-91"); + Logger logger = (Logger) LoggerFactory.getLogger( + RagIndexOperationRecoveryService.class); + Level priorLevel = logger.getLevel(); + logger.setLevel(Level.DEBUG); + ListAppender events = new ListAppender<>(); + events.start(); + logger.addAppender(events); + try { + recovery.failAbandonedOperations(); + recovery.failAbandonedOperations(); + + reset(jobs, projects, locks); + recovery.failAbandonedOperations(); + } finally { + logger.detachAppender(events); + logger.setLevel(priorLevel); + events.stop(); + } + + List transitionEvents = events.list.stream() + .filter(event -> event.getFormattedMessage() + .contains("Exact RAG operation recovery")) + .toList(); + assertThat(transitionEvents.stream() + .filter(event -> event.getLevel() == Level.WARN)).hasSize(1); + assertThat(transitionEvents.stream() + .filter(event -> event.getLevel() == Level.DEBUG)).isNotEmpty(); + assertThat(transitionEvents.stream() + .filter(event -> event.getLevel() == Level.INFO + && event.getFormattedMessage().contains("scan recovered"))) + .hasSize(1); + assertThat(events.list.stream() + .filter(event -> event.getLevel() == Level.ERROR)).isEmpty(); } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java index 072fe996..e628bb32 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java @@ -1,59 +1,326 @@ package org.rostilos.codecrow.ragengine.branch; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.project.config.RagConfig; -import org.rostilos.codecrow.core.model.rag.RagBranchIndex; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.springframework.test.util.ReflectionTestUtils; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.annotation.Transactional; import java.time.OffsetDateTime; import java.util.List; -import static org.mockito.Mockito.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; class RagTransientBranchIndexCleanupServiceTest { @Test - void deletesExpiredTransientGenerationUsingProjectTenantCoordinates() throws Exception { + void deletesExpiredGenerationOutsideSchedulerTransactionAndRechecksRegistry() throws Exception { RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); RagBranchIndexGenerationRepository generations = mock(RagBranchIndexGenerationRepository.class); RagPipelineClient pipeline = mock(RagPipelineClient.class); - Project project = new Project(); - ReflectionTestUtils.setField(project, "id", 42L); - project.setNamespace("namespace"); - Workspace workspace = new Workspace(); - workspace.setName("workspace"); - project.setWorkspace(workspace); - project.setConfiguration(new ProjectConfig( - false, "master", null, - new RagConfig(true, "master", null, null, - true, 30, List.of("develop"), true))); - RagBranchIndex index = new RagBranchIndex( - project, "release/candidate", RagBranchIndexKind.TRANSIENT); - index.setId(10L); - index.setLastAccessedAt(OffsetDateTime.now().minusDays(31)); - RagBranchIndexGeneration generation = mock(RagBranchIndexGeneration.class); - when(generation.getCollectionName()).thenReturn("opaque-transient-target"); - when(branches.findByIndexKind(RagBranchIndexKind.TRANSIENT)) - .thenReturn(List.of(index)); - when(generations.findByBranchIndexIdOrderByCreatedAtDesc(10L)) + var index = expiredCandidate(); + var generation = generation(100L, "opaque-transient-target"); + when(branches.findTransientCleanupCandidates()).thenReturn(List.of(index)); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) .thenReturn(List.of(generation)); - when(pipeline.deleteBranch( + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(pipeline.deleteBranchWithOutcome( "workspace", "namespace", "release/candidate", - "opaque-transient-target")) - .thenReturn(true); + "opaque-transient-target", "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success( + "opaque-transient-target")); + when(branches.deleteClaimedTransientById(eq(10L), anyString())) + .thenReturn(1); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + verify(branches).deleteClaimedTransientById(eq(10L), anyString()); + verify(pipeline).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", + "opaque-transient-target", "revision-100", "manifest-100"); + assertThat(RagTransientBranchIndexCleanupService.class + .getMethod("cleanupExpired") + .isAnnotationPresent(Transactional.class)) + .isFalse(); + } + + @Test + void serviceFailureStopsRemainingGenerationsAndWarnsOnlyOnDegradedTransition() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var first = generation(100L, "target-a"); + var second = generation(101L, "target-b"); + when(branches.findTransientCleanupCandidates()).thenReturn(List.of(index)); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(first, second)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "target-a", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "target-a", + RagPipelineClient.BranchDeletionFailure.SERVICE, + 503, + "unavailable")); + Logger logger = (Logger) LoggerFactory.getLogger( + RagTransientBranchIndexCleanupService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + try { + RagTransientBranchIndexCleanupService service = + new RagTransientBranchIndexCleanupService(branches, generations, pipeline); + service.cleanupExpired(); + service.cleanupExpired(); + } finally { + logger.detachAppender(appender); + } + + verify(pipeline, times(2)).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "target-a", + "revision-100", "manifest-100"); + verify(pipeline, never()).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "target-b", + "revision-101", "manifest-101"); + verify(branches, never()).deleteClaimedTransientById( + org.mockito.ArgumentMatchers.anyLong(), anyString()); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN + && event.getFormattedMessage().contains("target=target-a"))) + .hasSize(1); + } + + @Test + void refreshedCandidateThatLosesAtomicClaimIsNeverDeleted() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + when(branches.findTransientCleanupCandidates()) + .thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(0); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + verifyNoInteractions(generations, pipeline); + } + + @Test + void partialPhysicalCleanupKeepsDurableClaimAndDeletesActiveGenerationLast() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var superseded = generation(100L, "superseded", RagBranchIndexGenerationStatus.SUPERSEDED); + var active = generation(101L, "active", RagBranchIndexGenerationStatus.ACTIVE); + when(branches.findTransientCleanupCandidates()) + .thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(active, superseded)); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "superseded", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("superseded")); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "active", + "revision-101", "manifest-101")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "active", RagPipelineClient.BranchDeletionFailure.SERVICE, + 503, "unavailable")); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + var ordered = inOrder(pipeline); + ordered.verify(pipeline).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "superseded", + "revision-100", "manifest-100"); + ordered.verify(pipeline).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "active", + "revision-101", "manifest-101"); + verify(branches, never()).cancelTransientDeletion(eq(10L), anyString()); + verify(branches, never()).deleteClaimedTransientById(eq(10L), anyString()); + } + + @Test + void transportFailureKeepsClaimBecauseRemoteDeletionMayHaveCompleted() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var generation = generation(100L, "uncertain-target"); + when(branches.findTransientCleanupCandidates()) + .thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(generation)); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "uncertain-target", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "uncertain-target", + RagPipelineClient.BranchDeletionFailure.TRANSPORT, + null, + "connection reset")); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + verify(branches, never()).cancelTransientDeletion(eq(10L), anyString()); + verify(branches, never()).deleteClaimedTransientById(eq(10L), anyString()); + } + + @Test + void targetRejectionKeepsActiveTargetReadableAndReleasesUnusedClaim() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var rejected = generation( + 100L, "rejected", RagBranchIndexGenerationStatus.SUPERSEDED); + var active = generation( + 101L, "active", RagBranchIndexGenerationStatus.ACTIVE); + when(branches.findTransientCleanupCandidates()).thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(active, rejected)); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "rejected", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "rejected", RagPipelineClient.BranchDeletionFailure.TARGET, + 422, "manifest receipt rejected")); new RagTransientBranchIndexCleanupService( branches, generations, pipeline).cleanupExpired(); - verify(branches).delete(index); + verify(pipeline, never()).deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "active", + "revision-101", "manifest-101"); + verify(branches).cancelTransientDeletion(eq(10L), anyString()); + verify(branches, never()).deleteClaimedTransientById(eq(10L), anyString()); + } + + @Test + void globallyDisabledRagSilentlyRetainsRegistryRow() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + var index = expiredCandidate(); + var generation = generation(100L, "disabled-target"); + when(branches.findTransientCleanupCandidates()) + .thenReturn(List.of(index)); + when(branches.claimExpiredTransientForDeletion( + eq(10L), any(), any(), anyString(), any())).thenReturn(1); + when(branches.heartbeatTransientDeletionClaim( + eq(10L), anyString(), any())).thenReturn(1); + when(generations.findCleanupCandidatesByBranchIndexId(10L)) + .thenReturn(List.of(generation)); + when(pipeline.deleteBranchWithOutcome( + "workspace", "namespace", "release/candidate", "disabled-target", + "revision-100", "manifest-100")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "disabled-target", + RagPipelineClient.BranchDeletionFailure.TARGET, + null, + "RAG disabled")); + Logger logger = (Logger) LoggerFactory.getLogger( + RagTransientBranchIndexCleanupService.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + try { + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + } finally { + logger.detachAppender(appender); + } + + verify(branches).cancelTransientDeletion(eq(10L), anyString()); + verify(branches, never()).deleteClaimedTransientById(eq(10L), anyString()); + assertThat(appender.list).noneMatch(event -> event.getLevel() == Level.WARN); + } + + private static RagBranchIndexRepository.TransientCleanupCandidate expiredCandidate() { + var candidate = mock(RagBranchIndexRepository.TransientCleanupCandidate.class); + when(candidate.getBranchIndexId()).thenReturn(10L); + when(candidate.getProjectId()).thenReturn(42L); + when(candidate.getWorkspaceName()).thenReturn("workspace"); + when(candidate.getProjectNamespace()).thenReturn("namespace"); + when(candidate.getBranchName()).thenReturn("release/candidate"); + when(candidate.getLastAccessedAt()).thenReturn(OffsetDateTime.now().minusDays(31)); + when(candidate.getProjectConfiguration()).thenReturn(new ProjectConfig( + false, "master", null, + new RagConfig(true, "master", null, null, + true, 30, List.of("develop"), true))); + return candidate; + } + + private static RagBranchIndexGenerationRepository.CleanupGenerationCandidate generation( + Long id, + String target) { + return generation(id, target, RagBranchIndexGenerationStatus.SUPERSEDED); + } + + private static RagBranchIndexGenerationRepository.CleanupGenerationCandidate generation( + Long id, + String target, + RagBranchIndexGenerationStatus status) { + var candidate = mock( + RagBranchIndexGenerationRepository.CleanupGenerationCandidate.class); + when(candidate.getGenerationId()).thenReturn(id); + when(candidate.getCollectionName()).thenReturn(target); + when(candidate.getRevision()).thenReturn("revision-" + id); + when(candidate.getManifestDigest()).thenReturn("manifest-" + id); + when(candidate.getStatus()).thenReturn(status); + return candidate; } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java index 46d092d5..af202787 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -101,6 +102,29 @@ void testDeleteFiles_EmptyList() throws Exception { assertThat(result).containsEntry("status", "success"); } + @Test + @SuppressWarnings("unchecked") + void publishGenerationAliasesIncludesExpectedManifestDigest() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("{\"status\":\"published\"}") + .addHeader("Content-Type", "application/json")); + + client.publishGenerationAliases( + "ws", "proj", "main", "commit", "physical-target", + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + true, true); + + RecordedRequest request = mockWebServer.takeRequest(); + assertThat(request.getPath()).isEqualTo("/index/generation-aliases"); + Map payload = objectMapper.readValue( + request.getBody().readUtf8(), Map.class); + assertThat(payload) + .containsEntry("collection_target", "physical-target") + .containsEntry( + "generation_manifest_sha256", + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); + } + @Test @SuppressWarnings("unchecked") void testApplyChanges_SendsOneCompleteCommitPayload() throws Exception { @@ -446,6 +470,25 @@ void testIndexRepository_WithExcludePatterns() throws Exception { assertThat(body).contains("exclude_patterns"); } + @Test + void testIndexRepository_ForwardsPriorGenerationForVectorReuse() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("{\"document_count\":42}") + .addHeader("Content-Type", "application/json")); + + client.indexRepository( + repositoryPath.toString(), "ws", "proj", "main", "abc123", + null, null, "new-generation", false, false, + "prior-generation"); + + RecordedRequest request = mockWebServer.takeRequest(); + Map payload = objectMapper.readValue( + request.getBody().readUtf8(), Map.class); + assertThat(payload) + .containsEntry("collection_target", "new-generation") + .containsEntry("reuse_collection_target", "prior-generation"); + } + @Test void testIndexRepository_WhenDisabled() throws Exception { RagPipelineClient disabledClient = new RagPipelineClient( @@ -482,6 +525,27 @@ void testIndexRepository_StreamForwardsProgressAndReturnsTerminalResult() throws assertThat(request.getHeader("Accept")).isEqualTo("text/event-stream"); } + @Test + void testIndexRepository_StreamAcknowledgesSnapshotOwnershipTransfer() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("data: {\"type\":\"admitted\",\"repositoryOwnershipTransferred\":true}\n\n" + + "data: {\"type\":\"complete\",\"result\":{\"document_count\":2,\"chunk_count\":5}}\n\n") + .addHeader("Content-Type", "text/event-stream")); + AtomicBoolean admitted = new AtomicBoolean(false); + + Map result = client.indexRepository( + repositoryPath.toString(), "ws", "proj", "develop", "abc123", + List.of("src/**"), List.of("vendor/**"), "generation-target", + false, false, true, () -> admitted.set(true), ignored -> { }); + + assertThat(admitted).isTrue(); + assertThat(result).containsEntry("chunk_count", 5); + RecordedRequest request = mockWebServer.takeRequest(); + Map payload = objectMapper.readValue( + request.getBody().readUtf8(), Map.class); + assertThat(payload).containsEntry("transfer_repo_ownership", true); + } + // ── deleteBranch tests ─────────────────────────────────────────────────── @Test @@ -527,6 +591,30 @@ void testDeleteBranch_WithSlashInBranchName() throws Exception { assertThat(request.getPath()).contains("feature%2Fxyz"); } + @Test + void testDeleteBranchStructuredOutcomeClassifiesServiceFailureAndNamesTarget() + throws Exception { + mockWebServer.enqueue(new MockResponse() + .setResponseCode(503) + .setBody("{\"detail\":\"unavailable\"}")); + + RagPipelineClient.BranchDeletionOutcome outcome = + client.deleteBranchWithOutcome( + "ws", "proj", "feature", "physical target/1", + "revision/abc", "manifest+digest"); + + assertThat(outcome.successful()).isFalse(); + assertThat(outcome.failure()) + .isEqualTo(RagPipelineClient.BranchDeletionFailure.SERVICE); + assertThat(outcome.shouldStopRemainingTargets()).isTrue(); + assertThat(outcome.targetLabel()).isEqualTo("physical target/1"); + assertThat(outcome.statusCode()).isEqualTo(503); + assertThat(mockWebServer.takeRequest().getPath()) + .contains("collection_target=physical%20target%2F1") + .contains("generation_revision=revision%2Fabc") + .contains("generation_manifest_sha256=manifest%2Bdigest"); + } + // ── getIndexedBranches tests ───────────────────────────────────────────── @Test @@ -749,15 +837,49 @@ void testDeletePrFiles_Success() throws Exception { assertThat(request.getHeader("x-service-secret")).isEqualTo("test-secret"); } + @Test + void deletePrFilesTargetsExactGenerationAndEncodesQueryParameter() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setResponseCode(200) + .setBody("{\"deleted_count\": 1}")); + + boolean result = client.deletePrFiles( + "ws", "proj", 42, "project-generation/with space"); + + assertThat(result).isTrue(); + RecordedRequest request = mockWebServer.takeRequest(); + assertThat(request.getRequestUrl().queryParameter("collection_target")) + .isEqualTo("project-generation/with space"); + } + @Test void testDeletePrFiles_ServerError_ReturnsFalse() throws Exception { mockWebServer.enqueue(new MockResponse() .setResponseCode(500) .setBody("{\"error\": \"internal\"}")); - boolean result = client.deletePrFiles("ws", "proj", 42); + RagPipelineClient.PrFilesDeletionOutcome result = client.deletePrFilesWithOutcome( + "ws", "proj", 42, "generation-a"); - assertThat(result).isFalse(); + assertThat(result.successful()).isFalse(); + assertThat(result.targetLabel()).isEqualTo("generation-a"); + assertThat(result.statusCode()).isEqualTo(500); + assertThat(result.failure()).isEqualTo(RagPipelineClient.PrFilesDeletionFailure.SERVICE); + assertThat(result.shouldStopRemainingTargets()).isTrue(); + } + + @Test + void deletePrFilesLeaseConflictStopsRemainingTargets() { + mockWebServer.enqueue(new MockResponse() + .setResponseCode(409) + .setBody("{\"error\": \"mutation lease unavailable\"}")); + + RagPipelineClient.PrFilesDeletionOutcome result = client.deletePrFilesWithOutcome( + "ws", "proj", 42, "generation-a"); + + assertThat(result.statusCode()).isEqualTo(409); + assertThat(result.failure()).isEqualTo(RagPipelineClient.PrFilesDeletionFailure.SERVICE); + assertThat(result.shouldStopRemainingTargets()).isTrue(); } @Test @@ -786,8 +908,12 @@ void testDeletePrFiles_WhenDisabled_ReturnsTrue() { void testDeletePrFiles_NetworkError_ReturnsFalse() throws IOException { mockWebServer.shutdown(); - boolean result = client.deletePrFiles("ws", "proj", 42); + RagPipelineClient.PrFilesDeletionOutcome result = client.deletePrFilesWithOutcome( + "ws", "proj", 42, "generation-a"); - assertThat(result).isFalse(); + assertThat(result.successful()).isFalse(); + assertThat(result.targetLabel()).isEqualTo("generation-a"); + assertThat(result.failure()).isEqualTo(RagPipelineClient.PrFilesDeletionFailure.TRANSPORT); + assertThat(result.shouldStopRemainingTargets()).isTrue(); } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java index 7b5b3052..059e9930 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java @@ -12,9 +12,11 @@ import org.rostilos.codecrow.core.persistence.repository.rag.RagIndexOperationRepository; import org.springframework.test.util.ReflectionTestUtils; +import java.time.OffsetDateTime; import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; @@ -44,14 +46,14 @@ void setUp() { } return value; }); - when(generationRepository.save(any())).thenAnswer(invocation -> { + lenient().when(generationRepository.save(any())).thenAnswer(invocation -> { RagBranchIndexGeneration value = invocation.getArgument(0); if (value.getId() == null) { value.setId(20L); } return value; }); - when(operationRepository.save(any())).thenAnswer(invocation -> { + lenient().when(operationRepository.save(any())).thenAnswer(invocation -> { RagIndexOperation value = invocation.getArgument(0); if (value.getId() == null) { value.setId(30L); @@ -83,6 +85,7 @@ void registersIdempotentTenantScopedBuildWithoutLeakingBranchName() { .startsWith("cc_w0_p42_b") .doesNotContain("client", "private", "develop"); assertThat(registration.operation().getOperationKey()).hasSize(64); + assertThat(registration.sourceCollectionTarget()).isNull(); } @Test @@ -103,11 +106,13 @@ void publishesNewGenerationAndSupersedesPreviousOneAtomically() { var registration = service.registerBuild( project, "develop", RagBranchIndexKind.DURABLE, "develop-400", "develop-401", "representation"); - when(operationRepository.findById(30L)).thenReturn(Optional.of(registration.operation())); + assertThat(registration.sourceCollectionTarget()) + .isEqualTo("generation-400"); + when(operationRepository.findByIdForUpdate(30L)).thenReturn(Optional.of(registration.operation())); when(branchIndexRepository.findByIdForPublication(10L)) .thenReturn(Optional.of(branchIndex)); - service.startBuild(30L, 99L); + service.startBuild(30L, 99L, "rag-lock-owner-99"); RagBranchIndexGeneration published = service.publish(30L, "manifest-401", 501, 1504); assertThat(previous.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.SUPERSEDED); @@ -117,6 +122,8 @@ void publishesNewGenerationAndSupersedesPreviousOneAtomically() { assertThat(registration.operation().getStatus()).isEqualTo(RagIndexOperationStatus.SUCCEEDED); assertThat(registration.operation().getAttemptCount()).isEqualTo(1); assertThat(registration.operation().getJobId()).isEqualTo(99L); + assertThat(registration.operation().getAnalysisLockKey()) + .isEqualTo("rag-lock-owner-99"); } @Test @@ -141,7 +148,7 @@ void completedOlderGenerationDoesNotRegressNewerDesiredRevision() { operation.setId(30L); operation.setGeneration(late); operation.start(); - when(operationRepository.findById(30L)).thenReturn(Optional.of(operation)); + when(operationRepository.findByIdForUpdate(30L)).thenReturn(Optional.of(operation)); when(branchIndexRepository.findByIdForPublication(10L)) .thenReturn(Optional.of(branchIndex)); @@ -176,7 +183,7 @@ void failedReplacementKeepsLastVerifiedGenerationAvailable() { var registration = service.registerBuild( project, "master", RagBranchIndexKind.PRIMARY, "master-100", "master-101", "representation"); - when(operationRepository.findById(30L)).thenReturn(Optional.of(registration.operation())); + when(operationRepository.findByIdForUpdate(30L)).thenReturn(Optional.of(registration.operation())); when(branchIndexRepository.findByIdForPublication(10L)) .thenReturn(Optional.of(branchIndex)); @@ -206,7 +213,7 @@ void failedIdempotentOperationCanRetryWithoutCreatingDuplicateGeneration() { operation.setId(30L); operation.setGeneration(generation); operation.fail("worker restarted"); - when(operationRepository.findById(30L)).thenReturn(Optional.of(operation)); + when(operationRepository.findByIdForUpdate(30L)).thenReturn(Optional.of(operation)); service.startBuild(30L, 91L); @@ -236,7 +243,7 @@ void lateFailureDoesNotOverwriteNewerDesiredRevisionState() { operation.setId(30L); operation.setGeneration(late); operation.start(); - when(operationRepository.findById(30L)).thenReturn(Optional.of(operation)); + when(operationRepository.findByIdForUpdate(30L)).thenReturn(Optional.of(operation)); when(branchIndexRepository.findByIdForPublication(10L)) .thenReturn(Optional.of(branchIndex)); @@ -250,4 +257,66 @@ void lateFailureDoesNotOverwriteNewerDesiredRevisionState() { assertThat(branchIndex.getErrorMessage()).isNull(); verify(branchIndexRepository, never()).save(branchIndex); } + + @Test + void abandonmentClaimRechecksAHeartbeatUnderTheOperationLock() { + RagIndexOperation operation = new RagIndexOperation( + project, "develop", "develop-400", "develop-401", "heartbeat-key"); + operation.setId(30L); + operation.start(); + operation.setUpdatedAt(OffsetDateTime.now()); + when(operationRepository.findByIdForUpdate(30L)).thenReturn(Optional.of(operation)); + + boolean failed = service.failIfAbandoned( + 30L, OffsetDateTime.now().minusMinutes(30), "producer abandoned"); + + assertThat(failed).isFalse(); + assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.RUNNING); + verifyNoInteractions(branchIndexRepository); + verify(operationRepository, never()).save(operation); + } + + @Test + void cleanupClaimMakesExactGenerationUnavailableWithoutRefreshingIt() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "feature/expired", RagBranchIndexKind.TRANSIENT); + branchIndex.setId(10L); + branchIndex.setCleanupClaimToken("cleanup-owner"); + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate( + 42L, "feature/expired")) + .thenReturn(Optional.of(branchIndex)); + + assertThat(service.findAvailableGeneration( + 42L, "feature/expired", "revision-100")) + .isEmpty(); + + verify(branchIndexRepository, never()).save(branchIndex); + verifyNoInteractions(generationRepository); + } + + @Test + void cleanupClaimRejectsAConcurrentBuildRegistration() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "feature/expired", RagBranchIndexKind.TRANSIENT); + branchIndex.setId(10L); + branchIndex.setCleanupClaimToken("cleanup-owner"); + when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) + .thenReturn(Optional.empty()); + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate( + 42L, "feature/expired")) + .thenReturn(Optional.of(branchIndex)); + + assertThatThrownBy(() -> service.registerBuild( + project, + "feature/expired", + RagBranchIndexKind.TRANSIENT, + "revision-99", + "revision-100", + "representation")) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cleanup owns"); + + verify(branchIndexRepository, never()).save(any()); + verifyNoInteractions(generationRepository); + } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java index 3885eb5c..c3c9c209 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java @@ -88,10 +88,10 @@ void testGetIndexStatus_NotFound() { @Test void testMarkIndexingStarted_NewStatus() { - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.empty()); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.empty()); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); - RagIndexStatus result = service.markIndexingStarted(testProject, "main", "abc123"); + RagIndexStatus result = service.markIndexingStarted(testProject, "main", "abc123", 91L); ArgumentCaptor captor = ArgumentCaptor.forClass(RagIndexStatus.class); verify(ragIndexStatusRepository).save(captor.capture()); @@ -103,6 +103,7 @@ void testMarkIndexingStarted_NewStatus() { assertThat(saved.getIndexedCommitHash()).isEqualTo("abc123"); assertThat(saved.getWorkspaceName()).isEqualTo("test-workspace"); assertThat(saved.getProjectName()).isEqualTo("test-project"); + assertThat(saved.getActiveJobId()).isEqualTo(91L); } @Test @@ -112,10 +113,10 @@ void testMarkIndexingStarted_ExistingStatus() { existing.setStatus(RagIndexingStatus.FAILED); existing.setErrorMessage("Previous error"); - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.of(existing)); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); - service.markIndexingStarted(testProject, "develop", "xyz789"); + service.markIndexingStarted(testProject, "develop", "xyz789", 92L); ArgumentCaptor captor = ArgumentCaptor.forClass(RagIndexStatus.class); verify(ragIndexStatusRepository).save(captor.capture()); @@ -125,6 +126,7 @@ void testMarkIndexingStarted_ExistingStatus() { assertThat(saved.getIndexedBranch()).isEqualTo("develop"); assertThat(saved.getIndexedCommitHash()).isEqualTo("xyz789"); assertThat(saved.getErrorMessage()).isNull(); + assertThat(saved.getActiveJobId()).isEqualTo(92L); } @Test @@ -132,11 +134,13 @@ void testMarkIndexingCompleted() { RagIndexStatus existing = new RagIndexStatus(); existing.setProject(testProject); existing.setStatus(RagIndexingStatus.INDEXING); + existing.setActiveJobId(91L); - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.of(existing)); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); - RagIndexStatus result = service.markIndexingCompleted(testProject, "main", "abc123", 150, null); + RagIndexStatus result = service.markIndexingCompleted( + testProject, "main", "abc123", 150, null, 91L); ArgumentCaptor captor = ArgumentCaptor.forClass(RagIndexStatus.class); verify(ragIndexStatusRepository).save(captor.capture()); @@ -148,13 +152,15 @@ void testMarkIndexingCompleted() { assertThat(saved.getTotalFilesIndexed()).isEqualTo(150); assertThat(saved.getLastIndexedAt()).isNotNull(); assertThat(saved.getErrorMessage()).isNull(); + assertThat(saved.getActiveJobId()).isNull(); } @Test void testMarkIndexingCompleted_ThrowsWhenNotFound() { - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.empty()); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.empty()); - assertThatThrownBy(() -> service.markIndexingCompleted(testProject, "main", "abc123", 150, null)) + assertThatThrownBy(() -> service.markIndexingCompleted( + testProject, "main", "abc123", 150, null, 91L)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("RAG index status not found"); } @@ -165,7 +171,7 @@ void testMarkIndexingFailed_ExistingStatus() { existing.setProject(testProject); existing.setStatus(RagIndexingStatus.INDEXING); - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.of(existing)); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); service.markIndexingFailed(testProject, "Test error message"); @@ -180,7 +186,7 @@ void testMarkIndexingFailed_ExistingStatus() { @Test void testMarkIndexingFailed_NewStatus() { - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.empty()); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.empty()); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); service.markIndexingFailed(testProject, "New error"); @@ -202,7 +208,7 @@ void testMarkIndexingHeartbeat_RefreshesLiveStatusOnly() { existing.setUpdatedAt(OffsetDateTime.now().minusMinutes(5)); OffsetDateTime previousActivity = existing.getUpdatedAt(); - when(ragIndexStatusRepository.findByProjectId(100L)) + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) .thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))) .thenAnswer(i -> i.getArgument(0)); @@ -221,7 +227,7 @@ void testMarkIndexingHeartbeat_RefreshesIncrementalUpdate() { existing.setUpdatedAt(OffsetDateTime.now().minusMinutes(5)); OffsetDateTime previousActivity = existing.getUpdatedAt(); - when(ragIndexStatusRepository.findByProjectId(100L)) + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) .thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))) .thenAnswer(i -> i.getArgument(0)); @@ -232,6 +238,23 @@ void testMarkIndexingHeartbeat_RefreshesIncrementalUpdate() { verify(ragIndexStatusRepository).save(existing); } + @Test + void staleHeartbeatCannotRefreshANewerJobOwner() { + RagIndexStatus existing = new RagIndexStatus(); + existing.setProject(testProject); + existing.setStatus(RagIndexingStatus.INDEXING); + existing.setActiveJobId(92L); + OffsetDateTime previousActivity = OffsetDateTime.now().minusMinutes(5); + existing.setUpdatedAt(previousActivity); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) + .thenReturn(Optional.of(existing)); + + assertThat(service.markIndexingHeartbeat(testProject, 91L)).isFalse(); + assertThat(existing.getUpdatedAt()).isEqualTo(previousActivity); + assertThat(existing.getActiveJobId()).isEqualTo(92L); + verify(ragIndexStatusRepository, never()).save(any()); + } + @Test void testMarkIndexingHeartbeat_DoesNotMutateTerminalStatus() { RagIndexStatus existing = new RagIndexStatus(); @@ -239,7 +262,7 @@ void testMarkIndexingHeartbeat_DoesNotMutateTerminalStatus() { existing.setStatus(RagIndexingStatus.INDEXED); OffsetDateTime previousActivity = existing.getUpdatedAt(); - when(ragIndexStatusRepository.findByProjectId(100L)) + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) .thenReturn(Optional.of(existing)); assertThat(service.markIndexingHeartbeat(testProject)).isFalse(); @@ -249,7 +272,7 @@ void testMarkIndexingHeartbeat_DoesNotMutateTerminalStatus() { @Test void testMarkIndexingHeartbeat_WithoutStatusIsIgnored() { - when(ragIndexStatusRepository.findByProjectId(100L)) + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) .thenReturn(Optional.empty()); assertThat(service.markIndexingHeartbeat(testProject)).isFalse(); @@ -272,22 +295,23 @@ void testMarkUpdatingStarted_Success() { existing.setIndexedBranch("main"); existing.setIndexedCommitHash("abc123"); - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.of(existing)); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); - RagIndexStatus result = service.markUpdatingStarted(testProject, "main", "def456"); + RagIndexStatus result = service.markUpdatingStarted(testProject, "main", "def456", 93L); assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.UPDATING); assertThat(result.getIndexedBranch()).isEqualTo("main"); assertThat(result.getIndexedCommitHash()).isEqualTo("abc123"); assertThat(result.getErrorMessage()).isNull(); + assertThat(result.getActiveJobId()).isEqualTo(93L); } @Test void testMarkUpdatingStarted_ThrowsWhenNotFound() { - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.empty()); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.empty()); - assertThatThrownBy(() -> service.markUpdatingStarted(testProject, "main", "def456")) + assertThatThrownBy(() -> service.markUpdatingStarted(testProject, "main", "def456", 93L)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("Cannot update non-indexed project"); } @@ -299,24 +323,28 @@ void testMarkUpdatingCompleted_Success() { RagIndexStatus existing = new RagIndexStatus(); existing.setProject(testProject); existing.setStatus(RagIndexingStatus.UPDATING); + existing.setActiveJobId(93L); - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.of(existing)); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); - RagIndexStatus result = service.markUpdatingCompleted(testProject, "main", "ghi789", null, null, null); + RagIndexStatus result = service.markUpdatingCompleted( + testProject, "main", "ghi789", null, null, null, 93L); assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.INDEXED); assertThat(result.getIndexedBranch()).isEqualTo("main"); assertThat(result.getIndexedCommitHash()).isEqualTo("ghi789"); + assertThat(result.getActiveJobId()).isNull(); assertThat(result.getLastIndexedAt()).isNotNull(); assertThat(result.getErrorMessage()).isNull(); } @Test void testMarkUpdatingCompleted_ThrowsWhenNotFound() { - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.empty()); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.empty()); - assertThatThrownBy(() -> service.markUpdatingCompleted(testProject, "main", "ghi789", null, null, null)) + assertThatThrownBy(() -> service.markUpdatingCompleted( + testProject, "main", "ghi789", null, null, null, 93L)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("RAG index status not found"); } @@ -329,7 +357,7 @@ void testMarkUpdatingCompleted_NonBaseBranchPreservesProjectCheckpoint() { existing.setIndexedBranch("main"); existing.setIndexedCommitHash("main-commit"); - when(ragIndexStatusRepository.findByProjectId(100L)) + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) .thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))) .thenAnswer(i -> i.getArgument(0)); @@ -352,10 +380,12 @@ void testMarkIncrementalUpdateFailed_Success() { existing.setIndexedBranch("main"); existing.setIndexedCommitHash("abc123"); - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.of(existing)); + existing.setActiveJobId(93L); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); - RagIndexStatus result = service.markIncrementalUpdateFailed(testProject, "timeout error"); + RagIndexStatus result = service.markIncrementalUpdateFailed( + testProject, "timeout error", 93L); assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.INDEXED); assertThat(result.getIndexedBranch()).isEqualTo("main"); @@ -366,14 +396,88 @@ void testMarkIncrementalUpdateFailed_Success() { @Test void testMarkIncrementalUpdateFailed_ThrowsWhenNotFound() { - when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.empty()); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.empty()); - assertThatThrownBy(() -> service.markIncrementalUpdateFailed(testProject, "error")) + assertThatThrownBy(() -> service.markIncrementalUpdateFailed(testProject, "error", 93L)) .isInstanceOf(IllegalStateException.class); } + @Test + void staleCompletionCannotClearANewerJobOwner() { + RagIndexStatus existing = new RagIndexStatus(); + existing.setProject(testProject); + existing.setStatus(RagIndexingStatus.INDEXING); + existing.setActiveJobId(92L); + existing.setIndexedCommitHash("newer-commit"); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) + .thenReturn(Optional.of(existing)); + + RagIndexStatus result = service.markIndexingCompleted( + testProject, "main", "older-commit", 25, 50, 91L); + + assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.INDEXING); + assertThat(result.getActiveJobId()).isEqualTo(92L); + assertThat(result.getIndexedCommitHash()).isEqualTo("newer-commit"); + verify(ragIndexStatusRepository, never()).save(any()); + } + + @Test + void staleFailureCannotTerminalizeANewerJobOwner() { + RagIndexStatus existing = new RagIndexStatus(); + existing.setProject(testProject); + existing.setStatus(RagIndexingStatus.UPDATING); + existing.setActiveJobId(92L); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) + .thenReturn(Optional.of(existing)); + + RagIndexStatus result = service.markIncrementalUpdateFailed( + testProject, "older producer failed", 91L); + + assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.UPDATING); + assertThat(result.getActiveJobId()).isEqualTo(92L); + assertThat(result.getErrorMessage()).isNull(); + verify(ragIndexStatusRepository, never()).save(any()); + } + // ── canStartIndexing ───────────────────────────────────────────────────── + @Test + void succeededOperationRecoveryRequiresItsExactStatusOwner() { + RagIndexStatus ownerless = new RagIndexStatus(); + ownerless.setProject(testProject); + ownerless.setStatus(RagIndexingStatus.UPDATING); + ownerless.setIndexedCommitHash("newer-checkpoint"); + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) + .thenReturn(Optional.of(ownerless)); + + boolean reconciled = service.reconcilePublishedGeneration( + testProject, "main", "old-checkpoint", 10, 20, 91L); + + assertThat(reconciled).isFalse(); + assertThat(ownerless.getIndexedCommitHash()).isEqualTo("newer-checkpoint"); + assertThat(ownerless.getStatus()).isEqualTo(RagIndexingStatus.UPDATING); + verify(ragIndexStatusRepository, never()).save(any()); + } + + @Test + void sameRevisionLockOwnedReconciliationCanCreateMissingStatus() { + when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) + .thenReturn(Optional.empty()); + when(ragIndexStatusRepository.save(any(RagIndexStatus.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + + boolean reconciled = service.reconcilePublishedGeneration( + testProject, "main", "published-checkpoint", 10, 20); + + assertThat(reconciled).isTrue(); + ArgumentCaptor saved = ArgumentCaptor.forClass( + RagIndexStatus.class); + verify(ragIndexStatusRepository).save(saved.capture()); + assertThat(saved.getValue().getIndexedCommitHash()) + .isEqualTo("published-checkpoint"); + assertThat(saved.getValue().getStatus()).isEqualTo(RagIndexingStatus.INDEXED); + } + @Test void testCanStartIndexing_NoStatus() { when(ragIndexStatusRepository.findByProjectId(100L)).thenReturn(Optional.empty()); diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java index cbec2923..f130d89e 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java @@ -1,14 +1,20 @@ package org.rostilos.codecrow.ragengine.service; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.hibernate.LazyInitializationException; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; import org.rostilos.codecrow.core.model.branch.Branch; import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.project.config.RagConfig; @@ -23,14 +29,19 @@ import org.rostilos.codecrow.core.service.AnalysisJobService; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; +import org.rostilos.codecrow.ragengine.branch.BranchIndexBuildAdmissionService; +import org.rostilos.codecrow.ragengine.branch.LegacyRagJobLeaseService; +import org.rostilos.codecrow.ragengine.branch.LegacyRagUpdateCompletionService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.slf4j.LoggerFactory; import org.springframework.test.util.ReflectionTestUtils; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; @@ -65,6 +76,21 @@ class RagOperationsServiceImplTest { @Mock private RagPipelineClient ragPipelineClient; + @Mock + private BranchIndexBuildAdmissionService branchIndexBuildAdmissionService; + + @Mock + private LegacyRagJobLeaseService legacyRagJobLeaseService; + + @Mock + private LegacyRagJobLeaseService.JobLease legacyJobLease; + + @Mock + private AnalysisLockService.LockLease legacyLockLease; + + @Mock + private LegacyRagUpdateCompletionService legacyRagUpdateCompletionService; + private RagOperationsServiceImpl service; private Project testProject; @@ -77,7 +103,21 @@ void setUp() { analysisJobService, ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient); + ragPipelineClient, + null, + null, + null, + legacyRagJobLeaseService, + legacyRagUpdateCompletionService); + lenient().when(legacyRagJobLeaseService.start(anyLong())) + .thenReturn(legacyJobLease); + lenient().when(legacyJobLease.confirmOwnership()).thenReturn(true); + lenient().when(analysisLockService.maintainLockLease(anyString(), anyInt())) + .thenReturn(legacyLockLease); + lenient().when(legacyLockLease.confirmOwnership()).thenReturn(true); + lenient().when(legacyRagUpdateCompletionService.complete( + any(), anyString(), anyString(), anyLong(), any(), anyBoolean(), + anyInt(), anyInt(), any(), anySet())).thenReturn(true); ReflectionTestUtils.setField( service, "branchGenerationRepository", branchGenerationRepository); @@ -102,20 +142,23 @@ void exactFirstUseBuildsTargetSnapshotWithoutPrimaryToDevelopDiff() throws Excep ragIndexTrackingService, incrementalRagUpdateService, analysisLockService, analysisJobService, ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, registry, builder); + ragPipelineClient, registry, builder, branchIndexBuildAdmissionService); setupRagEnabled(); setupVcsBinding(); ReflectionTestUtils.setField(service, "ragApiEnabled", true); when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) .thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag("")) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of(), Set.of(), Set.of())); Job job = mock(Job.class); - when(job.getId()).thenReturn(77L); - when(analysisJobService.createRagIndexJob(any(), eq(false), any())) - .thenReturn(job); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 30L, "physical-target", false, null, "exact-feature-lock"); + when(branchIndexBuildAdmissionService.admit( + testProject, "feature", "develop-400", RagBranchIndexKind.DURABLE, + JobTriggerSource.WEBHOOK, "exact-feature-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.NONE)); when(analysisLockService.acquireLock( eq(testProject), eq("feature"), any(), eq("develop-400"), isNull())) .thenReturn(Optional.of("exact-feature-lock")); @@ -123,13 +166,11 @@ void exactFirstUseBuildsTargetSnapshotWithoutPrimaryToDevelopDiff() throws Excep when(vcsClientProvider.getClient(any())).thenReturn(vcs); when(vcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) .thenReturn("develop-400"); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.empty()); - when(builder.build( + when(builder.execute( eq(testProject), any(), eq("my-workspace"), eq("my-repo"), eq("feature"), eq("develop-400"), eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), - any(), any(), eq(77L))) + nullable(List.class), nullable(List.class), eq(prepared), isNull())) .thenReturn(Map.of( "generation_manifest_sha256", "manifest-400", "document_count", 231, @@ -143,11 +184,434 @@ void exactFirstUseBuildsTargetSnapshotWithoutPrimaryToDevelopDiff() throws Excep assertThat(ready).isTrue(); verify(vcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - verify(builder).build( + verify(builder).execute( eq(testProject), any(), eq("my-workspace"), eq("my-repo"), eq("feature"), eq("develop-400"), eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), - any(), any(), eq(77L)); + nullable(List.class), nullable(List.class), eq(prepared), isNull()); + verify(branchIndexBuildAdmissionService).admit( + testProject, "feature", "develop-400", RagBranchIndexKind.DURABLE, + JobTriggerSource.WEBHOOK, "exact-feature-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + verifyNoInteractions(registry); + } + + @Test + void exactMismatchUsesScalarProjectionWithoutTouchingDetachedActiveGenerationProxy() + throws Exception { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, registry, builder, branchIndexBuildAdmissionService); + setupRagEnabled(); + setupVcsBinding(); + setupProjectWithWorkspaceAndNamespace(); + + when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "main")) + .thenReturn(true); + RagBranchIndex detachedIndex = mock(RagBranchIndex.class); + RagBranchIndexGeneration detachedGeneration = mock( + RagBranchIndexGeneration.class); + lenient().when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "main")) + .thenReturn(Optional.of(detachedIndex)); + lenient().when(detachedIndex.getActiveGeneration()).thenReturn(detachedGeneration); + lenient().when(detachedGeneration.getRevision()).thenThrow( + new LazyInitializationException("no Session")); + when(ragBranchIndexRepository.markAccessedIfUnclaimed( + eq(100L), eq("main"), any())) + .thenReturn(1); + RagBranchIndexRepository.ActiveGenerationCoordinates source = + mock(RagBranchIndexRepository.ActiveGenerationCoordinates.class); + // Simulate returning A after B became active. An older successful A + // operation may exist, so automatic reconciliation must force a fresh + // generation instead of reusing the superseded one. + when(source.getRevision()).thenReturn("revision-b"); + AtomicBoolean lockAcquired = new AtomicBoolean(); + when(ragBranchIndexRepository.findActiveGenerationCoordinates(100L, "main")) + .thenAnswer(ignored -> { + assertThat(lockAcquired.get()).isTrue(); + return Optional.of(source); + }); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 31L, "target-generation", false, null, "rag-lock"); + when(branchIndexBuildAdmissionService.admit( + testProject, "main", "revision-a", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "rag-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.UPDATING)); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("revision-a"), isNull())) + .thenAnswer(ignored -> { + lockAcquired.set(true); + return Optional.of("rag-lock"); + }); + + when(builder.execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull())) + .thenReturn(Map.of( + "generation_manifest_sha256", "target-manifest", + "document_count", 214, + "chunk_count", 642, + "deletedFiles", 0)); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "revision-a", "caller diff", ignored -> { }); + + assertThat(result).isTrue(); + var sourceBindingOrder = inOrder( + analysisLockService, ragBranchIndexRepository, builder); + sourceBindingOrder.verify(analysisLockService).acquireLock( + eq(testProject), eq("main"), any(), eq("revision-a"), isNull()); + sourceBindingOrder.verify(ragBranchIndexRepository) + .findActiveGenerationCoordinates(100L, "main"); + sourceBindingOrder.verify(builder).execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull()); + verifyNoInteractions(vcsClientProvider, registry); + verify(incrementalRagUpdateService, never()).parseDiffForRag("caller diff"); + verify(analysisJobService).info(eq(job), eq("rag_init"), + contains("Starting exact RAG generation rebuild")); + verify(analysisJobService).info(eq(job), eq("rag_complete"), + contains("214 documents, 642 chunks")); + verify(branchIndexBuildAdmissionService).admit( + testProject, "main", "revision-a", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "rag-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); + verify(ragBranchIndexRepository, never()) + .findByProjectIdAndBranchName(anyLong(), anyString()); + verify(detachedIndex, never()).getActiveGeneration(); + verify(detachedGeneration, never()).getRevision(); + verify(ragBranchIndexRepository, never()).save(any()); + } + + @Test + void exactPublicationProjectionFailureLeavesSucceededOperationOwnedForRecovery() + throws Exception { + BranchIndexGenerationBuildService builder = mock( + BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + builder, branchIndexBuildAdmissionService); + setupRagEnabled(); + setupVcsBinding(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("revision-published"), isNull())) + .thenReturn(Optional.of("rag-lock")); + Job job = mock(Job.class); + when(job.getId()).thenReturn(91L); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 81L, "physical-generation", false, null, "rag-lock"); + when(branchIndexBuildAdmissionService.admit( + testProject, "main", "revision-published", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "rag-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); + when(builder.execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-published"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull())) + .thenReturn(Map.of("document_count", 12, "chunk_count", 34)); + when(ragIndexTrackingService.reconcilePublishedGeneration( + testProject, "main", "revision-published", 12, 34, 91L)) + .thenThrow(new IllegalStateException("status database unavailable")); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "revision-published", "ignored", ignored -> { }); + + assertThat(result).isFalse(); + verify(builder).execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-published"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull()); + verify(branchIndexBuildAdmissionService, never()).abortOperation(any(), anyString()); + verify(analysisJobService, never()).failJob(any(), anyString()); + verify(ragIndexTrackingService, never()).markIndexingFailed(any(), anyString(), any()); + verify(ragIndexTrackingService, never()).markIncrementalUpdateFailed( + any(), anyString(), any()); + verify(analysisLockService).releaseLock("rag-lock"); + } + + @Test + void exactLockCleanupFailureCannotReverseSuccessfulPublicationAndJob() + throws Exception { + BranchIndexGenerationBuildService builder = mock( + BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + builder, branchIndexBuildAdmissionService); + setupRagEnabled(); + setupVcsBinding(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("revision-complete"), isNull())) + .thenReturn(Optional.of("rag-lock")); + Job job = mock(Job.class); + when(job.getId()).thenReturn(91L); + var prepared = new BranchIndexGenerationBuildService.PreparedBuild( + 81L, "physical-generation", false, null, "rag-lock"); + when(branchIndexBuildAdmissionService.admit( + testProject, "main", "revision-complete", RagBranchIndexKind.PRIMARY, + JobTriggerSource.WEBHOOK, "rag-lock", + BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) + .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( + job, prepared, + BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); + when(builder.execute( + eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), + eq("main"), eq("revision-complete"), eq(RagBranchIndexKind.PRIMARY), + nullable(List.class), nullable(List.class), eq(prepared), isNull())) + .thenReturn(Map.of("document_count", 12, "chunk_count", 34)); + doThrow(new IllegalStateException("lock database unavailable")) + .when(analysisLockService).releaseLock("rag-lock"); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "revision-complete", "ignored", ignored -> { }); + + assertThat(result).isTrue(); + verify(ragIndexTrackingService).reconcilePublishedGeneration( + testProject, "main", "revision-complete", 12, 34, 91L); + verify(analysisJobService).completeJob(job, null); + verify(analysisJobService, never()).failJob(any(), anyString()); + verify(analysisLockService).releaseLock("rag-lock"); + } + + @Test + void exactIncrementalLockContentionSkipsBeforeBindingSource() { + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class)); + setupRagEnabled(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("target-revision"), isNull())) + .thenReturn(Optional.empty()); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "target-revision", "caller diff", eventConsumer); + + assertThat(result).isFalse(); + verifyNoInteractions(analysisJobService); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); + verify(eventConsumer).accept(argThat(event -> "rag_skip".equals(event.get("state")))); + } + + @Test + void exactIncrementalCleanupClaimSkipsWithoutReadingOrMutatingGeneration() { + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class)); + setupRagEnabled(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("feature"), any(), eq("target-revision"), isNull())) + .thenReturn(Optional.of("rag-lock")); + when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "feature")) + .thenReturn(true); + when(ragBranchIndexRepository.markAccessedIfUnclaimed( + eq(100L), eq("feature"), any())) + .thenReturn(0); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.triggerIncrementalUpdate( + testProject, "feature", "target-revision", "caller diff", eventConsumer); + + assertThat(result).isFalse(); + verifyNoInteractions(analysisJobService); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); + verify(analysisLockService).releaseLock("rag-lock"); + verify(eventConsumer).accept(argThat(event -> "rag_skipped".equals(event.get("state")))); + } + + @Test + void exactIncrementalAlreadyAtTargetCompletesJobAndRepairsPrimaryCheckpoint() + throws Exception { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, registry, builder); + setupRagEnabled(); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(analysisLockService.acquireLock( + eq(testProject), eq("main"), any(), eq("target-revision"), isNull())) + .thenReturn(Optional.of("rag-lock")); + when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "main")) + .thenReturn(true); + when(ragBranchIndexRepository.markAccessedIfUnclaimed( + eq(100L), eq("main"), any())) + .thenReturn(1); + RagBranchIndexRepository.ActiveGenerationCoordinates source = + mock(RagBranchIndexRepository.ActiveGenerationCoordinates.class); + when(source.getRevision()).thenReturn("target-revision"); + when(ragBranchIndexRepository.findActiveGenerationCoordinates(100L, "main")) + .thenReturn(Optional.of(source)); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "target-revision", "stale caller diff", eventConsumer); + + assertThat(result).isTrue(); + verify(ragIndexTrackingService).preparePublishedGenerationForUpdate( + testProject, "main", "target-revision", 0, 0); + verifyNoInteractions(analysisJobService); + verify(analysisLockService).releaseLock("rag-lock"); + verifyNoInteractions(vcsClientProvider, registry, builder); + verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); + verify(incrementalRagUpdateService, never()).performIncrementalUpdate( + any(), any(), anyString(), anyString(), anyString(), anyString(), + anySet(), anySet(), anySet(), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); + verify(eventConsumer).accept(argThat(event -> + "rag_complete".equals(event.get("state")))); + } + + @Test + void ensureExactMainAlwaysDelegatesToLockedTriggerWithoutPrediff() throws Exception { + service = spy(new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class))); + setupRagEnabled(); + setupVcsBinding(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + VcsClient vcsClient = mock(VcsClient.class); + when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); + when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "main")) + .thenReturn("main-head"); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("main"), eq("main-head"), eq(""), any()); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); + + assertThat(result).isTrue(); + verify(service).triggerIncrementalUpdate( + testProject, "main", "main-head", "", eventConsumer); + verify(ragIndexTrackingService, never()).getIndexStatus(testProject); + verify(vcsClient, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(ragBranchIndexRepository, never()) + .findByProjectIdAndBranchName(anyLong(), anyString()); + } + + @Test + void ensureExactBranchDelegatesMainAndBranchToLockedTriggersWithoutPrediff() + throws Exception { + service = spy(new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class))); + setupRagEnabled(); + setupVcsBinding(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + VcsClient vcsClient = mock(VcsClient.class); + when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); + when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "main")) + .thenReturn("main-head"); + when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "feature")) + .thenReturn("feature-head"); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("main"), eq("main-head"), eq(""), any()); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("feature"), eq("feature-head"), eq(""), any()); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.ensureRagIndexUpToDate(testProject, "feature", eventConsumer); + + assertThat(result).isTrue(); + verify(service).triggerIncrementalUpdate( + testProject, "feature", "feature-head", "", eventConsumer); + verify(service).triggerIncrementalUpdate( + testProject, "main", "main-head", "", eventConsumer); + verify(ragIndexTrackingService, never()).getIndexStatus(testProject); + verify(vcsClient, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(ragBranchIndexRepository, never()) + .findByProjectIdAndBranchName(anyLong(), anyString()); + verify(ragBranchIndexRepository, never()).save(any()); + } + + @Test + void updateExactBranchDelegatesToLockedTriggerWithoutTrustingMutableCheckpoint() + throws Exception { + service = spy(new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, mock(RagBranchIndexRegistryService.class), + mock(BranchIndexGenerationBuildService.class))); + setupRagEnabled(); + setupVcsBinding(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + VcsClient vcsClient = mock(VcsClient.class); + when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); + when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "feature")) + .thenReturn("feature-head"); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("feature"), eq("feature-head"), eq(""), any()); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); + + assertThat(result).isTrue(); + verify(service).triggerIncrementalUpdate( + testProject, "feature", "feature-head", "", eventConsumer); + verify(ragBranchIndexRepository, never()) + .findByProjectIdAndBranchName(anyLong(), anyString()); + verify(ragBranchIndexRepository, never()) + .findActiveGenerationCoordinates(anyLong(), anyString()); + verify(vcsClient, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); } @Test @@ -398,7 +862,9 @@ void testDeleteBranchIndex_NoVcsBinding() { void testDeleteBranchIndex_Success() throws Exception { setupRagEnabled(); setupVcsBinding(); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "feature")).thenReturn(true); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "feature", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("legacy-alias")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -412,7 +878,11 @@ void testDeleteBranchIndex_Success() throws Exception { void testDeleteBranchIndex_PipelineFailure() throws Exception { setupRagEnabled(); setupVcsBinding(); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "feature")).thenReturn(false); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "feature", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "legacy-alias", RagPipelineClient.BranchDeletionFailure.TARGET, + 404, "not found")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -422,11 +892,86 @@ void testDeleteBranchIndex_PipelineFailure() throws Exception { verify(ragBranchIndexRepository, never()).deleteByProjectIdAndBranchName(anyLong(), anyString()); } + @Test + void globalRagDisablementRetainsBranchWithoutCleanupWarning() { + setupRagEnabled(); + setupVcsBinding(); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "feature", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "legacy-alias", + RagPipelineClient.BranchDeletionFailure.TARGET, + null, + "RAG disabled")); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + Logger logger = (Logger) LoggerFactory.getLogger(RagOperationsServiceImpl.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + boolean result; + try { + result = service.deleteBranchIndex(testProject, "feature", eventConsumer); + } finally { + logger.detachAppender(appender); + } + + assertThat(result).isFalse(); + verify(ragBranchIndexRepository, never()) + .deleteByProjectIdAndBranchName(anyLong(), anyString()); + assertThat(appender.list).noneMatch(event -> event.getLevel() == Level.WARN + && event.getFormattedMessage().contains("delete branch RAG generation")); + } + + @Test + void exactDeletionKeepsActiveTargetWhenOlderTargetIsRejected() { + setupRagEnabled(); + setupVcsBinding(); + setupProjectWithWorkspaceAndNamespace(); + RagBranchIndex branchIndex = new RagBranchIndex( + testProject, "feature", RagBranchIndexKind.DURABLE); + branchIndex.setId(501L); + RagBranchIndexGeneration rejected = new RagBranchIndexGeneration(); + rejected.setCollectionName("superseded-target"); + rejected.setRevision("revision-1"); + rejected.activate("manifest-1", 1, 1); + rejected.supersede(); + RagBranchIndexGeneration active = new RagBranchIndexGeneration(); + active.setCollectionName("active-target"); + active.setRevision("revision-2"); + active.activate("manifest-2", 1, 1); + when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) + .thenReturn(Optional.of(branchIndex)); + when(branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc(501L)) + .thenReturn(List.of(active, rejected)); + when(ragPipelineClient.deleteBranchWithOutcome( + "test-ws", "test-ns", "feature", "superseded-target", + "revision-1", "manifest-1")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "superseded-target", + RagPipelineClient.BranchDeletionFailure.TARGET, + 422, + "manifest receipt rejected")); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.deleteBranchIndex(testProject, "feature", eventConsumer); + + assertThat(result).isFalse(); + verify(ragPipelineClient, never()).deleteBranchWithOutcome( + "test-ws", "test-ns", "feature", "active-target", + "revision-2", "manifest-2"); + verify(ragBranchIndexRepository, never()) + .deleteByProjectIdAndBranchName(anyLong(), anyString()); + } + @Test void testDeleteBranchIndex_PipelineException() throws Exception { setupRagEnabled(); setupVcsBinding(); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "feature")) + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "feature", null)) .thenThrow(new RuntimeException("Connection timeout")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -482,7 +1027,9 @@ void testCleanupStaleBranches_DeletesStaleBranch() throws Exception { setupVcsBinding(); when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) .thenReturn(java.util.List.of("main", "feature", "stale-branch")); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "stale-branch")).thenReturn(true); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale-branch", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("legacy-alias")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -508,18 +1055,24 @@ void cleanupStaleBranchesDeletesEveryRegisteredExactGeneration() throws Exceptio branchIndex.setId(501L); RagBranchIndexGeneration first = new RagBranchIndexGeneration(); first.setCollectionName("cc_generation_1"); + first.setRevision("revision-1"); + first.setManifestDigest("manifest-1"); RagBranchIndexGeneration second = new RagBranchIndexGeneration(); second.setCollectionName("cc_generation_2"); + second.setRevision("revision-2"); + second.setManifestDigest("manifest-2"); when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "stale-exact")) .thenReturn(Optional.of(branchIndex)); when(branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc(501L)) .thenReturn(List.of(first, second)); - when(ragPipelineClient.deleteBranch( - "test-ws", "test-ns", "stale-exact", "cc_generation_1")) - .thenReturn(true); - when(ragPipelineClient.deleteBranch( - "test-ws", "test-ns", "stale-exact", "cc_generation_2")) - .thenReturn(true); + when(ragPipelineClient.deleteBranchWithOutcome( + "test-ws", "test-ns", "stale-exact", "cc_generation_1", + "revision-1", "manifest-1")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("cc_generation_1")); + when(ragPipelineClient.deleteBranchWithOutcome( + "test-ws", "test-ns", "stale-exact", "cc_generation_2", + "revision-2", "manifest-2")) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("cc_generation_2")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -529,14 +1082,16 @@ void cleanupStaleBranchesDeletesEveryRegisteredExactGeneration() throws Exceptio assertThat(result).containsEntry("status", "success"); assertThat(result).containsEntry("total_deleted", 1); assertThat(result.get("deleted_branches")).isEqualTo(List.of("stale-exact")); - verify(ragPipelineClient).deleteBranch( - "test-ws", "test-ns", "stale-exact", "cc_generation_1"); - verify(ragPipelineClient).deleteBranch( - "test-ws", "test-ns", "stale-exact", "cc_generation_2"); + verify(ragPipelineClient).deleteBranchWithOutcome( + "test-ws", "test-ns", "stale-exact", "cc_generation_1", + "revision-1", "manifest-1"); + verify(ragPipelineClient).deleteBranchWithOutcome( + "test-ws", "test-ns", "stale-exact", "cc_generation_2", + "revision-2", "manifest-2"); verify(ragBranchIndexRepository) .deleteByProjectIdAndBranchName(100L, "stale-exact"); - verify(ragPipelineClient, never()) - .deleteBranch("my-workspace", "my-repo", "stale-exact"); + verify(ragPipelineClient, never()).deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale-exact", null); } @Test @@ -656,12 +1211,14 @@ void testTriggerIncrementalUpdate_FullSuccessFlow() throws Exception { @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(77L); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); when(incrementalRagUpdateService.parseDiffForRag("diff content")) .thenReturn(new IncrementalRagUpdateService.DiffResult(Set.of("src/A.java"), java.util.Collections.emptySet(), Set.of("src/B.java"))); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())).thenReturn(mockJob); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); when(analysisLockService.acquireLock(any(), eq("feature"), any(), eq("commit1"), isNull())) .thenReturn(Optional.of("lock-key")); when(incrementalRagUpdateService.performIncrementalUpdate( @@ -675,12 +1232,105 @@ void testTriggerIncrementalUpdate_FullSuccessFlow() throws Exception { testProject, "feature", "commit1", "diff content", eventConsumer); assertThat(result).isTrue(); - verify(ragIndexTrackingService).markUpdatingStarted(testProject, "feature", "commit1"); - verify(ragIndexTrackingService).markUpdatingCompleted( - testProject, "feature", "commit1", 0, 1, null, false); + verifyNoInteractions(ragIndexTrackingService); verify(analysisLockService).releaseLock("lock-key"); - verify(analysisJobService).completeJob(eq(mockJob), isNull()); - verify(ragBranchIndexRepository).save(any(RagBranchIndex.class)); + verify(analysisJobService, never()).completeJob(eq(mockJob), isNull()); + verify(analysisJobService).recordExternallyCompletedJob( + eq(mockJob), eq("rag_complete"), contains("RAG index updated")); + verify(legacyRagUpdateCompletionService).complete( + eq(testProject), eq("feature"), eq("commit1"), eq(77L), + any(), eq(false), eq(0), eq(1), isNull(), eq(Set.of("src/B.java"))); + verify(legacyRagJobLeaseService).start(mockJob.getId()); + verify(analysisLockService).maintainLockLease("lock-key", 360); + verify(legacyJobLease).confirmOwnership(); + verify(legacyLockLease).confirmOwnership(); + verify(legacyJobLease).close(); + verify(legacyLockLease).close(); + } + + @Test + void legacyOwnershipLossBeforeRemoteWorkDoesNotMutateOrOverwriteRecovery() + throws Exception { + setupRagEnabled(); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag("diff")) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of("src/A.java"), Set.of(), Set.of())); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())) + .thenReturn(job); + when(analysisLockService.acquireLock( + any(), eq("feature"), any(), eq("commit1"), isNull())) + .thenReturn(Optional.of("lock-key")); + when(legacyJobLease.isOwnershipLost()).thenReturn(true); + @SuppressWarnings("unchecked") + Consumer> events = mock(Consumer.class); + + boolean result = service.triggerIncrementalUpdate( + testProject, "feature", "commit1", "diff", events); + + assertThat(result).isFalse(); + verify(incrementalRagUpdateService, never()).performIncrementalUpdate( + any(), any(), anyString(), anyString(), anyString(), anyString(), + any(), any(), any()); + verify(analysisJobService, never()).failJob(any(), anyString()); + verifyNoInteractions(ragIndexTrackingService); + verify(legacyJobLease).close(); + verify(legacyLockLease).close(); + verify(analysisLockService).releaseLock("lock-key"); + verify(events).accept(argThat(event -> + "rag_error".equals(event.get("state")) + && String.valueOf(event.get("message")) + .contains("lost durable ownership"))); + } + + @Test + void legacyOwnershipIsReconfirmedBeforeCheckpointAndJobCompletion() + throws Exception { + setupRagEnabled(); + setupVcsBinding(); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag("diff")) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of("src/A.java"), Set.of(), Set.of())); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())) + .thenReturn(job); + when(analysisLockService.acquireLock( + any(), eq("main"), any(), eq("commit1"), isNull())) + .thenReturn(Optional.of("lock-key")); + when(incrementalRagUpdateService.performIncrementalUpdate( + any(), any(), anyString(), anyString(), anyString(), anyString(), + any(), any(), any())) + .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0)); + when(legacyLockLease.confirmOwnership()).thenReturn(false); + + boolean result = service.triggerIncrementalUpdate( + testProject, "main", "commit1", "diff", ignored -> { }); + + assertThat(result).isFalse(); + verify(incrementalRagUpdateService).performIncrementalUpdate( + any(), any(), anyString(), anyString(), eq("main"), eq("commit1"), + any(), any(), any()); + verify(legacyJobLease).confirmOwnership(); + verify(legacyLockLease).confirmOwnership(); + verify(ragIndexTrackingService, never()).markUpdatingCompleted( + any(), anyString(), anyString(), any(), any(), any(), any()); + verify(legacyRagUpdateCompletionService, never()).complete( + any(), anyString(), anyString(), anyLong(), any(), anyBoolean(), + anyInt(), anyInt(), any(), anySet()); + verify(analysisJobService, never()).completeJob(any(), any()); + verify(analysisJobService, never()).failJob(any(), anyString()); + verify(ragIndexTrackingService, never()).markIncrementalUpdateFailed( + any(), anyString(), any()); + verify(legacyJobLease).close(); + verify(legacyLockLease).close(); } @Test @@ -712,7 +1362,8 @@ void testTriggerIncrementalUpdate_LockNotAcquired() { when(incrementalRagUpdateService.parseDiffForRag(anyString())) .thenReturn(new IncrementalRagUpdateService.DiffResult(Set.of("a.java"), java.util.Collections.emptySet(), java.util.Collections.emptySet())); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())).thenReturn(mockJob); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) .thenReturn(Optional.empty()); @@ -720,10 +1371,40 @@ void testTriggerIncrementalUpdate_LockNotAcquired() { service.triggerIncrementalUpdate(testProject, "feature", "c1", "diff", eventConsumer); assertThat(result).isFalse(); - verify(analysisJobService).failJob(eq(mockJob), anyString()); + verify(analysisJobService).skipJob( + eq(mockJob), contains("previous checkpoint is retained for the next trigger")); + verify(analysisJobService, never()).failJob(any(), anyString()); + verify(eventConsumer).accept(argThat(event -> + "rag_skip".equals(event.get("state")) + && String.valueOf(event.get("message")).contains("next trigger"))); verifyNoInteractions(ragIndexTrackingService); } + @Test + void lockContentionObserverFailureCannotChangeSkippedTerminalState() { + setupRagEnabled(); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + Job mockJob = mock(Job.class); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag(anyString())) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of("a.java"), Set.of(), Set.of())); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); + when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) + .thenReturn(Optional.empty()); + doThrow(new IllegalStateException("observer disconnected")) + .when(eventConsumer).accept(anyMap()); + + boolean result = service.triggerIncrementalUpdate( + testProject, "feature", "c1", "diff", eventConsumer); + + assertThat(result).isFalse(); + verify(analysisJobService).skipJob(eq(mockJob), anyString()); + verify(analysisJobService, never()).failJob(any(), anyString()); + } + @Test void testTriggerIncrementalUpdate_IncrementalUpdateThrows() throws Exception { setupRagEnabled(); @@ -731,12 +1412,14 @@ void testTriggerIncrementalUpdate_IncrementalUpdateThrows() throws Exception { @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(78L); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); when(incrementalRagUpdateService.parseDiffForRag(anyString())) .thenReturn(new IncrementalRagUpdateService.DiffResult(Set.of("a.java"), java.util.Collections.emptySet(), java.util.Collections.emptySet())); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())).thenReturn(mockJob); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) .thenReturn(Optional.of("lock-key")); when(incrementalRagUpdateService.performIncrementalUpdate( @@ -748,9 +1431,15 @@ void testTriggerIncrementalUpdate_IncrementalUpdateThrows() throws Exception { service.triggerIncrementalUpdate(testProject, "feature", "c1", "diff", eventConsumer); assertThat(result).isFalse(); - // Should call markIncrementalUpdateFailed (keeps status INDEXED) NOT markIndexingFailed - verify(ragIndexTrackingService).markIncrementalUpdateFailed(eq(testProject), anyString()); - verify(ragIndexTrackingService, never()).markIndexingFailed(any(), anyString()); + // A retained branch has its own durable operation state and cannot + // overwrite the primary branch's project-level status. + verifyNoInteractions(ragIndexTrackingService); + verify(analysisJobService).failJob( + eq(mockJob), contains("RAG incremental update failed: Pipeline down")); + verify(analysisJobService, never()).error(eq(mockJob), eq("rag_error"), anyString()); + verify(eventConsumer).accept(argThat(event -> + "rag_error".equals(event.get("state")) + && "warning".equals(event.get("type")))); verify(analysisLockService).releaseLock("lock-key"); } @@ -762,6 +1451,7 @@ void testTriggerIncrementalUpdate_UsesCompletedMainCheckpointAfterEarlierFailure @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(79L); VcsClient vcsClient = mock(VcsClient.class); RagIndexStatus completedStatus = new RagIndexStatus(); completedStatus.setIndexedBranch("main"); @@ -780,7 +1470,8 @@ void testTriggerIncrementalUpdate_UsesCompletedMainCheckpointAfterEarlierFailure when(incrementalRagUpdateService.parseDiffForRag(catchUpDiff)) .thenReturn(new IncrementalRagUpdateService.DiffResult( Set.of("src/Recovered.java"), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())) + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())) .thenReturn(mockJob); when(analysisLockService.acquireLock( any(), eq("main"), any(), eq("current-head"), isNull())) @@ -789,9 +1480,6 @@ void testTriggerIncrementalUpdate_UsesCompletedMainCheckpointAfterEarlierFailure any(), any(), anyString(), anyString(), anyString(), anyString(), any(), any(), any())) .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0)); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "main")) - .thenReturn(Optional.empty()); - boolean result = service.triggerIncrementalUpdate( testProject, "main", "current-head", "diff --git a/src/OnlyLatest.java b/src/OnlyLatest.java\n+latest\n", @@ -803,8 +1491,9 @@ void testTriggerIncrementalUpdate_UsesCompletedMainCheckpointAfterEarlierFailure eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), eq("main"), eq("current-head"), eq(Set.of("src/Recovered.java")), eq(Set.of()), eq(Set.of())); - verify(ragIndexTrackingService).markUpdatingCompleted( - testProject, "main", "current-head", 0, 0, null, true); + verify(legacyRagUpdateCompletionService).complete( + eq(testProject), eq("main"), eq("current-head"), eq(79L), + any(), eq(true), eq(0), eq(0), isNull(), eq(Set.of())); } @Test @@ -814,12 +1503,14 @@ void testTriggerIncrementalUpdate_TrackBranchIndex_MergesDeletedFiles() throws E @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(80L); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); when(incrementalRagUpdateService.parseDiffForRag(anyString())) .thenReturn(new IncrementalRagUpdateService.DiffResult(java.util.Collections.emptySet(), java.util.Collections.emptySet(), Set.of("old.java"))); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())).thenReturn(mockJob); + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) .thenReturn(Optional.of("lock-key")); when(incrementalRagUpdateService.performIncrementalUpdate( @@ -835,8 +1526,9 @@ void testTriggerIncrementalUpdate_TrackBranchIndex_MergesDeletedFiles() throws E service.triggerIncrementalUpdate(testProject, "feature", "c1", "diff", eventConsumer); - verify(ragBranchIndexRepository).save(argThat( - idx -> idx.getDeletedFiles().contains("old.java") && idx.getDeletedFiles().contains("prev.java"))); + verify(legacyRagUpdateCompletionService).complete( + eq(testProject), eq("feature"), eq("c1"), eq(80L), + any(), eq(false), eq(0), eq(1), isNull(), eq(Set.of("old.java"))); } // ── updateBranchIndex full-flow tests ─────────────────────────────── @@ -870,7 +1562,8 @@ void updateBranchIndexUsesCompletedBranchCheckpointWithoutComparingMain() throws when(incrementalRagUpdateService.parseDiffForRag("checkpoint diff")) .thenReturn(new IncrementalRagUpdateService.DiffResult( Set.of(), Set.of("src/Changed.java"), Set.of())); - when(analysisJobService.createRagIndexJob(eq(testProject), eq(false), any())).thenReturn(mock(Job.class)); + when(analysisJobService.createRagIndexJob( + eq(testProject), eq(false), any(), anyString(), anyString())).thenReturn(mock(Job.class)); when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) .thenReturn(Optional.of("lock")); when(incrementalRagUpdateService.performIncrementalUpdate( @@ -926,7 +1619,6 @@ void updateBranchIndexEmptyDiffSeedsExactSnapshot() throws Exception { when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); VcsClient mockVcs = mock(VcsClient.class); when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getBranchDiff("my-workspace", "my-repo", "main", "feature")).thenReturn(""); when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) .thenReturn("feature-head"); doReturn(true).when(service).triggerIncrementalUpdate( @@ -937,7 +1629,6 @@ void updateBranchIndexEmptyDiffSeedsExactSnapshot() throws Exception { boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); assertThat(result).isTrue(); - verify(eventConsumer).accept(argThat(m -> "info".equals(m.get("type")))); verify(service).triggerIncrementalUpdate( testProject, "feature", "feature-head", "", eventConsumer); } @@ -1121,12 +1812,14 @@ void testEnsureRagIndexUpToDate_DifferentBranch_NoBranchIndex() throws Exception private void stubSuccessfulIncrementalUpdate(String branchName, String commitHash) throws Exception { Job mockJob = mock(Job.class); + when(mockJob.getId()).thenReturn(81L); when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) .thenReturn(true); when(incrementalRagUpdateService.parseDiffForRag(anyString())) .thenReturn(new IncrementalRagUpdateService.DiffResult( Set.of("src/A.java"), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob(any(), anyBoolean(), any())) + when(analysisJobService.createRagIndexJob( + any(), anyBoolean(), any(), anyString(), anyString())) .thenReturn(mockJob); when(analysisLockService.acquireLock( any(), eq(branchName), any(), eq(commitHash), isNull())) @@ -1260,8 +1953,14 @@ void testCleanupStaleBranches_PartialFailure() throws Exception { setupVcsBinding(); when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) .thenReturn(List.of("main", "stale1", "stale2")); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "stale1")).thenReturn(true); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "stale2")).thenReturn(false); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale1", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("legacy-alias")); + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale2", null)) + .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( + "legacy-alias", RagPipelineClient.BranchDeletionFailure.TARGET, + 404, "not found")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -1281,7 +1980,8 @@ void testCleanupStaleBranches_DeleteThrows() throws Exception { setupVcsBinding(); when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) .thenReturn(List.of("main", "stale1")); - when(ragPipelineClient.deleteBranch("my-workspace", "my-repo", "stale1")) + when(ragPipelineClient.deleteBranchWithOutcome( + "my-workspace", "my-repo", "stale1", null)) .thenThrow(new RuntimeException("Connection error")); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -1327,19 +2027,119 @@ private void setupProjectWithWorkspaceAndNamespace() { void testDeletePrFiles_Success() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); setupProjectWithWorkspaceAndNamespace(); - when(ragPipelineClient.deletePrFiles("test-ws", "test-ns", 42)).thenReturn(true); + when(ragPipelineClient.deletePrFilesWithOutcome("test-ws", "test-ns", 42, null)) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("legacy-alias")); + + boolean result = service.deletePrFiles(testProject, 42); + + assertThat(result).isTrue(); + verify(ragPipelineClient).deletePrFilesWithOutcome("test-ws", "test-ns", 42, null); + } + + @Test + void deletePrFilesCleansEveryPublishedGenerationAndDeduplicatesTargets() { + ReflectionTestUtils.setField(service, "ragApiEnabled", true); + setupProjectWithWorkspaceAndNamespace(); + when(branchGenerationRepository.findCollectionNamesByProjectIdAndStatusIn( + eq(100L), + eq(List.of( + org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus.ACTIVE, + org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus.SUPERSEDED)))) + .thenReturn(List.of("generation-a", "generation-a", " ", "generation-b")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("generation-a")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("generation-b")); boolean result = service.deletePrFiles(testProject, 42); assertThat(result).isTrue(); - verify(ragPipelineClient).deletePrFiles("test-ws", "test-ns", 42); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a"); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b"); + verify(ragPipelineClient, never()).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, null); + } + + @Test + void deletePrFilesContinuesAfterOneTargetSpecificRejection() { + ReflectionTestUtils.setField(service, "ragApiEnabled", true); + setupProjectWithWorkspaceAndNamespace(); + when(branchGenerationRepository.findCollectionNamesByProjectIdAndStatusIn( + eq(100L), anyList())) + .thenReturn(List.of("generation-a", "generation-b")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.failure( + "generation-a", + RagPipelineClient.PrFilesDeletionFailure.TARGET, + 404, + "collection missing")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("generation-b")); + + boolean result = service.deletePrFiles(testProject, 42); + + assertThat(result).isFalse(); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a"); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b"); + } + + @Test + void deletePrFilesStopsAfterServiceFailureAndNamesFailingTarget() { + ReflectionTestUtils.setField(service, "ragApiEnabled", true); + setupProjectWithWorkspaceAndNamespace(); + when(branchGenerationRepository.findCollectionNamesByProjectIdAndStatusIn( + eq(100L), anyList())) + .thenReturn(List.of("generation-a", "generation-b")); + when(ragPipelineClient.deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a")) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.failure( + "generation-a", + RagPipelineClient.PrFilesDeletionFailure.SERVICE, + 409, + "mutation lease unavailable")); + + Logger logger = (Logger) LoggerFactory.getLogger(RagOperationsServiceImpl.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + boolean result; + try { + result = service.deletePrFiles(testProject, 42); + } finally { + logger.detachAppender(appender); + appender.stop(); + } + + assertThat(result).isFalse(); + verify(ragPipelineClient).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-a"); + verify(ragPipelineClient, never()).deletePrFilesWithOutcome( + "test-ws", "test-ns", 42, "generation-b"); + assertThat(appender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage)) + .containsExactly("Failed to delete PR #42 files for project=100 " + + "target=generation-a: status=409 detail=mutation lease unavailable"); } @Test void testDeletePrFiles_PipelineReturnsFalse() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); setupProjectWithWorkspaceAndNamespace(); - when(ragPipelineClient.deletePrFiles("test-ws", "test-ns", 42)).thenReturn(false); + when(ragPipelineClient.deletePrFilesWithOutcome("test-ws", "test-ns", 42, null)) + .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.failure( + "legacy-alias", + RagPipelineClient.PrFilesDeletionFailure.TARGET, + 404, + "not found")); boolean result = service.deletePrFiles(testProject, 42); @@ -1350,7 +2150,7 @@ void testDeletePrFiles_PipelineReturnsFalse() { void testDeletePrFiles_PipelineThrowsException() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); setupProjectWithWorkspaceAndNamespace(); - when(ragPipelineClient.deletePrFiles("test-ws", "test-ns", 42)) + when(ragPipelineClient.deletePrFilesWithOutcome("test-ws", "test-ns", 42, null)) .thenThrow(new RuntimeException("Connection timeout")); boolean result = service.deletePrFiles(testProject, 42); diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingServiceTest.java index 0c32536b..3a028f86 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingServiceTest.java @@ -324,7 +324,8 @@ void shouldCompleteFullIndexing(boolean multiBranchEnabled) throws Exception { assertThat(result).containsEntry("status", "queued"); assertThat(result).containsEntry("branch", "main"); assertThat(result).containsEntry("jobId", "rag-job-123"); - verify(ragIndexTrackingService).markIndexingStarted(testProject, "main", "abc123"); + verify(ragIndexTrackingService).markIndexingStarted( + eq(testProject), eq("main"), eq("abc123"), anyLong()); verify(mockVcs).downloadRepositoryArchiveToFile( eq("my-workspace"), eq("my-repo"), @@ -375,7 +376,8 @@ void shouldHandleIndexingIOException() throws Exception { Map result = service.indexProjectFromVcs(createProjectDTO(100L), "main", messageConsumer); assertThat(result).containsEntry("status", "error"); - verify(ragIndexTrackingService).markIndexingFailed(eq(testProject), anyString()); + verify(ragIndexTrackingService).markIndexingFailed( + eq(testProject), anyString(), eq(0L)); verify(jobService).failJob(eq(mockJob), anyString()); verify(analysisLockService).releaseLock("lock-key"); } @@ -405,7 +407,8 @@ void pollingFailureDoesNotDeleteConsumerOwnedWorkspace() throws Exception { assertThat(consumerWorkspace).exists(); verify(analysisLockService).releaseLock("lock-key"); verify(analysisLockService).renewLock("lock-key", 30); - verify(ragIndexTrackingService).markIndexingFailed(testProject, "worker failed"); + verify(ragIndexTrackingService).markIndexingFailed( + testProject, "worker failed", null); } finally { Files.deleteIfExists(consumerWorkspace); } @@ -433,7 +436,7 @@ void workerStatusHeartbeatRefreshesObservableIndexStatus() { "codecrow:queue:rag", "queued-payload"); - verify(ragIndexTrackingService).markIndexingHeartbeat(testProject); + verify(ragIndexTrackingService).markIndexingHeartbeat(testProject, 0L); verify(jobService).logToJob( eq(job), eq(JobLogLevel.INFO), @@ -446,7 +449,8 @@ void workerStatusHeartbeatRefreshesObservableIndexStatus() { "main", "abc123", 12, - 34); + 34, + 0L); verify(analysisLockService, times(2)).renewLock("lock-key", 30); verify(analysisLockService).releaseLock("lock-key"); } diff --git a/java-ecosystem/libs/task-management/src/main/java/org/rostilos/codecrow/taskmanagement/jira/cloud/JiraCloudClient.java b/java-ecosystem/libs/task-management/src/main/java/org/rostilos/codecrow/taskmanagement/jira/cloud/JiraCloudClient.java index 84e07abe..2d56f32c 100644 --- a/java-ecosystem/libs/task-management/src/main/java/org/rostilos/codecrow/taskmanagement/jira/cloud/JiraCloudClient.java +++ b/java-ecosystem/libs/task-management/src/main/java/org/rostilos/codecrow/taskmanagement/jira/cloud/JiraCloudClient.java @@ -664,6 +664,23 @@ private ArrayNode buildInlineContent(String text) { while (pos < len) { char c = text.charAt(pos); + // ── Bare HTTP(S) URL ── + // Jira does not auto-link plain text inside an ADF document. Add + // the link mark explicitly, preserving fragments used by opaque + // public-share URLs. + if (text.startsWith("https://", pos) || text.startsWith("http://", pos)) { + int end = pos; + while (end < len && !Character.isWhitespace(text.charAt(end))) { + end++; + } + end = trimBareUrlEnd(text, pos, end); + String url = text.substring(pos, end); + flushPlain(nodes, plain); + addLinkTextNode(nodes, url, url); + pos = end; + continue; + } + // ── Backslash escape: \* \_ \` \\ ── if (c == '\\' && pos + 1 < len) { char next = text.charAt(pos + 1); @@ -731,6 +748,47 @@ private ArrayNode buildInlineContent(String text) { return nodes; } + private static int trimBareUrlEnd(String text, int start, int end) { + int trimmedEnd = end; + while (trimmedEnd > start) { + char trailing = text.charAt(trimmedEnd - 1); + if (".,;!?".indexOf(trailing) >= 0) { + trimmedEnd--; + continue; + } + char opening = switch (trailing) { + case ')' -> '('; + case ']' -> '['; + case '}' -> '{'; + default -> '\0'; + }; + if (opening == '\0' || !hasUnmatchedClosingDelimiter( + text, start, trimmedEnd, opening, trailing)) { + break; + } + trimmedEnd--; + } + return trimmedEnd; + } + + private static boolean hasUnmatchedClosingDelimiter( + String text, + int start, + int end, + char opening, + char closing) { + int balance = 0; + for (int index = start; index < end; index++) { + char current = text.charAt(index); + if (current == opening) { + balance++; + } else if (current == closing) { + balance--; + } + } + return balance < 0; + } + /** Flush accumulated plain text into a text node, then clear the buffer. */ private void flushPlain(ArrayNode nodes, StringBuilder buffer) { if (buffer.isEmpty()) { @@ -756,6 +814,23 @@ private void addTextNode(ArrayNode nodes, String text, List markTypes) { nodes.add(textNode); } + private void addLinkTextNode(ArrayNode nodes, String text, String href) { + ObjectNode textNode = objectMapper.createObjectNode(); + textNode.put("type", "text"); + textNode.put("text", text); + + ObjectNode link = objectMapper.createObjectNode(); + link.put("type", "link"); + ObjectNode attributes = objectMapper.createObjectNode(); + attributes.put("href", href); + link.set("attrs", attributes); + + ArrayNode marks = objectMapper.createArrayNode(); + marks.add(link); + textNode.set("marks", marks); + nodes.add(textNode); + } + // ─── Response helpers ──────────────────────────────────────────── private void ensureSuccess(Response response, String operation) throws IOException { diff --git a/java-ecosystem/libs/task-management/src/test/java/org/rostilos/codecrow/taskmanagement/jira/cloud/JiraCloudClientTest.java b/java-ecosystem/libs/task-management/src/test/java/org/rostilos/codecrow/taskmanagement/jira/cloud/JiraCloudClientTest.java index 770cae1f..d9cb6d1a 100644 --- a/java-ecosystem/libs/task-management/src/test/java/org/rostilos/codecrow/taskmanagement/jira/cloud/JiraCloudClientTest.java +++ b/java-ecosystem/libs/task-management/src/test/java/org/rostilos/codecrow/taskmanagement/jira/cloud/JiraCloudClientTest.java @@ -145,6 +145,60 @@ void postCommentMovesCodeCrowMarkersOutOfAdfContent() throws Exception { "codecrow-ask-response"); } + @Test + @DisplayName("renders a bare public-share URL as a clickable Jira link") + void postCommentAddsAdfLinkMarkToBarePublicShareUrl() throws Exception { + server.enqueue(commentResponse(201)); + String shareUrl = "https://codecrow.cloud/share#token=ccs_opaque-token&tab=environment"; + + client.postComment("PROJ-123", """ + ### 3. Test Scenarios + + %s + """.formatted(shareUrl)); + + JsonNode payload = mapper.readTree(server.takeRequest().getBody().readUtf8()); + JsonNode linkedText = payload.path("body").path("content").get(1) + .path("content").get(0); + assertThat(linkedText.path("text").asText()).isEqualTo(shareUrl); + assertThat(linkedText.path("marks").get(0).path("type").asText()) + .isEqualTo("link"); + assertThat(linkedText.path("marks").get(0).path("attrs").path("href").asText()) + .isEqualTo(shareUrl); + } + + @Test + @DisplayName("keeps sentence delimiters outside bare URL links") + void postCommentExcludesUnmatchedClosingDelimitersFromBareUrls() throws Exception { + server.enqueue(commentResponse(201)); + String shareUrl = "https://codecrow.cloud/share#token=ccs_opaque-token&tab=test-cases"; + + client.postComment("PROJ-123", "Open (" + shareUrl + ") or [" + shareUrl + "]."); + + JsonNode paragraph = mapper.readTree(server.takeRequest().getBody().readUtf8()) + .path("body").path("content").get(0); + assertThat(paragraph.path("content").findValuesAsText("text")) + .containsExactly("Open (", shareUrl, ") or [", shareUrl, "]."); + assertThat(paragraph.path("content").findValues("attrs")) + .extracting(node -> node.path("href").asText()) + .containsExactly(shareUrl, shareUrl); + } + + @Test + @DisplayName("preserves balanced delimiters that belong to a bare URL") + void postCommentKeepsBalancedUrlDelimiters() throws Exception { + server.enqueue(commentResponse(201)); + String shareUrl = "https://example.test/docs/function_(value)"; + + client.postComment("PROJ-123", shareUrl); + + JsonNode linkedText = mapper.readTree(server.takeRequest().getBody().readUtf8()) + .path("body").path("content").get(0).path("content").get(0); + assertThat(linkedText.path("text").asText()).isEqualTo(shareUrl); + assertThat(linkedText.path("marks").get(0).path("attrs").path("href").asText()) + .isEqualTo(shareUrl); + } + @Test @DisplayName("finds a Jira comment by non-rendered CodeCrow marker property") void findCommentByMarkerUsesCommentProperties() throws Exception { diff --git a/java-ecosystem/pom.xml b/java-ecosystem/pom.xml index 51123c1f..8d382772 100644 --- a/java-ecosystem/pom.xml +++ b/java-ecosystem/pom.xml @@ -110,6 +110,12 @@ 1.0 + + org.rostilos.codecrow + codecrow-public-share + 1.0 + + org.rostilos.codecrow codecrow-commit-graph @@ -588,6 +594,7 @@ libs/vcs-client libs/core + libs/public-share libs/scm-evidence libs/commit-graph libs/file-content diff --git a/java-ecosystem/services/pipeline-agent/pom.xml b/java-ecosystem/services/pipeline-agent/pom.xml index 88ec41f5..475e21bc 100644 --- a/java-ecosystem/services/pipeline-agent/pom.xml +++ b/java-ecosystem/services/pipeline-agent/pom.xml @@ -60,6 +60,11 @@ codecrow-core + + org.rostilos.codecrow + codecrow-public-share + + org.rostilos.codecrow codecrow-commit-graph diff --git a/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/BranchResolverFlowIT.java b/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/BranchResolverFlowIT.java index dae90b21..bc8dd379 100644 --- a/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/BranchResolverFlowIT.java +++ b/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/BranchResolverFlowIT.java @@ -53,6 +53,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anySet; @@ -91,9 +92,12 @@ class BranchResolverFlowIT extends BasePipelineAgentIT { void configureMocks() throws Exception { vcsAiClientService = mock(VcsAiClientService.class); vcsClient = mock(VcsClient.class); + AnalysisLockService.LockLease lockLease = mock(AnalysisLockService.LockLease.class); when(analysisLockService.acquireLockWithWait(any(Project.class), anyString(), any(), anyString(), any(), any())) .thenReturn(Optional.of("branch-resolver-it-lock")); + when(analysisLockService.maintainLockLease(anyString(), anyInt())).thenReturn(lockLease); + when(lockLease.confirmOwnership()).thenReturn(true); when(analysisLockService.isLocked(any(), anyString(), any())).thenReturn(false); when(vcsServiceFactory.getAiClientService(EVcsProvider.GITHUB)).thenReturn(vcsAiClientService); diff --git a/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/LineTrackingFlowIT.java b/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/LineTrackingFlowIT.java index 8f7eeaf0..4714b113 100644 --- a/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/LineTrackingFlowIT.java +++ b/java-ecosystem/services/pipeline-agent/src/it/java/org/rostilos/codecrow/pipelineagent/LineTrackingFlowIT.java @@ -85,9 +85,12 @@ void configureMocks() throws Exception { vcsAiClientService = mock(VcsAiClientService.class); vcsReportingService = mock(VcsReportingService.class); vcsClient = mock(VcsClient.class); + AnalysisLockService.LockLease lockLease = mock(AnalysisLockService.LockLease.class); when(analysisLockService.acquireLockWithWait(any(Project.class), anyString(), any(), anyString(), any(), any())) .thenReturn(Optional.of("it-lock")); + when(analysisLockService.maintainLockLease(anyString(), anyInt())).thenReturn(lockLease); + when(lockLease.confirmOwnership()).thenReturn(true); when(analysisLockService.isLocked(anyLong(), anyString(), any())).thenReturn(false); when(vcsServiceFactory.getAiClientService(EVcsProvider.GITHUB)).thenReturn(vcsAiClientService); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java index a8b330a2..2fc22991 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java @@ -266,7 +266,8 @@ private void cleanupPrRagData(WebhookPayload payload, Project project) { if (deleted) { log.info("Cleaned up PR #{} RAG data for project {} on merge", prNumber, project.getId()); } else { - log.warn("Failed to cleanup PR #{} RAG data for project {}", prNumber, project.getId()); + log.info("PR #{} RAG cleanup did not complete for project {}; " + + "the RAG client recorded the failure detail", prNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up PR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudPullRequestWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudPullRequestWebhookHandler.java index 5379f641..2786fb5d 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudPullRequestWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudPullRequestWebhookHandler.java @@ -308,7 +308,8 @@ private void cleanupPrRagData(WebhookPayload payload, Project project) { if (deleted) { log.info("Cleaned up PR #{} RAG data for project {} on close/merge", prNumber, project.getId()); } else { - log.warn("Failed to cleanup PR #{} RAG data for project {}", prNumber, project.getId()); + log.info("PR #{} RAG cleanup did not complete for project {}; " + + "the cleanup operation recorded the failure detail", prNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up PR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java index 5f6b4228..2fa44337 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java @@ -6,6 +6,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import java.util.concurrent.Executor; @@ -22,6 +23,23 @@ public class AsyncConfig { private static final Logger log = LoggerFactory.getLogger(AsyncConfig.class); + /** + * Keep liveness/recovery schedules independent from optional maintenance. + * A slow provider or Qdrant enrichment task must not prevent durable job + * recovery, heartbeat checks, or queue reconciliation from running. + */ + @Bean(name = "taskScheduler") + public ThreadPoolTaskScheduler taskScheduler( + @Value("${spring.task.scheduling.pool.size:4}") int poolSize) { + ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler(); + scheduler.setPoolSize(poolSize); + scheduler.setThreadNamePrefix("scheduling-"); + scheduler.setWaitForTasksToCompleteOnShutdown(true); + scheduler.setAwaitTerminationSeconds(30); + log.info("Scheduled task executor initialized with pool={}", poolSize); + return scheduler; + } + /** * Default executor for general async tasks. */ diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessor.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessor.java index b2f0814a..412ddf0a 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessor.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessor.java @@ -11,6 +11,7 @@ import org.rostilos.codecrow.core.model.project.config.QaAutoDocConfig; import org.rostilos.codecrow.core.model.project.config.TaskManagementConfig; import org.rostilos.codecrow.core.model.qadoc.QaDocState; +import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; import org.rostilos.codecrow.core.model.taskmanagement.TaskManagementConnection; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.core.model.vcs.VcsRepoInfo; @@ -24,6 +25,8 @@ import org.rostilos.codecrow.pipelineagent.qadoc.QaAutoDocListener; import org.rostilos.codecrow.pipelineagent.qadoc.QaDocGenerationContext; import org.rostilos.codecrow.pipelineagent.qadoc.QaDocGenerationService; +import org.rostilos.codecrow.pipelineagent.qadoc.QaDocHandoffRetryPolicy; +import org.rostilos.codecrow.pipelineagent.qadoc.QaDocPublicPreviewService; import org.rostilos.codecrow.taskmanagement.ETaskManagementPlatform; import org.rostilos.codecrow.taskmanagement.TaskManagementClient; import org.rostilos.codecrow.taskmanagement.TaskManagementClientFactory; @@ -33,9 +36,7 @@ import org.rostilos.codecrow.taskmanagement.model.TaskDetails; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; -import org.rostilos.codecrow.security.oauth.TokenEncryptionService; import org.rostilos.codecrow.vcsclient.utils.VcsConnectionCredentialsExtractor; -import org.rostilos.codecrow.vcsclient.utils.VcsConnectionCredentialsExtractor.VcsConnectionCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -48,8 +49,8 @@ /** * Processor for the {@code /codecrow qa-doc} command. *

- * Generates QA testing documentation for the current PR and posts it as a comment - * on the linked Jira task. Reuses the same inference-orchestrator endpoint and + * Generates QA testing documentation for the current PR and posts its public + * QA-document preview links on the linked Jira task. Reuses the same inference-orchestrator endpoint and * task management client infrastructure as the automatic {@link QaAutoDocListener}. *

* Usage: @@ -70,8 +71,8 @@ public class QaDocCommandProcessor implements CommentCommandProcessor { private final VcsClientProvider vcsClientProvider; private final QaDocStateRepository qaDocStateRepository; private final QaDocDocumentService qaDocDocumentService; + private final QaDocPublicPreviewService qaDocPublicPreviewService; private final PrFileEnrichmentService enrichmentService; - private final VcsConnectionCredentialsExtractor credentialsExtractor; public QaDocCommandProcessor( TaskManagementConnectionRepository connectionRepository, @@ -81,8 +82,8 @@ public QaDocCommandProcessor( VcsClientProvider vcsClientProvider, QaDocStateRepository qaDocStateRepository, QaDocDocumentService qaDocDocumentService, - PrFileEnrichmentService enrichmentService, - TokenEncryptionService tokenEncryptionService + QaDocPublicPreviewService qaDocPublicPreviewService, + PrFileEnrichmentService enrichmentService ) { this.connectionRepository = connectionRepository; this.clientFactory = clientFactory; @@ -91,8 +92,8 @@ public QaDocCommandProcessor( this.vcsClientProvider = vcsClientProvider; this.qaDocStateRepository = qaDocStateRepository; this.qaDocDocumentService = qaDocDocumentService; + this.qaDocPublicPreviewService = qaDocPublicPreviewService; this.enrichmentService = enrichmentService; - this.credentialsExtractor = new VcsConnectionCredentialsExtractor(tokenEncryptionService); } @Override @@ -270,28 +271,29 @@ public WebhookResult process( vcsConnection, workspace, repoSlug, commitHash, changedFilePaths); } - // 5e. Extract VCS credentials for Python-side RAG access - String oauthKey = null; - String oauthSecret = null; - String bearerToken = null; - String vcsProviderStr = null; - if (vcsConnection != null) { - try { - VcsConnectionCredentials creds = credentialsExtractor.extractCredentials(vcsConnection); - oauthKey = creds.oAuthClient(); - oauthSecret = creds.oAuthSecret(); - bearerToken = creds.accessToken(); - vcsProviderStr = creds.vcsProviderString(); - } catch (Exception e) { - log.warn("qa-doc command: failed to extract VCS credentials: {}", e.getMessage()); - } - } + // 5e. Keep only the non-secret provider identifier used by the prompt. + String vcsProviderStr = vcsConnection == null + ? null + : VcsConnectionCredentialsExtractor.getVcsProviderString( + vcsConnection.getProviderType()); // 6. Load server-side state and check for existing Jira comment QaDocState state = (prNumber != null) ? qaDocStateRepository.findByProjectIdAndTaskId(project.getId(), taskId).orElse(null) : null; boolean isSamePrRerun = (state != null && prNumber != null && state.isPrDocumented(prNumber)); + Optional pendingHandoffDocument = Optional.empty(); + if (prNumber != null) { + try { + pendingHandoffDocument = qaDocDocumentService + .findLatestDocument(project.getId(), prNumber) + .filter(document -> QaDocHandoffRetryPolicy.shouldReuse( + document, state, commitHash, taskId)); + } catch (Exception e) { + log.warn("qa-doc command: failed to inspect pending handoff for project {} PR #{}: {}", + project.getId(), prNumber, e.getMessage()); + } + } Optional existingComment = Optional.empty(); try { @@ -318,7 +320,9 @@ public WebhookResult process( taskId, e.getMessage()); } - if (previousDocumentation == null && existingComment.isPresent()) { + if (previousDocumentation == null + && existingComment.isPresent() + && !QaAutoDocListener.isPublicPreviewOnlyComment(existingComment.get().body())) { previousDocumentation = existingComment.get().body(); log.info("qa-doc command: using existing task comment as previous documentation for task {}", taskId); @@ -326,7 +330,8 @@ public WebhookResult process( // 6a. Compute delta diff for same-PR re-runs String deltaDiff = null; - if (isSamePrRerun && state.getLastCommitHash() != null + if (pendingHandoffDocument.isEmpty() + && isSamePrRerun && state.getLastCommitHash() != null && commitHash != null && vcsClient != null) { DiffContentFilter contentFilter = new DiffContentFilter(); final VcsClient clientForDiff = vcsClient; @@ -356,13 +361,17 @@ public WebhookResult process( .repoSlug(repoSlug) .sourceBranch(sourceBranch) .targetBranch(targetBranch) - .oauthKey(oauthKey) - .oauthSecret(oauthSecret) - .bearerToken(bearerToken) .build(); - String qaDocument = qaDocGenerationService.generateQaDocumentation( - project, prNumber, issuesFound, filesAnalyzed, prMetadata, ctx); + String qaDocument; + if (pendingHandoffDocument.isPresent()) { + qaDocument = pendingHandoffDocument.get().getMarkdownContent(); + log.info("qa-doc command: retrying pending task handoff without regenerating project {} PR #{}", + project.getId(), prNumber); + } else { + qaDocument = qaDocGenerationService.generateQaDocumentation( + project, prNumber, issuesFound, filesAnalyzed, prMetadata, ctx); + } if (qaDocument == null || qaDocument.isBlank()) { return WebhookResult.ignored( @@ -370,18 +379,27 @@ public WebhookResult process( } Long analysisId = (analysis != null) ? analysis.getId() : null; - if (prNumber != null) { - upsertQaDocDocument(project, prNumber, taskId, commitHash, analysisId, qaDocument); + Optional persistedDocument = pendingHandoffDocument; + if (persistedDocument.isEmpty() && prNumber != null) { + persistedDocument = upsertQaDocDocument( + project, prNumber, taskId, commitHash, analysisId, qaDocument); } + if (persistedDocument.isEmpty()) { + return WebhookResult.error( + "QA documentation was generated but its secure public preview could not be created."); + } + + String previewUrl = qaDocPublicPreviewService.createPreviewUrl(persistedDocument.get()); eventConsumer.accept(Map.of( "type", "status", "state", "posting_comment", - "message", "Posting documentation to " + taskId + "..." + "message", "Posting QA-document preview links to " + taskId + "..." )); - // 8. Post or update comment on Jira task - String commentBody = QaAutoDocListener.COMMENT_MARKER + "\n\n" + qaDocument; + // 8. Keep the QA guide in Jira, replacing only its large test-case + // section with the public preview URL. + String commentBody = qaDocPublicPreviewService.buildTaskComment(qaDocument, previewUrl); TaskCommentVisibility visibility = toTaskCommentVisibility(qaConfig.commentVisibility()); String action; @@ -450,16 +468,18 @@ public WebhookResult process( } } - private void upsertQaDocDocument(Project project, Long prNumber, String taskId, - String commitHash, Long analysisId, String qaDocument) { + private Optional upsertQaDocDocument(Project project, Long prNumber, String taskId, + String commitHash, Long analysisId, String qaDocument) { try { - qaDocDocumentService.upsertLatestDocument( + QaDocDocument document = qaDocDocumentService.upsertLatestDocument( project, prNumber, taskId, analysisId, commitHash, qaDocument); log.debug("qa-doc command: persisted latest document for project {} PR #{}", project.getId(), prNumber); + return Optional.of(document); } catch (Exception e) { log.warn("qa-doc command: failed to persist latest document for project {} PR #{}: {}", project.getId(), prNumber, e.getMessage()); + return Optional.empty(); } } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthScheduler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthScheduler.java index 5deaafeb..a62936dd 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthScheduler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthScheduler.java @@ -1,9 +1,7 @@ package org.rostilos.codecrow.pipelineagent.generic.service; -import org.rostilos.codecrow.core.model.branch.Branch; import org.rostilos.codecrow.core.model.branch.BranchHealthStatus; import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; -import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.persistence.repository.branch.BranchRepository; import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; import org.rostilos.codecrow.analysisengine.processor.analysis.BranchAnalysisProcessor; @@ -11,11 +9,12 @@ import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.Map; +import java.util.function.Consumer; /** * Scheduled service that retries analysis for STALE branches. @@ -30,7 +29,7 @@ * - 2 failures → retry after 20 min * - 3 failures → retry after 30 min * - ... - * - MAX_CONSECUTIVE_FAILURES (10) → marked CRITICAL, no more retries + * - MAX_CONSECUTIVE_FAILURES (10) → left STALE until the next push * * Batch size is limited to prevent overwhelming the system. */ @@ -42,11 +41,14 @@ public class BranchHealthScheduler { /** Base backoff in minutes per consecutive failure. */ private static final long BASE_BACKOFF_MINUTES = 10; - /** Maximum consecutive failures before promoting to CRITICAL (stops retrying). */ + /** Maximum consecutive failures before scheduled retries stop. */ private static final int MAX_CONSECUTIVE_FAILURES = 10; /** Maximum branches to retry per scheduler run to avoid overloading. */ private static final int BATCH_SIZE = 5; + + /** Scheduled retries have no request stream, but downstream stages may emit. */ + private static final Consumer> NO_OP_OBSERVER = ignored -> { }; private final BranchRepository branchRepository; private final BranchAnalysisProcessor branchAnalysisProcessor; @@ -65,9 +67,12 @@ public BranchHealthScheduler( * based on its consecutiveFailures count. */ @Scheduled(fixedDelayString = "${branch.health.retry.interval.ms:600000}") - @Transactional public void retryStaleBranches() { - List staleBranches = branchRepository.findByHealthStatusWithProject(BranchHealthStatus.STALE); + // The repository materializes scalar data in its own short read + // transaction. Never hold a scheduler transaction/connection while the + // processor performs VCS, RAG, or AI calls. + List staleBranches = + branchRepository.findStaleRetryCandidates(BranchHealthStatus.STALE); if (staleBranches.isEmpty()) { return; @@ -78,7 +83,7 @@ public void retryStaleBranches() { OffsetDateTime now = OffsetDateTime.now(); int retried = 0; - for (Branch branch : staleBranches) { + for (BranchRepository.StaleRetryCandidate branch : staleBranches) { if (retried >= BATCH_SIZE) { log.info("Batch size limit reached ({}), remaining branches will be retried next cycle", BATCH_SIZE); break; @@ -88,8 +93,9 @@ public void retryStaleBranches() { // The branch stays STALE but backoff is so large it effectively won't retry // until a new push event resets the failure counter. if (branch.getConsecutiveFailures() >= MAX_CONSECUTIVE_FAILURES) { - log.warn("Branch {} (project={}, branch='{}') exceeded max retries ({}) — skipping until next push event", - branch.getId(), branch.getProject().getId(), branch.getBranchName(), + log.debug("Branch {} (project={}, branch='{}') remains above max retries ({}) — " + + "waiting for the next push event", + branch.getBranchId(), branch.getProjectId(), branch.getBranchName(), branch.getConsecutiveFailures()); continue; } @@ -99,22 +105,20 @@ public void retryStaleBranches() { continue; } - Project project = branch.getProject(); - // Skip if branch analysis is disabled for this project - if (!project.isBranchAnalysisEnabled()) { + if (!branch.getBranchAnalysisEnabled()) { log.debug("Skipping STALE branch {} — branch analysis disabled for project {}", - branch.getBranchName(), project.getId()); + branch.getBranchName(), branch.getProjectId()); continue; } // Skip if no commit hash to retry with if (branch.getCommitHash() == null || branch.getCommitHash().isBlank()) { - log.warn("Skipping STALE branch {} — no commit hash available", branch.getId()); + log.debug("Skipping STALE branch {} — no commit hash available", branch.getBranchId()); continue; } - retryBranchAnalysis(branch, project); + retryBranchAnalysis(branch); retried++; } @@ -127,7 +131,9 @@ public void retryStaleBranches() { * Check if a branch is eligible for retry based on backoff. * A branch must wait (consecutiveFailures * BASE_BACKOFF_MINUTES) since its last health check. */ - private boolean isEligibleForRetry(Branch branch, OffsetDateTime now) { + private boolean isEligibleForRetry( + BranchRepository.StaleRetryCandidate branch, + OffsetDateTime now) { OffsetDateTime lastCheck = branch.getLastHealthCheckAt(); if (lastCheck == null) { // Never checked — eligible @@ -139,7 +145,7 @@ private boolean isEligibleForRetry(Branch branch, OffsetDateTime now) { if (minutesSinceLastCheck < requiredWaitMinutes) { log.debug("Branch {} backoff not elapsed: waited {}m of required {}m (failures={})", - branch.getId(), minutesSinceLastCheck, requiredWaitMinutes, + branch.getBranchId(), minutesSinceLastCheck, requiredWaitMinutes, branch.getConsecutiveFailures()); return false; } @@ -150,29 +156,29 @@ private boolean isEligibleForRetry(Branch branch, OffsetDateTime now) { /** * Retry branch analysis for a single STALE branch. */ - private void retryBranchAnalysis(Branch branch, Project project) { + private void retryBranchAnalysis(BranchRepository.StaleRetryCandidate branch) { try { log.info("Retrying STALE branch: id={}, project={}, branch='{}', commit={}, failures={}", - branch.getId(), project.getId(), branch.getBranchName(), + branch.getBranchId(), branch.getProjectId(), branch.getBranchName(), branch.getCommitHash(), branch.getConsecutiveFailures()); BranchProcessRequest request = new BranchProcessRequest(); - request.projectId = project.getId(); + request.projectId = branch.getProjectId(); request.targetBranchName = branch.getBranchName(); request.commitHash = branch.getCommitHash(); request.analysisType = AnalysisType.BRANCH_ANALYSIS; // No sourcePrNumber — this is a scheduled retry, not triggered by a PR merge - branchAnalysisProcessor.process(request, null); + branchAnalysisProcessor.process(request, NO_OP_OBSERVER); log.info("STALE branch retry completed successfully: branch={}, project={}", - branch.getBranchName(), project.getId()); + branch.getBranchName(), branch.getProjectId()); } catch (Exception e) { // The processor itself handles markStale() on failure, // so we just log the error and move on to the next branch. log.error("STALE branch retry failed: branch={}, project={}, error={}", - branch.getBranchName(), project.getId(), e.getMessage()); + branch.getBranchName(), branch.getProjectId(), e.getMessage()); } } } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java index 89af6e8c..4c982464 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java @@ -368,6 +368,32 @@ public void failJob(Job job, String errorMessage) { } } + @Override + public void skipJob(Job job, String reason) { + if (job == null) return; + + try { + jobService.skipJob(job, reason); + } catch (Exception e) { + log.error("Error skipping redundant job {}", job.getExternalId(), e); + } + } + + @Override + public void recordExternallyCompletedJob( + Job job, + String state, + String message) { + if (job == null) return; + + try { + jobService.recordExternallyCompletedJob(job, state, message); + } catch (Exception e) { + log.warn("Could not record terminal notification for completed job {}: {}", + job.getExternalId(), e.getMessage()); + } + } + public JobService getJobService() { return jobService; } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoveryScheduler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoveryScheduler.java index 8913d457..35fc85ac 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoveryScheduler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoveryScheduler.java @@ -52,6 +52,9 @@ public void recoverAcceptedJobs() { OffsetDateTime pendingThreshold = OffsetDateTime.now().minusSeconds(30); List candidates = jobService.findRecoverableWebhookJobs(pendingThreshold, 50); for (Job candidate : candidates) { + if (!isSupportedWebhookAnalysis(candidate)) { + continue; + } if (!jobService.claimRecoverableWebhookJob( candidate.getId(), pendingThreshold)) { continue; @@ -63,6 +66,9 @@ public void recoverAcceptedJobs() { List abandoned = jobService.findAbandonedRunningWebhookJobs( abandonedThreshold, 20); for (Job candidate : abandoned) { + if (!isSupportedWebhookAnalysis(candidate)) { + continue; + } if (!jobService.claimAbandonedRunningWebhookJob( candidate.getId(), abandonedThreshold)) { continue; @@ -73,6 +79,11 @@ public void recoverAcceptedJobs() { } } + private static boolean isSupportedWebhookAnalysis(Job job) { + return job != null && (job.getJobType() == JobType.PR_ANALYSIS + || job.getJobType() == JobType.BRANCH_ANALYSIS); + } + private void recover(Job job) { try { Optional persistedPayload = diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java index 80ad0679..78271b3c 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java @@ -400,7 +400,8 @@ private void cleanupPrRagData(WebhookPayload payload, Project project) { if (deleted) { log.info("Cleaned up PR #{} RAG data for project {} on close/merge", prNumber, project.getId()); } else { - log.warn("Failed to cleanup PR #{} RAG data for project {}", prNumber, project.getId()); + log.info("PR #{} RAG cleanup did not complete for project {}; " + + "the cleanup operation recorded the failure detail", prNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up PR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMergeRequestWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMergeRequestWebhookHandler.java index e7b42f2a..bee07e1e 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMergeRequestWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMergeRequestWebhookHandler.java @@ -309,7 +309,8 @@ private void cleanupPrRagData(WebhookPayload payload, Project project) { if (deleted) { log.info("Cleaned up MR !{} RAG data for project {} on close/merge", prNumber, project.getId()); } else { - log.warn("Failed to cleanup MR !{} RAG data for project {}", prNumber, project.getId()); + log.info("MR !{} RAG cleanup did not complete for project {}; " + + "the cleanup operation recorded the failure detail", prNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up MR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java index 51993f7d..165aa3a6 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java @@ -181,7 +181,8 @@ private void cleanupPrRagData(Project project, Long mrNumber) { if (deleted) { log.info("Cleaned up MR !{} RAG data for project {} on merge", mrNumber, project.getId()); } else { - log.warn("Failed to cleanup MR !{} RAG data for project {}", mrNumber, project.getId()); + log.info("MR !{} RAG cleanup did not complete for project {}; " + + "the cleanup operation recorded the failure detail", mrNumber, project.getId()); } } catch (Exception e) { log.warn("Error cleaning up MR RAG data for project {}: {}", project.getId(), e.getMessage()); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListener.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListener.java index 6357cf37..4853b522 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListener.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListener.java @@ -11,6 +11,7 @@ import org.rostilos.codecrow.core.model.project.config.QaAutoDocConfig; import org.rostilos.codecrow.core.model.project.config.TaskManagementConfig; import org.rostilos.codecrow.core.model.qadoc.QaDocState; +import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; import org.rostilos.codecrow.core.model.taskmanagement.TaskManagementConnection; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.core.model.vcs.VcsRepoInfo; @@ -29,9 +30,7 @@ import org.rostilos.codecrow.taskmanagement.model.TaskDetails; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; -import org.rostilos.codecrow.security.oauth.TokenEncryptionService; import org.rostilos.codecrow.vcsclient.utils.VcsConnectionCredentialsExtractor; -import org.rostilos.codecrow.vcsclient.utils.VcsConnectionCredentialsExtractor.VcsConnectionCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.event.EventListener; @@ -73,8 +72,8 @@ public class QaAutoDocListener { private final VcsClientProvider vcsClientProvider; private final QaDocStateRepository qaDocStateRepository; private final QaDocDocumentService qaDocDocumentService; + private final QaDocPublicPreviewService qaDocPublicPreviewService; private final PrFileEnrichmentService enrichmentService; - private final VcsConnectionCredentialsExtractor credentialsExtractor; public QaAutoDocListener(ProjectRepository projectRepository, TaskManagementConnectionRepository connectionRepository, @@ -84,8 +83,8 @@ public QaAutoDocListener(ProjectRepository projectRepository, VcsClientProvider vcsClientProvider, QaDocStateRepository qaDocStateRepository, QaDocDocumentService qaDocDocumentService, - PrFileEnrichmentService enrichmentService, - TokenEncryptionService tokenEncryptionService) { + QaDocPublicPreviewService qaDocPublicPreviewService, + PrFileEnrichmentService enrichmentService) { this.projectRepository = projectRepository; this.connectionRepository = connectionRepository; this.clientFactory = clientFactory; @@ -94,8 +93,8 @@ public QaAutoDocListener(ProjectRepository projectRepository, this.vcsClientProvider = vcsClientProvider; this.qaDocStateRepository = qaDocStateRepository; this.qaDocDocumentService = qaDocDocumentService; + this.qaDocPublicPreviewService = qaDocPublicPreviewService; this.enrichmentService = enrichmentService; - this.credentialsExtractor = new VcsConnectionCredentialsExtractor(tokenEncryptionService); } @Async @@ -200,6 +199,17 @@ private void processQaAutoDoc(AnalysisCompletedEvent event) throws Exception { String sourceBranch = (metrics != null) ? (String) metrics.get("sourceBranch") : null; String targetBranch = (metrics != null) ? (String) metrics.get("targetBranch") : null; + Optional pendingHandoffDocument = Optional.empty(); + try { + pendingHandoffDocument = qaDocDocumentService + .findLatestDocument(project.getId(), prNumber) + .filter(document -> QaDocHandoffRetryPolicy.shouldReuse( + document, state, currentCommitHash, taskId)); + } catch (Exception e) { + log.warn("QA auto-doc: failed to inspect pending handoff for project {} PR #{}: {}", + project.getId(), prNumber, e.getMessage()); + } + // 5a. Fetch full PR diff String diff = null; VcsClient vcsClient = null; @@ -229,7 +239,8 @@ private void processQaAutoDoc(AnalysisCompletedEvent event) throws Exception { // 5c. Compute delta diff for same-PR re-runs (incremental update) String deltaDiff = null; - if (isSamePrRerun && state.getLastCommitHash() != null + if (pendingHandoffDocument.isEmpty() + && isSamePrRerun && state.getLastCommitHash() != null && currentCommitHash != null && vcsClient != null) { DiffContentFilter contentFilter = new DiffContentFilter(); final VcsClient client = vcsClient; @@ -249,22 +260,11 @@ private void processQaAutoDoc(AnalysisCompletedEvent event) throws Exception { currentCommitHash, changedFilePaths); } - // 5e. Extract VCS credentials for Python-side RAG queries - String oauthKey = null; - String oauthSecret = null; - String bearerToken = null; - String vcsProviderStr = null; - if (vcsConnection != null) { - try { - VcsConnectionCredentials creds = credentialsExtractor.extractCredentials(vcsConnection); - oauthKey = creds.oAuthClient(); - oauthSecret = creds.oAuthSecret(); - bearerToken = creds.accessToken(); - vcsProviderStr = creds.vcsProviderString(); - } catch (Exception e) { - log.warn("QA auto-doc: failed to extract VCS credentials (non-critical): {}", e.getMessage()); - } - } + // 5e. Keep only the non-secret provider identifier used by the prompt. + String vcsProviderStr = vcsConnection == null + ? null + : VcsConnectionCredentialsExtractor.getVcsProviderString( + vcsConnection.getProviderType()); // 6. Resolve task management connection + fetch task details TaskManagementConnection connection = connectionRepository @@ -327,7 +327,9 @@ private void processQaAutoDoc(AnalysisCompletedEvent event) throws Exception { taskId, e.getMessage()); } - if (previousDocumentation == null && existingComment.isPresent()) { + if (previousDocumentation == null + && existingComment.isPresent() + && !isPublicPreviewOnlyComment(existingComment.get().body())) { previousDocumentation = existingComment.get().body(); log.info("QA auto-doc: using existing task comment as previous documentation for task {}", taskId); @@ -349,13 +351,16 @@ private void processQaAutoDoc(AnalysisCompletedEvent event) throws Exception { .repoSlug(repoSlug) .sourceBranch(sourceBranch) .targetBranch(targetBranch) - .oauthKey(oauthKey) - .oauthSecret(oauthSecret) - .bearerToken(bearerToken) .build(); - String qaDocument = qaDocGenerationService.generateQaDocumentation( - project, event, ctx); + String qaDocument; + if (pendingHandoffDocument.isPresent()) { + qaDocument = pendingHandoffDocument.get().getMarkdownContent(); + log.info("QA auto-doc: retrying pending task handoff without regenerating project {} PR #{}", + project.getId(), prNumber); + } else { + qaDocument = qaDocGenerationService.generateQaDocumentation(project, event, ctx); + } if (qaDocument == null || qaDocument.isBlank()) { log.info("QA auto-doc: LLM determined no documentation needed for task {}", taskId); @@ -369,13 +374,28 @@ private void processQaAutoDoc(AnalysisCompletedEvent event) throws Exception { return; } - // 9. Update server-side state (secure, tamper-proof) + // 9. Persist the full document before disclosing a preview link. Long analysisId = (analysis != null) ? analysis.getId() : null; - upsertQaDocState(project, taskId, currentCommitHash, analysisId, prNumber, state); - upsertQaDocDocument(project, prNumber, taskId, currentCommitHash, analysisId, qaDocument); + Optional persistedDocument = pendingHandoffDocument.isPresent() + ? pendingHandoffDocument + : upsertQaDocDocument( + project, prNumber, taskId, currentCommitHash, analysisId, qaDocument); + if (persistedDocument.isEmpty()) { + log.warn("QA auto-doc: not updating task {} because a secure public preview could not be created", taskId); + return; + } - // 10. Post or update Jira comment - String commentBody = COMMENT_MARKER + "\n\n" + qaDocument; + // 10. Keep the QA guide in Jira, replacing only its large test-case + // section with the public preview URL. + String commentBody; + try { + String previewUrl = qaDocPublicPreviewService.createPreviewUrl(persistedDocument.get()); + commentBody = qaDocPublicPreviewService.buildTaskComment(qaDocument, previewUrl); + } catch (Exception e) { + log.warn("QA auto-doc: failed to create public QA-document preview for task {}: {}", + taskId, e.getMessage()); + return; + } TaskCommentVisibility visibility = toTaskCommentVisibility(qaConfig.commentVisibility()); try { @@ -399,7 +419,12 @@ private void processQaAutoDoc(AnalysisCompletedEvent event) throws Exception { String errorMessage = describeTaskManagementFailure(e); log.error("QA auto-doc: failed to post/update comment on task {}: {}", taskId, errorMessage, e); + return; } + + // Advance generation state only after the durable document, preview, + // and task handoff all succeeded, so a failed handoff remains retryable. + upsertQaDocState(project, taskId, currentCommitHash, analysisId, prNumber, state); } // ── Enrichment helpers ────────────────────────────────────────── @@ -476,18 +501,19 @@ protected void upsertQaDocState(Project project, String taskId, /** * Upsert the latest rendered QA doc markdown for the PR. */ - protected void upsertQaDocDocument(Project project, Long prNumber, String taskId, - String commitHash, Long analysisId, - String qaDocument) { + protected Optional upsertQaDocDocument(Project project, Long prNumber, String taskId, + String commitHash, Long analysisId, + String qaDocument) { try { - qaDocDocumentService.upsertLatestDocument( + QaDocDocument document = qaDocDocumentService.upsertLatestDocument( project, prNumber, taskId, analysisId, commitHash, qaDocument); log.debug("QA auto-doc: persisted latest document for project {} PR #{}", project.getId(), prNumber); + return Optional.of(document); } catch (Exception e) { - // Non-critical: Jira posting should still proceed. log.warn("QA auto-doc: failed to persist latest document for project {} PR #{}: {}", project.getId(), prNumber, e.getMessage()); + return Optional.empty(); } } @@ -554,6 +580,18 @@ private static String describeTaskManagementFailure(Exception e) { return e.getMessage(); } + public static boolean isPublicPreviewOnlyComment(String body) { + if (body == null || (!body.contains("/share#token=ccs_") + && !body.contains("/share?token=ccs_"))) { + return false; + } + String withoutPreviewLink = body + .replaceAll("https?://\\S+/share(?:#|\\?)token=ccs_[A-Za-z0-9_-]+", "") + .replace("View QA test cases in CodeCrow", "") + .replaceAll("[\\[\\]()\\s]", ""); + return withoutPreviewLink.isBlank(); + } + private static String truncateProviderMessage(String providerMessage) { if (providerMessage == null || providerMessage.isBlank()) { return null; diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationContext.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationContext.java index 3a768f71..fb3c4ead 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationContext.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationContext.java @@ -34,7 +34,7 @@ public record QaDocGenerationContext( List changedFilePaths, // ── State / mode flags ─────────────────────────────────── - /** Existing QA doc comment body from an earlier PR on the same task (for merging). May be null. */ + /** Existing stored QA document from an earlier PR on the same task (for merging). May be null. */ String previousDocumentation, /** True when the current PR was already documented and this is a re-analysis (e.g., new commits pushed). */ boolean isSamePrRerun, @@ -49,15 +49,7 @@ public record QaDocGenerationContext( /** Source branch of the PR. May be null. */ String sourceBranch, /** Target branch of the PR. May be null. */ - String targetBranch, - - // ── VCS credentials (for Python-side RAG deterministic API) ── - /** OAuth consumer key (Bitbucket). May be null. */ - String oauthKey, - /** OAuth consumer secret (Bitbucket). May be null. */ - String oauthSecret, - /** Bearer / PAT token (GitHub, GitLab, Bitbucket APP). May be null. */ - String bearerToken + String targetBranch ) { /** * Builder for constructing a context step-by-step as data becomes available. @@ -81,9 +73,6 @@ public static final class Builder { private String repoSlug; private String sourceBranch; private String targetBranch; - private String oauthKey; - private String oauthSecret; - private String bearerToken; private Builder() {} @@ -101,9 +90,6 @@ private Builder() {} public Builder repoSlug(String v) { this.repoSlug = v; return this; } public Builder sourceBranch(String v) { this.sourceBranch = v; return this; } public Builder targetBranch(String v) { this.targetBranch = v; return this; } - public Builder oauthKey(String v) { this.oauthKey = v; return this; } - public Builder oauthSecret(String v) { this.oauthSecret = v; return this; } - public Builder bearerToken(String v) { this.bearerToken = v; return this; } public QaDocGenerationContext build() { return new QaDocGenerationContext( @@ -111,8 +97,7 @@ public QaDocGenerationContext build() { diff, deltaDiff, enrichmentData, changedFilePaths, previousDocumentation, isSamePrRerun, - vcsProvider, workspaceSlug, repoSlug, sourceBranch, targetBranch, - oauthKey, oauthSecret, bearerToken); + vcsProvider, workspaceSlug, repoSlug, sourceBranch, targetBranch); } } } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationService.java index 97b6a3fc..f14e53ff 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationService.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationService.java @@ -38,7 +38,7 @@ *

    *
  • Assembling the request payload (analysis summary, task context, template config)
    • *
  • Making the HTTP call with retry logic
    • - *
  • Parsing the response into a ready-to-post comment body
    • + *
  • Parsing the response into the full document stored by CodeCrow
    • *
*/ @Service @@ -67,6 +67,9 @@ public QaDocGenerationService( this.tokenEncryptionService = tokenEncryptionService; this.objectMapper = new ObjectMapper(); this.httpClient = HttpClient.newBuilder() + // Uvicorn serves HTTP/1.1 on the internal clear-text endpoint. + // Prevent the JDK client from attempting an h2c upgrade. + .version(HttpClient.Version.HTTP_1_1) .connectTimeout(Duration.ofSeconds(30)) .build(); } @@ -252,7 +255,7 @@ private Map buildPayloadFromContext(Project project, payload.put("changed_file_paths", ctx.changedFilePaths()); } - // ── VCS connection info (for Python-side RAG queries) ── + // ── Non-secret VCS identifiers used in the generated document ── if (ctx.vcsProvider() != null) { payload.put("vcs_provider", ctx.vcsProvider()); } @@ -269,17 +272,6 @@ private Map buildPayloadFromContext(Project project, payload.put("target_branch", ctx.targetBranch()); } - // ── OAuth credentials (for Python-side VCS/RAG access) ── - if (ctx.oauthKey() != null) { - payload.put("oauth_key", ctx.oauthKey()); - } - if (ctx.oauthSecret() != null) { - payload.put("oauth_secret", ctx.oauthSecret()); - } - if (ctx.bearerToken() != null) { - payload.put("bearer_token", ctx.bearerToken()); - } - // ── PR metadata enriched with analysis summary ── Map prMeta = (prMetadata != null) ? new LinkedHashMap<>(prMetadata) diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocHandoffRetryPolicy.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocHandoffRetryPolicy.java new file mode 100644 index 00000000..8625cc22 --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocHandoffRetryPolicy.java @@ -0,0 +1,52 @@ +package org.rostilos.codecrow.pipelineagent.qadoc; + +import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; +import org.rostilos.codecrow.core.model.qadoc.QaDocState; + +import java.time.OffsetDateTime; +import java.util.Locale; +import java.util.Objects; + +/** Identifies a durable QA document whose external task handoff is still pending. */ +public final class QaDocHandoffRetryPolicy { + + private QaDocHandoffRetryPolicy() { + } + + public static boolean shouldReuse( + QaDocDocument document, + QaDocState state, + String currentCommitHash, + String expectedTaskId) { + if (document == null + || currentCommitHash == null + || currentCommitHash.isBlank() + || expectedTaskId == null + || expectedTaskId.isBlank() + || document.getMarkdownContent() == null + || document.getMarkdownContent().isBlank() + || !Objects.equals(normalize(document.getCommitHash()), normalize(currentCommitHash)) + || !Objects.equals( + normalizeTaskId(document.getTaskId()), + normalizeTaskId(expectedTaskId))) { + return false; + } + + OffsetDateTime documentGeneratedAt = document.getGeneratedAt(); + if (documentGeneratedAt == null) { + return false; + } + return state == null + || state.getLastGeneratedAt() == null + || documentGeneratedAt.isAfter(state.getLastGeneratedAt()); + } + + private static String normalize(String value) { + return value == null ? null : value.trim(); + } + + private static String normalizeTaskId(String value) { + String normalized = normalize(value); + return normalized == null ? null : normalized.toUpperCase(Locale.ROOT); + } +} diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocPublicPreviewService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocPublicPreviewService.java new file mode 100644 index 00000000..1943cab4 --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocPublicPreviewService.java @@ -0,0 +1,56 @@ +package org.rostilos.codecrow.pipelineagent.qadoc; + +import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; +import org.rostilos.codecrow.core.service.SiteSettingsProvider; +import org.rostilos.codecrow.core.service.qadoc.QaDocContentParser; +import org.rostilos.codecrow.core.service.qadoc.QaDocPublicShareResource; +import org.rostilos.codecrow.publicshare.api.IssuedPublicShare; +import org.rostilos.codecrow.publicshare.service.PublicShareLinkService; +import org.springframework.stereotype.Service; + +/** QA-doc adapter over the purpose-neutral public-share credential service. */ +@Service +public class QaDocPublicPreviewService { + + private final PublicShareLinkService publicShareLinkService; + private final SiteSettingsProvider siteSettingsProvider; + + public QaDocPublicPreviewService(PublicShareLinkService publicShareLinkService, + SiteSettingsProvider siteSettingsProvider) { + this.publicShareLinkService = publicShareLinkService; + this.siteSettingsProvider = siteSettingsProvider; + } + + public String createPreviewUrl(QaDocDocument document) { + if (document == null || document.getId() == null) { + throw new IllegalArgumentException("A persisted QA document is required for public preview."); + } + if (QaDocContentParser.parseMarkedTestCases(document.getMarkdownContent()).isEmpty()) { + throw new IllegalArgumentException( + "A marked QA test-case section is required for public preview."); + } + IssuedPublicShare share = publicShareLinkService.issue( + QaDocPublicShareResource.DOCUMENT, + document.getId().toString() + ); + return share.toFrontendUrl(siteSettingsProvider.getBaseUrlSettings().frontendUrl()); + } + + public String buildTaskComment(String qaDocument, String previewUrl) { + if (previewUrl == null || previewUrl.isBlank()) { + throw new IllegalArgumentException("A public preview URL is required."); + } + String normalizedPreviewUrl = previewUrl.trim(); + return QaAutoDocListener.COMMENT_MARKER + + "\n\n" + + QaDocContentParser.replaceShareableSections( + qaDocument, + withTab(normalizedPreviewUrl, "test-cases"), + withTab(normalizedPreviewUrl, "environment") + ); + } + + private static String withTab(String previewUrl, String tab) { + return previewUrl + (previewUrl.contains("#") ? "&" : "#") + "tab=" + tab; + } +} diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java index 63942429..41f90812 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java @@ -2,7 +2,9 @@ import org.junit.jupiter.api.Test; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import java.time.Instant; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -11,6 +13,33 @@ class AsyncConfigTest { + @Test + void scheduledRecoveryIsNotStarvedBySlowOptionalMaintenance() throws Exception { + ThreadPoolTaskScheduler scheduler = new AsyncConfig().taskScheduler(2); + scheduler.initialize(); + CountDownLatch started = new CountDownLatch(2); + CountDownLatch release = new CountDownLatch(1); + + try { + Runnable blockingTask = () -> { + started.countDown(); + try { + release.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + }; + scheduler.schedule(blockingTask, Instant.now()); + scheduler.schedule(blockingTask, Instant.now()); + + assertThat(started.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(scheduler.getPoolSize()).isEqualTo(2); + } finally { + release.countDown(); + scheduler.shutdown(); + } + } + @Test void webhookExecutorRunsFiveAcceptedReviewsInParallel() throws Exception { Executor configured = new AsyncConfig().webhookExecutor(2, 5); diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessorTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessorTest.java index 5e1e97e5..caebc2e0 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessorTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/command/QaDocCommandProcessorTest.java @@ -18,6 +18,8 @@ import org.rostilos.codecrow.core.model.project.config.TaskManagementConfig; import org.rostilos.codecrow.core.model.taskmanagement.ETaskManagementProvider; import org.rostilos.codecrow.core.model.taskmanagement.TaskManagementConnection; +import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; +import org.rostilos.codecrow.core.model.qadoc.QaDocState; import org.rostilos.codecrow.core.model.vcs.EVcsProvider; import org.rostilos.codecrow.core.persistence.repository.taskmanagement.TaskManagementConnectionRepository; import org.rostilos.codecrow.core.service.CodeAnalysisService; @@ -27,9 +29,9 @@ import org.rostilos.codecrow.pipelineagent.generic.webhookhandler.WebhookHandler.WebhookResult; import org.rostilos.codecrow.pipelineagent.qadoc.QaDocGenerationContext; import org.rostilos.codecrow.pipelineagent.qadoc.QaDocGenerationService; +import org.rostilos.codecrow.pipelineagent.qadoc.QaDocPublicPreviewService; import org.rostilos.codecrow.core.persistence.repository.qadoc.QaDocStateRepository; import org.rostilos.codecrow.analysisengine.service.pr.PrFileEnrichmentService; -import org.rostilos.codecrow.security.oauth.TokenEncryptionService; import org.rostilos.codecrow.taskmanagement.ETaskManagementPlatform; import org.rostilos.codecrow.taskmanagement.TaskManagementClient; import org.rostilos.codecrow.taskmanagement.TaskManagementClientFactory; @@ -60,8 +62,8 @@ class QaDocCommandProcessorTest { @Mock private VcsClientProvider vcsClientProvider; @Mock private QaDocStateRepository qaDocStateRepository; @Mock private QaDocDocumentService qaDocDocumentService; + @Mock private QaDocPublicPreviewService qaDocPublicPreviewService; @Mock private PrFileEnrichmentService enrichmentService; - @Mock private TokenEncryptionService tokenEncryptionService; private QaDocCommandProcessor processor; private Project project; @@ -75,6 +77,22 @@ class QaDocCommandProcessorTest { private static final Long CONNECTION_ID = 100L; private static final String TASK_ID = "PROJ-123"; private static final String PR_ID = "7"; + private static final String GENERATED_QA_DOC = """ + ### 1. Change Summary + Login validation changed. + + ### 2. Scope + Web login flow. + + + ### 3. Test Scenarios + **Reject an invalid password** (HIGH) + - **Expected Result:** Test steps here + + + ### 4. Edge Cases + Verify an empty password. + """; @BeforeEach void setUp() { @@ -86,14 +104,29 @@ void setUp() { vcsClientProvider, qaDocStateRepository, qaDocDocumentService, - enrichmentService, - tokenEncryptionService + qaDocPublicPreviewService, + enrichmentService ); project = new Project(); ReflectionTestUtils.setField(project, "id", PROJECT_ID); ReflectionTestUtils.setField(project, "name", "Test Project"); + QaDocDocument persistedDocument = new QaDocDocument(project, 7L); + ReflectionTestUtils.setField(persistedDocument, "id", 701L); + lenient().when(qaDocDocumentService.upsertLatestDocument( + any(), anyLong(), anyString(), nullable(Long.class), nullable(String.class), anyString() + )).thenReturn(persistedDocument); + lenient().when(qaDocPublicPreviewService.createPreviewUrl(any())) + .thenReturn("https://app.codecrow.example/share#token=ccs_test-token"); + lenient().when(qaDocPublicPreviewService.buildTaskComment(anyString(), anyString())) + .thenReturn("\n\n" + + "### 1. Change Summary\nLogin validation changed.\n\n" + + "### 2. Scope\nWeb login flow.\n\n" + + "### 3. Test Scenarios\n\n" + + "https://app.codecrow.example/share#token=ccs_test-token\n\n" + + "### 4. Edge Cases\nVerify an empty password."); + capturedEvents = new ArrayList<>(); eventConsumer = capturedEvents::add; } @@ -203,10 +236,10 @@ private void setupHappyPath() throws IOException { when(codeAnalysisService.getPreviousVersionCodeAnalysis(PROJECT_ID, 7L)) .thenReturn(Optional.of(analysis)); - when(qaDocGenerationService.generateQaDocumentation( + lenient().when(qaDocGenerationService.generateQaDocumentation( any(Project.class), anyLong(), anyInt(), anyInt(), anyMap(), any(QaDocGenerationContext.class) - )).thenReturn("## QA Documentation\n\nTest steps here..."); + )).thenReturn(GENERATED_QA_DOC); } // ════════════════════════════════════════════════════════════════ @@ -493,12 +526,25 @@ void shouldGenerateAndPostNewComment() throws IOException { anyMap(), any(QaDocGenerationContext.class) ); - // Verify comment was posted (not updated) - verify(taskManagementClient).postComment(eq(TASK_ID), contains("codecrow-qa-autodoc")); + // Every section remains in Jira except the broad test-case body. + ArgumentCaptor commentCaptor = ArgumentCaptor.forClass(String.class); + verify(taskManagementClient).postComment(eq(TASK_ID), commentCaptor.capture()); + assertThat(commentCaptor.getValue()) + .contains( + "codecrow-qa-autodoc", + "### 1. Change Summary", + "Login validation changed", + "### 2. Scope", + "### 3. Test Scenarios", + "/share#token=ccs_", + "### 4. Edge Cases", + "Verify an empty password") + .doesNotContain("Reject an invalid password") + .doesNotContain("Test steps here"); verify(taskManagementClient, never()).updateComment(anyString(), anyString(), anyString()); verify(qaDocDocumentService).upsertLatestDocument( eq(project), eq(7L), eq(TASK_ID), isNull(), eq("abc123"), - eq("## QA Documentation\n\nTest steps here...") + eq(GENERATED_QA_DOC) ); } @@ -549,6 +595,59 @@ void shouldUpdateExistingComment() throws IOException { verify(taskManagementClient, never()).postComment(anyString(), anyString()); } + @Test + @DisplayName("should reuse a current document when only the Jira handoff is pending") + void shouldRetryPendingHandoffWithoutRegenerating() throws IOException { + setupHappyPath(); + QaDocDocument pendingDocument = new QaDocDocument(project, 7L); + ReflectionTestUtils.setField(pendingDocument, "id", 702L); + pendingDocument.setCommitHash("abc123"); + pendingDocument.setTaskId(TASK_ID); + pendingDocument.setMarkdownContent(GENERATED_QA_DOC); + pendingDocument.setGeneratedAt(OffsetDateTime.now()); + QaDocState state = new QaDocState(project, TASK_ID); + state.recordGeneration("previous-commit", null, 7L); + state.setLastGeneratedAt(OffsetDateTime.now().minusMinutes(5)); + when(qaDocStateRepository.findByProjectIdAndTaskId(PROJECT_ID, TASK_ID)) + .thenReturn(Optional.of(state)); + when(qaDocDocumentService.findLatestDocument(PROJECT_ID, 7L)) + .thenReturn(Optional.of(pendingDocument)); + + WebhookResult result = processor.process( + createPayload("feature/PROJ-123-add-login"), project, eventConsumer, Map.of()); + + assertThat(result.success()).isTrue(); + verify(qaDocGenerationService, never()).generateQaDocumentation( + any(), anyLong(), anyInt(), anyInt(), anyMap(), any()); + verify(qaDocDocumentService, never()).upsertLatestDocument( + any(), anyLong(), anyString(), any(), any(), anyString()); + verify(qaDocPublicPreviewService).createPreviewUrl(pendingDocument); + verify(taskManagementClient).postComment(eq(TASK_ID), contains("codecrow-qa-autodoc")); + verify(qaDocStateRepository).save(state); + } + + @Test + @DisplayName("should regenerate when the pending document belongs to another task") + void shouldNotReusePendingHandoffForAnotherTask() throws IOException { + setupHappyPath(); + QaDocDocument pendingDocument = new QaDocDocument(project, 7L); + pendingDocument.setCommitHash("abc123"); + pendingDocument.setTaskId("OTHER-456"); + pendingDocument.setMarkdownContent(GENERATED_QA_DOC); + pendingDocument.setGeneratedAt(OffsetDateTime.now()); + when(qaDocDocumentService.findLatestDocument(PROJECT_ID, 7L)) + .thenReturn(Optional.of(pendingDocument)); + + WebhookResult result = processor.process( + createPayload("feature/PROJ-123-add-login"), project, eventConsumer, Map.of()); + + assertThat(result.success()).isTrue(); + verify(qaDocGenerationService).generateQaDocumentation( + any(), anyLong(), anyInt(), anyInt(), anyMap(), any()); + verify(qaDocDocumentService).upsertLatestDocument( + eq(project), eq(7L), eq(TASK_ID), any(), eq("abc123"), anyString()); + } + @Test @DisplayName("should emit correct status events in order") void shouldEmitStatusEventsInOrder() throws IOException { @@ -749,12 +848,10 @@ void shouldHandleNullPrNumber() throws IOException { )).thenReturn("doc content"); when(taskManagementClient.findCommentByMarker(eq(TASK_ID), anyString())) .thenReturn(Optional.empty()); - when(taskManagementClient.postComment(eq(TASK_ID), anyString())) - .thenReturn(new TaskComment("c-1", "bot", "doc", OffsetDateTime.now(), null)); - WebhookResult result = processor.process(payload, project, eventConsumer, Map.of()); - assertThat(result.success()).isTrue(); + assertThat(result.success()).isFalse(); + assertThat(result.message()).contains("secure public preview"); // Should not attempt to look up analysis when PR number is null verify(codeAnalysisService, never()).getPreviousVersionCodeAnalysis(anyLong(), anyLong()); } diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthSchedulerTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthSchedulerTest.java new file mode 100644 index 00000000..c2d91458 --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/BranchHealthSchedulerTest.java @@ -0,0 +1,134 @@ +package org.rostilos.codecrow.pipelineagent.generic.service; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; +import org.rostilos.codecrow.analysisengine.processor.analysis.BranchAnalysisProcessor; +import org.rostilos.codecrow.core.model.branch.BranchHealthStatus; +import org.rostilos.codecrow.core.persistence.repository.branch.BranchRepository; +import org.rostilos.codecrow.events.EventNotificationEmitter; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.notNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class BranchHealthSchedulerTest { + + @Test + void retryRunsOutsideTransactionUsingMaterializedCandidate() throws Exception { + BranchRepository repository = mock(BranchRepository.class); + BranchAnalysisProcessor processor = mock(BranchAnalysisProcessor.class); + BranchRepository.StaleRetryCandidate candidate = candidate(2, true, "abc123"); + when(repository.findStaleRetryCandidates(BranchHealthStatus.STALE)) + .thenReturn(List.of(candidate)); + + new BranchHealthScheduler(repository, processor).retryStaleBranches(); + + verify(processor).process( + any(BranchProcessRequest.class), + notNull()); + assertThat(BranchHealthScheduler.class + .getMethod("retryStaleBranches") + .isAnnotationPresent(Transactional.class)) + .isFalse(); + } + + @Test + void scheduledRetryCanEmitProgressWithoutAnAttachedObserver() throws Exception { + BranchRepository repository = mock(BranchRepository.class); + BranchAnalysisProcessor processor = mock(BranchAnalysisProcessor.class); + BranchRepository.StaleRetryCandidate retryable = candidate(2, true, "abc123"); + when(repository.findStaleRetryCandidates(BranchHealthStatus.STALE)) + .thenReturn(List.of(retryable)); + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + java.util.function.Consumer> observer = invocation.getArgument(1); + EventNotificationEmitter.emitStatus(observer, "started", "Started"); + return Map.of("status", "accepted"); + }).when(processor).process(any(BranchProcessRequest.class), notNull()); + + new BranchHealthScheduler(repository, processor).retryStaleBranches(); + + verify(processor).process(any(BranchProcessRequest.class), notNull()); + } + + @Test + void retryCeilingDoesNotWarnEveryScheduledRun() throws Exception { + BranchRepository repository = mock(BranchRepository.class); + BranchAnalysisProcessor processor = mock(BranchAnalysisProcessor.class); + BranchRepository.StaleRetryCandidate atCeiling = candidate(10, true, "abc123"); + when(repository.findStaleRetryCandidates(BranchHealthStatus.STALE)) + .thenReturn(List.of(atCeiling)); + Logger logger = (Logger) LoggerFactory.getLogger(BranchHealthScheduler.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + try { + BranchHealthScheduler scheduler = new BranchHealthScheduler(repository, processor); + scheduler.retryStaleBranches(); + scheduler.retryStaleBranches(); + } finally { + logger.detachAppender(appender); + } + + verify(processor, never()).process(any(), any()); + assertThat(appender.list) + .noneMatch(event -> event.getLevel() == Level.WARN + && event.getFormattedMessage().contains("max retries")); + } + + @Test + void missingCommitDoesNotWarnEveryScheduledRun() throws Exception { + BranchRepository repository = mock(BranchRepository.class); + BranchAnalysisProcessor processor = mock(BranchAnalysisProcessor.class); + BranchRepository.StaleRetryCandidate missingCommit = candidate(2, true, " "); + when(repository.findStaleRetryCandidates(BranchHealthStatus.STALE)) + .thenReturn(List.of(missingCommit)); + Logger logger = (Logger) LoggerFactory.getLogger(BranchHealthScheduler.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + try { + BranchHealthScheduler scheduler = new BranchHealthScheduler(repository, processor); + scheduler.retryStaleBranches(); + scheduler.retryStaleBranches(); + } finally { + logger.detachAppender(appender); + } + + verify(processor, never()).process(any(), any()); + assertThat(appender.list) + .noneMatch(event -> event.getLevel() == Level.WARN + && event.getFormattedMessage().contains("no commit hash")); + } + + private static BranchRepository.StaleRetryCandidate candidate( + int failures, + boolean analysisEnabled, + String commitHash) { + BranchRepository.StaleRetryCandidate candidate = + mock(BranchRepository.StaleRetryCandidate.class); + when(candidate.getBranchId()).thenReturn(7L); + when(candidate.getProjectId()).thenReturn(42L); + when(candidate.getBranchName()).thenReturn("main"); + when(candidate.getCommitHash()).thenReturn(commitHash); + when(candidate.getConsecutiveFailures()).thenReturn(failures); + when(candidate.getBranchAnalysisEnabled()).thenReturn(analysisEnabled); + return candidate; + } +} diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoverySchedulerTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoverySchedulerTest.java index 8e73c437..c9aa49bf 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoverySchedulerTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/service/WebhookJobRecoverySchedulerTest.java @@ -4,6 +4,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -45,6 +46,7 @@ void replaysPersistedWebhookPayloadAfterLostDispatch() throws Exception { when(job.getId()).thenReturn(10L); when(job.getExternalId()).thenReturn("job-10"); + when(job.getJobType()).thenReturn(JobType.BRANCH_ANALYSIS); when(job.getProject()).thenReturn(project); when(project.getId()).thenReturn(20L); when(jobService.findWebhookDispatchPayload(10L)) @@ -95,4 +97,45 @@ void restoresLegacyPendingBranchJobWithoutStoredPayload() { assertThat(payload.getValue().sourceBranch()).isEqualTo("release/10x"); assertThat(payload.getValue().commitHash()).isEqualTo("abc123"); } + + @Test + void neverClaimsPendingRagChildrenOrConsultsPersistedPayload() { + Job initial = mock(Job.class); + Job incremental = mock(Job.class); + when(initial.getId()).thenReturn(12L); + when(initial.getJobType()).thenReturn(JobType.RAG_INITIAL_INDEX); + when(incremental.getId()).thenReturn(13L); + when(incremental.getJobType()).thenReturn(JobType.RAG_INCREMENTAL_INDEX); + when(jobService.findRecoverableWebhookJobs(any(), eq(50))) + .thenReturn(List.of(initial, incremental)); + scheduler.recoverAcceptedJobs(); + + verify(jobService, never()).claimRecoverableWebhookJob(eq(12L), any()); + verify(jobService, never()).claimRecoverableWebhookJob(eq(13L), any()); + verify(jobService, never()).findWebhookDispatchPayload(12L); + verify(jobService, never()).findWebhookDispatchPayload(13L); + verify(asyncProcessor, never()).processWebhookAsync( + any(), any(), any(), any(), any()); + } + + @Test + void neverClaimsAbandonedRunningRagChildren() { + Job initial = mock(Job.class); + Job incremental = mock(Job.class); + when(initial.getId()).thenReturn(14L); + when(initial.getJobType()).thenReturn(JobType.RAG_INITIAL_INDEX); + when(incremental.getId()).thenReturn(15L); + when(incremental.getJobType()).thenReturn(JobType.RAG_INCREMENTAL_INDEX); + when(jobService.findAbandonedRunningWebhookJobs(any(), eq(20))) + .thenReturn(List.of(initial, incremental)); + + scheduler.recoverAcceptedJobs(); + + verify(jobService, never()).claimAbandonedRunningWebhookJob(eq(14L), any()); + verify(jobService, never()).claimAbandonedRunningWebhookJob(eq(15L), any()); + verify(jobService, never()).findWebhookDispatchPayload(14L); + verify(jobService, never()).findWebhookDispatchPayload(15L); + verify(asyncProcessor, never()).processWebhookAsync( + any(), any(), any(), any(), any()); + } } diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListenerTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListenerTest.java index 5ba230f9..5f8e5fa3 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListenerTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaAutoDocListenerTest.java @@ -11,7 +11,6 @@ import org.rostilos.codecrow.core.service.CodeAnalysisService; import org.rostilos.codecrow.core.service.QaDocDocumentService; import org.rostilos.codecrow.events.analysis.AnalysisCompletedEvent; -import org.rostilos.codecrow.security.oauth.TokenEncryptionService; import org.rostilos.codecrow.taskmanagement.TaskManagementClientFactory; import org.rostilos.codecrow.vcsclient.VcsClientProvider; @@ -19,6 +18,7 @@ import java.util.Map; import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -43,10 +43,9 @@ class QaAutoDocListenerTest { @Mock private QaDocDocumentService qaDocDocumentService; @Mock - private PrFileEnrichmentService enrichmentService; + private QaDocPublicPreviewService qaDocPublicPreviewService; @Mock - private TokenEncryptionService tokenEncryptionService; - + private PrFileEnrichmentService enrichmentService; @Test void loadsProjectWithVcsConnectionsForAsyncProcessing() { QaAutoDocListener listener = new QaAutoDocListener( @@ -58,8 +57,8 @@ void loadsProjectWithVcsConnectionsForAsyncProcessing() { vcsClientProvider, qaDocStateRepository, qaDocDocumentService, - enrichmentService, - tokenEncryptionService); + qaDocPublicPreviewService, + enrichmentService); AnalysisCompletedEvent event = new AnalysisCompletedEvent( this, "correlation-id", @@ -81,4 +80,25 @@ void loadsProjectWithVcsConnectionsForAsyncProcessing() { verify(projectRepository).findByIdWithFullDetails(402L); verify(projectRepository, never()).findById(402L); } + + @Test + void distinguishesLegacyLinkOnlyCommentsFromRedactedFullDocuments() { + String previewUrl = "https://codecrow.cloud/share#token=ccs_opaque-token"; + + assertThat(QaAutoDocListener.isPublicPreviewOnlyComment(previewUrl)).isTrue(); + assertThat(QaAutoDocListener.isPublicPreviewOnlyComment( + "[View QA test cases in CodeCrow](" + previewUrl + ")")) + .isTrue(); + assertThat(QaAutoDocListener.isPublicPreviewOnlyComment(""" + ### 1. Change Summary + Login validation changed. + + ### 3. Test Scenarios + %s + + ### 4. Edge Cases + Verify an empty password. + """.formatted(previewUrl))) + .isFalse(); + } } diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationServiceTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationServiceTest.java index 51c2d04b..fa38553c 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationServiceTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocGenerationServiceTest.java @@ -17,6 +17,7 @@ import org.springframework.test.util.ReflectionTestUtils; import java.io.IOException; +import java.net.http.HttpClient; import java.time.OffsetDateTime; import java.util.Base64; import java.util.LinkedHashMap; @@ -86,6 +87,14 @@ private QaAutoDocConfig customTemplateConfig() { ); } + @Test + void internalInferenceClientUsesHttp11WithoutH2cUpgrade() { + HttpClient configuredClient = (HttpClient) ReflectionTestUtils.getField(service, "httpClient"); + + assertThat(configuredClient).isNotNull(); + assertThat(configuredClient.version()).isEqualTo(HttpClient.Version.HTTP_1_1); + } + private TaskDetails sampleTaskDetails() { return new TaskDetails( "PROJ-123", "Implement feature", "Full description", @@ -150,6 +159,9 @@ void shouldSendCorrectPayloadAndReturnDoc() throws Exception { assertThat(body.get("template_mode").asText()).isEqualTo("BASE"); assertThat(body.has("pr_metadata")).isTrue(); assertThat(body.get("pr_metadata").get("sourceBranch").asText()).isEqualTo("feature/PROJ-123"); + assertThat(body.has("oauth_key")).isFalse(); + assertThat(body.has("oauth_secret")).isFalse(); + assertThat(body.has("bearer_token")).isFalse(); // Verify task context (keys must match Python placeholder names) JsonNode taskContext = body.get("task_context"); diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocHandoffRetryPolicyTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocHandoffRetryPolicyTest.java new file mode 100644 index 00000000..020f3a55 --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocHandoffRetryPolicyTest.java @@ -0,0 +1,65 @@ +package org.rostilos.codecrow.pipelineagent.qadoc; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; +import org.rostilos.codecrow.core.model.qadoc.QaDocState; + +import java.time.OffsetDateTime; + +import static org.assertj.core.api.Assertions.assertThat; + +class QaDocHandoffRetryPolicyTest { + + @Test + void reusesCurrentCommitDocumentCreatedAfterTheLastSuccessfulHandoff() { + QaDocDocument document = document("commit-b", OffsetDateTime.parse("2026-08-12T10:05:00Z")); + QaDocState state = state("commit-a", OffsetDateTime.parse("2026-08-12T10:00:00Z")); + + assertThat(QaDocHandoffRetryPolicy.shouldReuse( + document, state, "commit-b", "task-1")).isTrue(); + } + + @Test + void regeneratesAfterASuccessfulHandoffOrForAnotherCommit() { + QaDocDocument delivered = document("commit-b", OffsetDateTime.parse("2026-08-12T10:00:00Z")); + QaDocState state = state("commit-b", OffsetDateTime.parse("2026-08-12T10:05:00Z")); + + assertThat(QaDocHandoffRetryPolicy.shouldReuse( + delivered, state, "commit-b", "TASK-1")).isFalse(); + assertThat(QaDocHandoffRetryPolicy.shouldReuse( + delivered, state, "commit-c", "TASK-1")).isFalse(); + } + + @Test + void doesNotReuseBlankDocument() { + QaDocDocument document = document("commit-a", OffsetDateTime.now()); + document.setMarkdownContent(" "); + + assertThat(QaDocHandoffRetryPolicy.shouldReuse( + document, null, "commit-a", "TASK-1")).isFalse(); + } + + @Test + void doesNotReuseDocumentGeneratedForAnotherTask() { + QaDocDocument document = document("commit-a", OffsetDateTime.now()); + + assertThat(QaDocHandoffRetryPolicy.shouldReuse( + document, null, "commit-a", "TASK-2")).isFalse(); + } + + private static QaDocDocument document(String commit, OffsetDateTime generatedAt) { + QaDocDocument document = new QaDocDocument(null, 17L); + document.setCommitHash(commit); + document.setGeneratedAt(generatedAt); + document.setMarkdownContent("# QA document"); + document.setTaskId("TASK-1"); + return document; + } + + private static QaDocState state(String commit, OffsetDateTime generatedAt) { + QaDocState state = new QaDocState(); + state.setLastCommitHash(commit); + state.setLastGeneratedAt(generatedAt); + return state; + } +} diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocPublicPreviewServiceTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocPublicPreviewServiceTest.java new file mode 100644 index 00000000..29eafb20 --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/qadoc/QaDocPublicPreviewServiceTest.java @@ -0,0 +1,99 @@ +package org.rostilos.codecrow.pipelineagent.qadoc; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.dto.admin.BaseUrlSettingsDTO; +import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; +import org.rostilos.codecrow.core.service.SiteSettingsProvider; +import org.rostilos.codecrow.core.service.qadoc.QaDocPublicShareResource; +import org.rostilos.codecrow.publicshare.api.IssuedPublicShare; +import org.rostilos.codecrow.publicshare.service.PublicShareLinkService; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class QaDocPublicPreviewServiceTest { + + @Test + void createsAnOpaquePublicLinkAndRedactsShareableBodiesFromTheTaskComment() { + PublicShareLinkService publicShares = mock(PublicShareLinkService.class); + SiteSettingsProvider settings = mock(SiteSettingsProvider.class); + when(publicShares.issue(QaDocPublicShareResource.DOCUMENT, "88")) + .thenReturn(new IssuedPublicShare("ccs_opaque-public-credential")); + when(settings.getBaseUrlSettings()) + .thenReturn(new BaseUrlSettingsDTO( + "https://api.codecrow.example", + "https://app.codecrow.example/", + "https://hooks.codecrow.example" + )); + QaDocPublicPreviewService service = new QaDocPublicPreviewService(publicShares, settings); + QaDocDocument document = new QaDocDocument(null, 17L); + document.setId(88L); + document.setMarkdownContent(""" + ### 1. Change Summary + PRESERVED OVERVIEW CONTENT + + ### 2. Scope + PRESERVED SCOPE CONTENT + + + ### 3. Test Scenarios + **A shared scenario** (HIGH) + - **Expected Result:** REDACTED TEST DETAILS + + ### 4. Edge Cases + PRESERVED EDGE CASE CONTENT + + ### 5. Regression Risks + PRESERVED REGRESSION RISK CONTENT + + ### 6. Environment and Setup Notes + PRESERVED SETUP CONTENT + + + --- + *🐦 Generated by [CodeCrow](https://codecrow.app) QA Auto-Documentation* + + """); + + String previewUrl = service.createPreviewUrl(document); + String comment = service.buildTaskComment(document.getMarkdownContent(), previewUrl); + + assertThat(previewUrl) + .isEqualTo("https://app.codecrow.example/share#token=ccs_opaque-public-credential"); + assertThat(comment) + .contains(QaAutoDocListener.COMMENT_MARKER) + .contains("### 1. Change Summary", "PRESERVED OVERVIEW CONTENT") + .contains("### 2. Scope", "PRESERVED SCOPE CONTENT") + .contains("### 3. Test Scenarios\n\n" + previewUrl + "&tab=test-cases") + .contains("### 4. Edge Cases", "PRESERVED EDGE CASE CONTENT") + .contains("### 5. Regression Risks", "PRESERVED REGRESSION RISK CONTENT") + .contains( + "### 6. Environment and Setup Notes\n\n" + + previewUrl + "&tab=environment", + "Generated by [CodeCrow]", + "") + .doesNotContain( + "A shared scenario", + "REDACTED TEST DETAILS", + "PRESERVED SETUP CONTENT") + .doesNotContain("[View QA test cases in CodeCrow]"); + verify(publicShares).issue(QaDocPublicShareResource.DOCUMENT, "88"); + } + + @Test + void refusesToIssueALinkForAnUnmarkedDocument() { + PublicShareLinkService publicShares = mock(PublicShareLinkService.class); + SiteSettingsProvider settings = mock(SiteSettingsProvider.class); + QaDocPublicPreviewService service = new QaDocPublicPreviewService(publicShares, settings); + QaDocDocument document = new QaDocDocument(null, 17L); + document.setId(88L); + document.setMarkdownContent("SECRET OVERVIEW CONTENT"); + + org.assertj.core.api.Assertions.assertThatThrownBy( + () -> service.createPreviewUrl(document)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("marked QA test-case section"); + } +} diff --git a/java-ecosystem/services/web-server/pom.xml b/java-ecosystem/services/web-server/pom.xml index dbe365c6..ddb172b1 100644 --- a/java-ecosystem/services/web-server/pom.xml +++ b/java-ecosystem/services/web-server/pom.xml @@ -77,6 +77,11 @@ codecrow-core + + org.rostilos.codecrow + codecrow-public-share + + org.rostilos.codecrow codecrow-commit-graph diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/analysis/dto/response/QaDocDocumentResponse.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/analysis/dto/response/QaDocDocumentResponse.java index 341af568..962780bc 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/analysis/dto/response/QaDocDocumentResponse.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/analysis/dto/response/QaDocDocumentResponse.java @@ -1,8 +1,11 @@ package org.rostilos.codecrow.webserver.analysis.dto.response; import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; +import org.rostilos.codecrow.core.service.qadoc.QaDocContent; +import org.rostilos.codecrow.core.service.qadoc.QaDocContentParser; import java.time.OffsetDateTime; +import java.util.List; public record QaDocDocumentResponse( boolean available, @@ -11,13 +14,18 @@ public record QaDocDocumentResponse( Long lastAnalysisId, String commitHash, String markdownContent, + String overviewMarkdown, + List testCases, + String environmentMarkdown, OffsetDateTime generatedAt ) { public static QaDocDocumentResponse missing(Long prNumber) { - return new QaDocDocumentResponse(false, prNumber, null, null, null, null, null); + return new QaDocDocumentResponse( + false, prNumber, null, null, null, null, null, List.of(), null, null); } public static QaDocDocumentResponse fromDocument(QaDocDocument document) { + QaDocContent content = QaDocContentParser.parse(document.getMarkdownContent()); return new QaDocDocumentResponse( true, document.getPrNumber(), @@ -25,6 +33,9 @@ public static QaDocDocumentResponse fromDocument(QaDocDocument document) { document.getLastAnalysisId(), document.getCommitHash(), document.getMarkdownContent(), + content.overviewMarkdown(), + content.testCases().stream().map(QaDocTestCaseResponse::fromTestCase).toList(), + content.environmentMarkdown(), document.getGeneratedAt() ); } diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/analysis/dto/response/QaDocTestCaseResponse.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/analysis/dto/response/QaDocTestCaseResponse.java new file mode 100644 index 00000000..6837a850 --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/analysis/dto/response/QaDocTestCaseResponse.java @@ -0,0 +1,20 @@ +package org.rostilos.codecrow.webserver.analysis.dto.response; + +import org.rostilos.codecrow.core.service.qadoc.QaDocTestCase; + +public record QaDocTestCaseResponse( + String title, + String priority, + String functionalArea, + String descriptionMarkdown +) { + public static QaDocTestCaseResponse fromTestCase(QaDocTestCase testCase) { + return new QaDocTestCaseResponse( + testCase.title(), + testCase.priority(), + testCase.functionalArea(), + testCase.descriptionMarkdown() + ); + } +} + diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicShareController.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicShareController.java new file mode 100644 index 00000000..14b8ab34 --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicShareController.java @@ -0,0 +1,72 @@ +package org.rostilos.codecrow.webserver.publicshare; + +import org.rostilos.codecrow.publicshare.api.ResolvedPublicShare; +import org.rostilos.codecrow.publicshare.service.PublicShareLinkService; +import org.springframework.http.CacheControl; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@RestController +@RequestMapping("/api/public/shares") +public class PublicShareController { + + private final PublicShareLinkService shareLinkService; + private final Map providers; + + public PublicShareController(PublicShareLinkService shareLinkService, + List providers) { + this.shareLinkService = shareLinkService; + this.providers = providers.stream().collect(Collectors.toUnmodifiableMap( + PublicShareResourceProvider::resourceType, + Function.identity() + )); + } + + @PostMapping("/resolve") + public ResponseEntity resolvePublicPreview(@RequestBody PublicShareResolveRequest request, + Authentication authentication) { + String token = request == null ? null : request.token(); + return shareLinkService.resolve(token) + .flatMap(share -> resolvePreview(share, authentication)) + .map(this::ok) + .orElseGet(this::notFound); + } + + private ResponseEntity ok(Object body) { + return ResponseEntity.ok() + .cacheControl(CacheControl.noStore()) + .header("Referrer-Policy", "no-referrer") + .body(body); + } + + private ResponseEntity notFound() { + return ResponseEntity.status(404) + .cacheControl(CacheControl.noStore()) + .header("Referrer-Policy", "no-referrer") + .build(); + } + + private java.util.Optional resolvePreview( + ResolvedPublicShare share, + Authentication authentication) { + PublicShareResourceProvider provider = providers.get(share.resourceType()); + if (provider == null) { + return java.util.Optional.empty(); + } + return provider.getPublicPreview(share.resourceKey()) + .map(content -> new PublicSharePreviewResponse( + share.resourceType(), + content, + provider.getAuthorizedPath(share.resourceKey(), authentication).orElse(null) + )); + } +} diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicSharePreviewResponse.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicSharePreviewResponse.java new file mode 100644 index 00000000..d59769c9 --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicSharePreviewResponse.java @@ -0,0 +1,13 @@ +package org.rostilos.codecrow.webserver.publicshare; + +/** + * Generic public-share transport envelope. The content is always the provider's + * sanitized public DTO. The optional path is emitted only after normal tenant + * authorization succeeds for the authenticated principal. + */ +public record PublicSharePreviewResponse( + String resourceType, + Object content, + String authorizedPath +) { +} diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicShareResolveRequest.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicShareResolveRequest.java new file mode 100644 index 00000000..17c1cc3f --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicShareResolveRequest.java @@ -0,0 +1,4 @@ +package org.rostilos.codecrow.webserver.publicshare; + +public record PublicShareResolveRequest(String token) { +} diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicShareResourceProvider.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicShareResourceProvider.java new file mode 100644 index 00000000..1a505f5b --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/PublicShareResourceProvider.java @@ -0,0 +1,21 @@ +package org.rostilos.codecrow.webserver.publicshare; + +import org.springframework.security.core.Authentication; + +import java.util.Optional; + +/** Resolves one internal resource type into its explicitly sanitized public DTO. */ +public interface PublicShareResourceProvider { + + String resourceType(); + + Optional getPublicPreview(String resourceKey); + + /** + * Returns a protected in-app destination only when the current principal is + * authorized for the underlying tenant resource. + */ + default Optional getAuthorizedPath(String resourceKey, Authentication authentication) { + return Optional.empty(); + } +} diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/qadoc/QaDocPublicPreview.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/qadoc/QaDocPublicPreview.java new file mode 100644 index 00000000..6b55c3c5 --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/qadoc/QaDocPublicPreview.java @@ -0,0 +1,21 @@ +package org.rostilos.codecrow.webserver.publicshare.qadoc; + +import org.rostilos.codecrow.webserver.analysis.dto.response.QaDocTestCaseResponse; + +import java.util.List; + +/** + * QA content returned only after a public-share credential resolves. It mirrors + * the QA document tabs while excluding workspace details and internal document, + * project, pull-request, analysis, and commit identifiers. + */ +public record QaDocPublicPreview( + String title, + String projectName, + String taskKey, + String taskSummary, + String overviewMarkdown, + List testCases, + String environmentMarkdown +) { +} diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/qadoc/QaDocShareProvider.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/qadoc/QaDocShareProvider.java new file mode 100644 index 00000000..07769e1b --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/publicshare/qadoc/QaDocShareProvider.java @@ -0,0 +1,155 @@ +package org.rostilos.codecrow.webserver.publicshare.qadoc; + +import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; +import org.rostilos.codecrow.core.service.CodeAnalysisService; +import org.rostilos.codecrow.core.service.QaDocDocumentService; +import org.rostilos.codecrow.core.service.qadoc.QaDocContent; +import org.rostilos.codecrow.core.service.qadoc.QaDocContentParser; +import org.rostilos.codecrow.core.service.qadoc.QaDocPublicShareResource; +import org.rostilos.codecrow.security.service.UserDetailsImpl; +import org.rostilos.codecrow.security.web.WorkspaceSecurity; +import org.rostilos.codecrow.webserver.analysis.dto.response.QaDocTestCaseResponse; +import org.rostilos.codecrow.webserver.publicshare.PublicShareResourceProvider; +import org.springframework.security.core.Authentication; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.util.UriComponentsBuilder; + +import java.util.Objects; +import java.util.Optional; + +/** Resolves the explicitly shareable Overview, Test cases, and Environment QA tabs. */ +@Component +public class QaDocShareProvider implements PublicShareResourceProvider { + + private final QaDocDocumentService qaDocDocumentService; + private final CodeAnalysisService codeAnalysisService; + private final WorkspaceSecurity workspaceSecurity; + + public QaDocShareProvider(QaDocDocumentService qaDocDocumentService, + CodeAnalysisService codeAnalysisService, + WorkspaceSecurity workspaceSecurity) { + this.qaDocDocumentService = qaDocDocumentService; + this.codeAnalysisService = codeAnalysisService; + this.workspaceSecurity = workspaceSecurity; + } + + @Override + public String resourceType() { + return QaDocPublicShareResource.DOCUMENT; + } + + @Override + @Transactional(readOnly = true) + public Optional getPublicPreview(String resourceKey) { + return parseDocumentId(resourceKey) + .flatMap(qaDocDocumentService::findDocumentById) + .flatMap(this::toPublicPreview); + } + + @Override + @Transactional(readOnly = true) + public Optional getAuthorizedPath(String resourceKey, Authentication authentication) { + if (authentication == null || !authentication.isAuthenticated() + || !(authentication.getPrincipal() instanceof UserDetailsImpl)) { + return Optional.empty(); + } + + return parseDocumentId(resourceKey) + .flatMap(qaDocDocumentService::findDocumentById) + .flatMap(document -> authorizedDocumentPath(document, authentication)); + } + + private Optional toPublicPreview(QaDocDocument document) { + var testCases = QaDocContentParser.parseMarkedTestCases(document.getMarkdownContent()); + if (testCases.isEmpty()) { + return Optional.empty(); + } + QaDocContent content = QaDocContentParser.parse(document.getMarkdownContent()); + + String projectName = document.getProject() == null + ? null + : normalize(document.getProject().getName()); + String taskKey = normalize(document.getTaskId()); + + return Optional.of(new QaDocPublicPreview( + "QA documentation", + projectName, + taskKey, + findTaskSummary(document), + content.overviewMarkdown(), + testCases.stream() + .map(QaDocTestCaseResponse::fromTestCase) + .toList(), + content.environmentMarkdown() + )); + } + + private Optional authorizedDocumentPath( + QaDocDocument document, + Authentication authentication) { + Project project = document.getProject(); + if (project == null || project.getId() == null || project.getWorkspace() == null + || !workspaceSecurity.isProjectWorkspaceMember(project.getId(), authentication)) { + return Optional.empty(); + } + + String workspaceSlug = normalize(project.getWorkspace().getSlug()); + String projectNamespace = normalize(project.getNamespace()); + if (workspaceSlug == null || projectNamespace == null || document.getPrNumber() == null) { + return Optional.empty(); + } + + return Optional.of(UriComponentsBuilder + .fromPath("/dashboard/{workspaceSlug}/projects/{projectNamespace}") + .queryParam("prNumber", document.getPrNumber()) + .queryParam("subTab", "qa-doc") + .buildAndExpand(workspaceSlug, projectNamespace) + .encode() + .toUriString()); + } + + private String findTaskSummary(QaDocDocument document) { + if (document.getLastAnalysisId() == null || document.getProject() == null + || document.getProject().getId() == null) { + return null; + } + + Long documentProjectId = document.getProject().getId(); + String documentTaskKey = normalize(document.getTaskId()); + return codeAnalysisService.findById(document.getLastAnalysisId()) + .filter(analysis -> belongsToProject(analysis, documentProjectId)) + .filter(analysis -> hasCompatibleTaskKey(analysis, documentTaskKey)) + .map(CodeAnalysis::getTaskSummary) + .map(QaDocShareProvider::normalize) + .orElse(null); + } + + private boolean belongsToProject(CodeAnalysis analysis, Long projectId) { + return analysis.getProject() != null + && Objects.equals(analysis.getProject().getId(), projectId); + } + + private boolean hasCompatibleTaskKey(CodeAnalysis analysis, String documentTaskKey) { + String analysisTaskKey = normalize(analysis.getTaskId()); + return documentTaskKey == null || analysisTaskKey == null + || Objects.equals(documentTaskKey, analysisTaskKey); + } + + private static String normalize(String value) { + if (value == null || value.isBlank()) { + return null; + } + return value.trim(); + } + + private static Optional parseDocumentId(String resourceKey) { + try { + return Optional.of(Long.valueOf(resourceKey)); + } catch (NumberFormatException ignored) { + return Optional.empty(); + } + } +} diff --git a/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/publicshare/PublicShareControllerTest.java b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/publicshare/PublicShareControllerTest.java new file mode 100644 index 00000000..b9cf6c16 --- /dev/null +++ b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/publicshare/PublicShareControllerTest.java @@ -0,0 +1,63 @@ +package org.rostilos.codecrow.webserver.publicshare; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.publicshare.api.ResolvedPublicShare; +import org.rostilos.codecrow.publicshare.service.PublicShareLinkService; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class PublicShareControllerTest { + + @Test + void resolvesThroughTheRegisteredSanitizingProvider() { + PublicShareLinkService links = mock(PublicShareLinkService.class); + PublicShareResourceProvider provider = mock(PublicShareResourceProvider.class); + Authentication authentication = mock(Authentication.class); + when(provider.resourceType()).thenReturn("safe-preview"); + doReturn(Optional.of(Map.of("title", "Shared content"))) + .when(provider).getPublicPreview("internal-9"); + when(provider.getAuthorizedPath("internal-9", authentication)) + .thenReturn(Optional.of("/dashboard/acme/projects/shop?prNumber=9&subTab=qa-doc")); + when(links.resolve("ccs_public-token")) + .thenReturn(Optional.of(new ResolvedPublicShare("safe-preview", "internal-9"))); + PublicShareController controller = new PublicShareController(links, List.of(provider)); + + ResponseEntity response = controller.resolvePublicPreview( + new PublicShareResolveRequest("ccs_public-token"), + authentication + ); + + assertThat(response.getStatusCode().value()).isEqualTo(200); + assertThat(response.getBody()).isEqualTo(new PublicSharePreviewResponse( + "safe-preview", + Map.of("title", "Shared content"), + "/dashboard/acme/projects/shop?prNumber=9&subTab=qa-doc" + )); + assertThat(response.getHeaders().getCacheControl()).isEqualTo("no-store"); + assertThat(response.getHeaders().getFirst("Referrer-Policy")).isEqualTo("no-referrer"); + } + + @Test + void usesTheSameNotFoundResponseForInvalidAndUnsupportedTokens() { + PublicShareLinkService links = mock(PublicShareLinkService.class); + PublicShareResourceProvider provider = mock(PublicShareResourceProvider.class); + when(provider.resourceType()).thenReturn("safe-preview"); + when(links.resolve("invalid")).thenReturn(Optional.empty()); + PublicShareController controller = new PublicShareController(links, List.of(provider)); + + assertThat(controller.resolvePublicPreview(new PublicShareResolveRequest("invalid"), null) + .getStatusCode().value()).isEqualTo(404); + verify(provider, never()).getPublicPreview(org.mockito.ArgumentMatchers.anyString()); + } +} diff --git a/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/publicshare/qadoc/QaDocShareProviderTest.java b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/publicshare/qadoc/QaDocShareProviderTest.java new file mode 100644 index 00000000..6af3a1f2 --- /dev/null +++ b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/publicshare/qadoc/QaDocShareProviderTest.java @@ -0,0 +1,170 @@ +package org.rostilos.codecrow.webserver.publicshare.qadoc; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.qadoc.QaDocDocument; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.service.CodeAnalysisService; +import org.rostilos.codecrow.core.service.QaDocDocumentService; +import org.rostilos.codecrow.security.service.UserDetailsImpl; +import org.rostilos.codecrow.security.web.WorkspaceSecurity; +import org.springframework.security.core.Authentication; + +import java.util.Arrays; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class QaDocShareProviderTest { + + @Test + void returnsTheExplicitlyShareableDocumentTabs() { + QaDocDocumentService documents = mock(QaDocDocumentService.class); + CodeAnalysisService analyses = mock(CodeAnalysisService.class); + WorkspaceSecurity workspaceSecurity = mock(WorkspaceSecurity.class); + Project project = mock(Project.class); + when(project.getId()).thenReturn(12L); + when(project.getName()).thenReturn("Acme Checkout"); + + QaDocDocument document = new QaDocDocument(project, 17L); + document.setId(88L); + document.setTaskId("SHOP-42"); + document.setLastAnalysisId(71L); + document.setCommitHash("0123456789012345678901234567890123456789"); + document.setMarkdownContent(""" + # QA Testing Guide — Saved cards + ## 1. What Changed + Customers can reuse a saved card. + + + ### Test Scenarios + **Save the form** (HIGH) + - **Expected Result:** The confirmation appears + + + ## 6. Setup and Environment Notes + - Enable saved cards in the QA environment. + """); + CodeAnalysis analysis = mock(CodeAnalysis.class); + when(analysis.getProject()).thenReturn(project); + when(analysis.getTaskId()).thenReturn("SHOP-42"); + when(analysis.getTaskSummary()).thenReturn("Add a saved-card checkout flow"); + when(documents.findDocumentById(88L)).thenReturn(Optional.of(document)); + when(analyses.findById(71L)).thenReturn(Optional.of(analysis)); + QaDocShareProvider provider = new QaDocShareProvider( + documents, analyses, workspaceSecurity); + + QaDocPublicPreview preview = provider.getPublicPreview("88").orElseThrow(); + + assertThat(preview.title()).isEqualTo("QA documentation"); + assertThat(preview.projectName()).isEqualTo("Acme Checkout"); + assertThat(preview.taskKey()).isEqualTo("SHOP-42"); + assertThat(preview.taskSummary()).isEqualTo("Add a saved-card checkout flow"); + assertThat(preview.overviewMarkdown()) + .contains("What Changed", "Customers can reuse a saved card") + .doesNotContain("Save the form", "Setup and Environment Notes"); + assertThat(preview.testCases()).singleElement() + .satisfies(testCase -> assertThat(testCase.title()).isEqualTo("Save the form")); + assertThat(preview.environmentMarkdown()) + .isEqualTo("- Enable saved cards in the QA environment."); + assertThat(Arrays.stream(QaDocPublicPreview.class.getRecordComponents()) + .map(component -> component.getName())) + .containsExactly( + "title", "projectName", "taskKey", "taskSummary", + "overviewMarkdown", "testCases", "environmentMarkdown") + .doesNotContain("id", "project", "workspace", "prNumber", "taskId", "commitHash"); + } + + @Test + void neverFallsBackToSharingAnUnmarkedLegacyDocument() { + QaDocDocumentService documents = mock(QaDocDocumentService.class); + CodeAnalysisService analyses = mock(CodeAnalysisService.class); + WorkspaceSecurity workspaceSecurity = mock(WorkspaceSecurity.class); + QaDocDocument document = new QaDocDocument(null, 17L); + document.setId(89L); + document.setMarkdownContent(""" + # Secret project overview + Workspace: Acme + **A legacy test** (HIGH) + - **Expected Result:** It works + """); + when(documents.findDocumentById(89L)).thenReturn(Optional.of(document)); + QaDocShareProvider provider = new QaDocShareProvider( + documents, analyses, workspaceSecurity); + + assertThat(provider.getPublicPreview("89")).isEmpty(); + } + + @Test + void doesNotExposeTaskSummaryFromAnotherProject() { + QaDocDocumentService documents = mock(QaDocDocumentService.class); + CodeAnalysisService analyses = mock(CodeAnalysisService.class); + WorkspaceSecurity workspaceSecurity = mock(WorkspaceSecurity.class); + Project documentProject = mock(Project.class); + Project analysisProject = mock(Project.class); + when(documentProject.getId()).thenReturn(12L); + when(documentProject.getName()).thenReturn("Checkout"); + when(analysisProject.getId()).thenReturn(99L); + + QaDocDocument document = new QaDocDocument(documentProject, 17L); + document.setId(90L); + document.setTaskId("SHOP-42"); + document.setLastAnalysisId(72L); + document.setMarkdownContent(""" + + ### Test Scenarios + **Save the form** (HIGH) + - **Expected Result:** The confirmation appears + + """); + CodeAnalysis analysis = mock(CodeAnalysis.class); + when(analysis.getProject()).thenReturn(analysisProject); + when(analysis.getTaskId()).thenReturn("SHOP-42"); + when(analysis.getTaskSummary()).thenReturn("Secret from another project"); + when(documents.findDocumentById(90L)).thenReturn(Optional.of(document)); + when(analyses.findById(72L)).thenReturn(Optional.of(analysis)); + + QaDocPublicPreview preview = new QaDocShareProvider( + documents, analyses, workspaceSecurity) + .getPublicPreview("90") + .orElseThrow(); + + assertThat(preview.projectName()).isEqualTo("Checkout"); + assertThat(preview.taskKey()).isEqualTo("SHOP-42"); + assertThat(preview.taskSummary()).isNull(); + } + + @Test + void returnsTheRealQaDocRouteOnlyForAnAuthorizedWorkspaceMember() { + QaDocDocumentService documents = mock(QaDocDocumentService.class); + CodeAnalysisService analyses = mock(CodeAnalysisService.class); + WorkspaceSecurity workspaceSecurity = mock(WorkspaceSecurity.class); + Project project = mock(Project.class); + Workspace workspace = mock(Workspace.class); + Authentication authentication = mock(Authentication.class); + UserDetailsImpl principal = mock(UserDetailsImpl.class); + when(authentication.isAuthenticated()).thenReturn(true); + when(authentication.getPrincipal()).thenReturn(principal); + when(project.getId()).thenReturn(12L); + when(project.getNamespace()).thenReturn("checkout-service"); + when(project.getWorkspace()).thenReturn(workspace); + when(workspace.getSlug()).thenReturn("acme"); + when(workspaceSecurity.isProjectWorkspaceMember(12L, authentication)).thenReturn(true); + + QaDocDocument document = new QaDocDocument(project, 524L); + document.setId(91L); + when(documents.findDocumentById(91L)).thenReturn(Optional.of(document)); + QaDocShareProvider provider = new QaDocShareProvider( + documents, analyses, workspaceSecurity); + + assertThat(provider.getAuthorizedPath("91", authentication)) + .contains("/dashboard/acme/projects/checkout-service?prNumber=524&subTab=qa-doc"); + + when(workspaceSecurity.isProjectWorkspaceMember(12L, authentication)).thenReturn(false); + assertThat(provider.getAuthorizedPath("91", authentication)).isEmpty(); + assertThat(provider.getAuthorizedPath("91", null)).isEmpty(); + } +} diff --git a/python-ecosystem/inference-orchestrator/integration/test_rag_client_extended.py b/python-ecosystem/inference-orchestrator/integration/test_rag_client_extended.py index c82f46f3..dff06c8e 100644 --- a/python-ecosystem/inference-orchestrator/integration/test_rag_client_extended.py +++ b/python-ecosystem/inference-orchestrator/integration/test_rag_client_extended.py @@ -314,14 +314,14 @@ async def test_delete_pr_files_success(rag_client): @pytest.mark.asyncio(loop_scope="function") @respx.mock async def test_delete_pr_files_not_found(rag_client): - """Collection doesn't exist → skipped status → False.""" + """An already-absent overlay is an idempotent cleanup success.""" respx.delete("http://rag-pipeline:8001/index/pr-files/ws/proj/99").mock( return_value=httpx.Response(200, json={"status": "skipped"}) ) result = await rag_client.delete_pr_files( workspace="ws", project="proj", pr_number=99, ) - assert result is False # status != "deleted" + assert result is True await rag_client.close() diff --git a/python-ecosystem/inference-orchestrator/src/api/app.py b/python-ecosystem/inference-orchestrator/src/api/app.py index db79f4c6..52e653da 100644 --- a/python-ecosystem/inference-orchestrator/src/api/app.py +++ b/python-ecosystem/inference-orchestrator/src/api/app.py @@ -4,6 +4,7 @@ Creates and configures the FastAPI application with all routers. Uses lifespan context manager for proper startup/shutdown of shared resources. """ +import asyncio import os import logging from contextlib import asynccontextmanager @@ -48,11 +49,22 @@ async def lifespan(app: FastAPI): # --- Shutdown --- logger.info("Shutting down application services...") + consumer_stops = [] if hasattr(app.state, "queue_consumer"): - await app.state.queue_consumer.stop() - + consumer_stops.append(app.state.queue_consumer.stop()) if hasattr(app.state, "command_queue_consumer"): - await app.state.command_queue_consumer.stop() + consumer_stops.append(app.state.command_queue_consumer.stop()) + if consumer_stops: + # Stop both intake loops immediately. A sequential drain could let the + # second consumer keep admitting work for the full duration of a long + # review shutdown. + stop_results = await asyncio.gather( + *consumer_stops, + return_exceptions=True, + ) + for stop_result in stop_results: + if isinstance(stop_result, BaseException): + logger.warning("Error stopping queue consumer: %s", stop_result) # Close the RagClient HTTP pools owned by each service try: diff --git a/python-ecosystem/inference-orchestrator/src/api/routers/commands.py b/python-ecosystem/inference-orchestrator/src/api/routers/commands.py index de6046e6..58f9794b 100644 --- a/python-ecosystem/inference-orchestrator/src/api/routers/commands.py +++ b/python-ecosystem/inference-orchestrator/src/api/routers/commands.py @@ -3,6 +3,7 @@ """ import json import asyncio +import logging from typing import Dict, Any from fastapi import APIRouter, Request from starlette.responses import StreamingResponse @@ -14,6 +15,7 @@ from service.command.command_service import CommandService router = APIRouter(tags=["commands"]) +logger = logging.getLogger(__name__) def get_command_service(request: Request) -> CommandService: @@ -50,29 +52,52 @@ async def summarize_endpoint(req: SummarizeRequestDto, request: Request): # Streaming behavior async def event_stream(): queue = asyncio.Queue() - - yield _json_event({"type": "status", "state": "queued", "message": "summarize request received"}) - - def event_callback(event: Dict[str, Any]): - try: - queue.put_nowait(event) - except asyncio.QueueFull: - pass - - async def runner(): - try: - result = await command_service.process_summarize(req, event_callback=event_callback) - await queue.put({ - "type": "final", - "result": result - }) - except Exception as e: - await queue.put({"type": "error", "message": str(e)}) - - task = asyncio.create_task(runner()) - - async for event in _drain_queue_until_final(queue, task): - yield _json_event(event) + task = None + terminal_event_type = None + + try: + yield _json_event({"type": "status", "state": "queued", "message": "summarize request received"}) + + def event_callback(event: Dict[str, Any]): + nonlocal terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring streamed summarize event type=%s after " + "terminal type=%s", + event_type, + terminal_event_type, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type + try: + queue.put_nowait(event) + except asyncio.QueueFull: + pass + + async def runner(): + try: + result = await command_service.process_summarize(req, event_callback=event_callback) + event_callback({ + "type": "final", + "result": result + }) + except Exception as e: + event_callback({"type": "error", "message": str(e)}) + + task = asyncio.create_task(runner()) + + async for event in _drain_queue_until_final(queue, task): + yield _json_event(event) + finally: + if task is not None: + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass return StreamingResponse(event_stream(), media_type="application/x-ndjson") @@ -107,29 +132,52 @@ async def ask_endpoint(req: AskRequestDto, request: Request): # Streaming behavior async def event_stream(): queue = asyncio.Queue() - - yield _json_event({"type": "status", "state": "queued", "message": "ask request received"}) - - def event_callback(event: Dict[str, Any]): - try: - queue.put_nowait(event) - except asyncio.QueueFull: - pass - - async def runner(): - try: - result = await command_service.process_ask(req, event_callback=event_callback) - await queue.put({ - "type": "final", - "result": result - }) - except Exception as e: - await queue.put({"type": "error", "message": str(e)}) - - task = asyncio.create_task(runner()) - - async for event in _drain_queue_until_final(queue, task): - yield _json_event(event) + task = None + terminal_event_type = None + + try: + yield _json_event({"type": "status", "state": "queued", "message": "ask request received"}) + + def event_callback(event: Dict[str, Any]): + nonlocal terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring streamed ask event type=%s after terminal " + "type=%s", + event_type, + terminal_event_type, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type + try: + queue.put_nowait(event) + except asyncio.QueueFull: + pass + + async def runner(): + try: + result = await command_service.process_ask(req, event_callback=event_callback) + event_callback({ + "type": "final", + "result": result + }) + except Exception as e: + event_callback({"type": "error", "message": str(e)}) + + task = asyncio.create_task(runner()) + + async for event in _drain_queue_until_final(queue, task): + yield _json_event(event) + finally: + if task is not None: + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass return StreamingResponse(event_stream(), media_type="application/x-ndjson") diff --git a/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py b/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py index cbfdd63b..a04a0242 100644 --- a/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py +++ b/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py @@ -1,8 +1,8 @@ """ QA Auto-Documentation Router. -Generates QA-oriented documentation for PR changes, intended to be posted -as a comment on the linked task management ticket (e.g., Jira). +Generates QA-oriented documentation for PR changes. The full document is kept +in CodeCrow; task-management comments receive a public test-case preview link. Supports both single-pass (legacy / small PRs) and multi-stage ULTRATHINKING pipeline (large PRs with enrichment data). @@ -59,18 +59,13 @@ class QaDocumentationRequest(BaseModel): # Changed file paths extracted from diff changed_file_paths: Optional[List[str]] = None - # VCS connection info (for RAG queries) + # Non-secret VCS identifiers used in document context vcs_provider: Optional[str] = None workspace_slug: Optional[str] = None repo_slug: Optional[str] = None source_branch: Optional[str] = None target_branch: Optional[str] = None - # OAuth credentials (for Python-side RAG/VCS access) - oauth_key: Optional[str] = None - oauth_secret: Optional[str] = None - bearer_token: Optional[str] = None - @field_validator("task_context", mode="before") @classmethod def normalize_task_context_values(cls, value): @@ -103,7 +98,7 @@ async def generate_qa_documentation( 2. Determines if documentation is needed (LLM decides). 3. For large PRs: runs 3-stage ULTRATHINKING pipeline (batch → cross-impact → aggregate). 4. For small PRs: runs single-pass generation. - 5. Returns the document text ready to be posted as a task comment. + 5. Returns the document text for CodeCrow storage and test-case extraction. """ logger.info( "QA documentation request: project=%s, pr=#%s, mode=%s, diff_size=%d, enrichment=%s, delta=%s", diff --git a/python-ecosystem/inference-orchestrator/src/api/routers/review.py b/python-ecosystem/inference-orchestrator/src/api/routers/review.py index 194ec3e6..d6244f5c 100644 --- a/python-ecosystem/inference-orchestrator/src/api/routers/review.py +++ b/python-ecosystem/inference-orchestrator/src/api/routers/review.py @@ -53,41 +53,67 @@ async def review_endpoint(req: ReviewRequestDto, request: Request): # Streaming behavior async def event_stream(): queue = asyncio.Queue() - - # Emit initial queued status - yield _json_event({"type": "status", "state": "queued", "message": "request received"}) - - # Event callback to capture service events - def event_callback(event: Dict[str, Any]): - try: - queue.put_nowait(event) - except asyncio.QueueFull: - pass # Skip if queue is full - - # Run processing in background - async def runner(): - try: - result = await review_service.process_review_request( - req, - event_callback=event_callback - ) - # Emit final event with result - final_event = { - "type": "final", - "result": result.get("result") - } - await queue.put(final_event) - except Exception as e: - await queue.put({ - "type": "error", - "message": str(e) - }) - - task = asyncio.create_task(runner()) - - # Drain queue and yield events - async for event in _drain_queue_until_final(queue, task): - yield _json_event(event) + task = None + terminal_event_type = None + + try: + # Emit initial queued status + yield _json_event({"type": "status", "state": "queued", "message": "request received"}) + + # Event callback to capture service events + def event_callback(event: Dict[str, Any]): + nonlocal terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring streamed review event type=%s after " + "terminal type=%s", + event_type, + terminal_event_type, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type + try: + queue.put_nowait(event) + except asyncio.QueueFull: + pass # Skip if queue is full + + # Run processing in background + async def runner(): + try: + result = await review_service.process_review_request( + req, + event_callback=event_callback + ) + # Emit final event with result + final_event = { + "type": "final", + "result": result.get("result") + } + event_callback(final_event) + except Exception as e: + event_callback({ + "type": "error", + "message": str(e) + }) + + task = asyncio.create_task(runner()) + + # Drain queue and yield events + async for event in _drain_queue_until_final(queue, task): + yield _json_event(event) + finally: + # A disconnected streaming client no longer owns useful work. + # Retire its runner before application shutdown can close the + # shared RagClient underneath it. + if task is not None: + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass return StreamingResponse(event_stream(), media_type="application/x-ndjson") diff --git a/python-ecosystem/inference-orchestrator/src/server/command_queue_consumer.py b/python-ecosystem/inference-orchestrator/src/server/command_queue_consumer.py index fb488468..a7952110 100644 --- a/python-ecosystem/inference-orchestrator/src/server/command_queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/src/server/command_queue_consumer.py @@ -2,7 +2,6 @@ import json import logging import os -import traceback from typing import Dict, Any, Optional import redis.asyncio as redis from pydantic import ValidationError @@ -35,6 +34,22 @@ def __init__(self, command_service: CommandService): self.is_running = False self._redis: Optional[redis.Redis] = None self._task: Optional[asyncio.Task] = None + self._consumer_heartbeat_task: Optional[asyncio.Task] = None + self._job_tasks: set[asyncio.Task] = set() + self._redis_outage_channels: set[str] = set() + self.consumer_heartbeat_key = "codecrow:commands:consumer:heartbeat" + self.consumer_heartbeat_seconds = max( + 1.0, + float(os.environ.get("COMMAND_CONSUMER_HEARTBEAT_SECONDS", "5")), + ) + self.consumer_heartbeat_ttl_seconds = max( + 15, + int(self.consumer_heartbeat_seconds * 3), + ) + self.event_ttl_seconds = max( + 60, + int(os.environ.get("COMMAND_EVENT_TTL_SECONDS", "3600")), + ) max_concurrent = int(os.environ.get("MAX_CONCURRENT_COMMANDS", "10")) self._job_semaphore = asyncio.Semaphore(max_concurrent) @@ -52,6 +67,10 @@ async def start(self): health_check_interval=30, ) self.is_running = True + await self._publish_consumer_heartbeat() + self._consumer_heartbeat_task = asyncio.create_task( + self._consumer_heartbeat_loop() + ) self._task = asyncio.create_task(self._consume_loop()) async def stop(self): @@ -63,32 +82,94 @@ async def stop(self): await self._task except asyncio.CancelledError: pass + if self._consumer_heartbeat_task: + self._consumer_heartbeat_task.cancel() + try: + await self._consumer_heartbeat_task + except asyncio.CancelledError: + pass + + # A dequeued command is admitted work. Do not close Redis or the + # shared RagClient underneath an in-flight command. + active_jobs = tuple(self._job_tasks) + if active_jobs: + logger.info( + "Waiting for %s admitted command jobs before shutdown", + len(active_jobs), + ) + await asyncio.gather(*active_jobs, return_exceptions=True) if self._redis: await self._redis.aclose() logger.info("Command Queue Consumer stopped") + async def _publish_consumer_heartbeat(self): + if self._redis: + try: + await self._redis.set( + self.consumer_heartbeat_key, + "alive", + ex=self.consumer_heartbeat_ttl_seconds, + ) + self._record_redis_success("command consumer heartbeat") + except Exception as error: + self._record_redis_failure("command consumer heartbeat", error) + raise + + async def _consumer_heartbeat_loop(self): + while self.is_running: + try: + await self._publish_consumer_heartbeat() + except asyncio.CancelledError: + raise + except Exception: + pass + await asyncio.sleep(self.consumer_heartbeat_seconds) + async def _consume_loop(self): """Infinite loop blocking on the Redis queue for new jobs.""" logger.info(f"Listening for jobs on '{self.job_queue_key}'...") while self.is_running: + permit_acquired = False try: + # Reserve capacity before removing durable work from Redis. + await self._job_semaphore.acquire() + permit_acquired = True + if not self.is_running: + break + result = await self._redis.brpop([self.job_queue_key], timeout=1) + self._record_redis_success("command queue read") if not result: continue queue_name, payload_str = result logger.debug(f"Received raw command job payload from {queue_name}") - asyncio.create_task(self._bounded_handle_job(payload_str)) + job_task = asyncio.create_task( + self._handle_admitted_job(payload_str) + ) + self._job_tasks.add(job_task) + job_task.add_done_callback(self._job_tasks.discard) + permit_acquired = False except asyncio.CancelledError: break except RedisTimeoutError as error: - logger.warning("Redis command queue read timed out; retrying: %s", error) + self._record_redis_failure("command queue read", error) await asyncio.sleep(1) except Exception as e: - logger.error(f"Error in Command Queue consume loop: {e}", exc_info=True) + self._record_redis_failure("command queue read", e) await asyncio.sleep(2) + finally: + if permit_acquired: + self._job_semaphore.release() + + async def _handle_admitted_job(self, payload_str: str): + """Process a job using the capacity reserved before dequeue.""" + try: + await self._handle_job(payload_str) + finally: + self._job_semaphore.release() async def _bounded_handle_job(self, payload_str: str): """Acquire the concurrency semaphore before processing a job.""" @@ -100,6 +181,8 @@ async def _handle_job(self, payload_str: str): job_id = "UNKNOWN" event_queue_key = None command_type = "UNKNOWN" + publish_tail: Optional[asyncio.Future] = None + terminal_event_type: Optional[str] = None try: payload = json.loads(payload_str) @@ -113,9 +196,34 @@ async def _handle_job(self, payload_str: str): event_queue_key = f"codecrow:analysis:events:{job_id}" logger.info(f"Processing Command Job ID: {job_id} (Type: {command_type})") - + + # Serialize callback publications so progress cannot overtake the + # terminal result, and retain the tail until the job completes. + loop = asyncio.get_running_loop() + publish_tail = loop.create_future() + publish_tail.set_result(None) + def event_callback(event: Dict[str, Any]): - asyncio.create_task(self._publish_event(event_queue_key, event)) + nonlocal publish_tail, terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring command event type=%s after terminal type=%s " + "for job=%s", + event_type, + terminal_event_type, + job_id, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type + previous = publish_tail + + async def publish_after_previous(): + await previous + await self._publish_event(event_queue_key, event) + + publish_tail = asyncio.create_task(publish_after_previous()) event_callback({ "type": "status", @@ -135,11 +243,12 @@ def event_callback(event: Dict[str, Any]): if self._has_error(result): error_message = self._get_result_value(result, "error", "AI command failed") - await self._publish_event(event_queue_key, { + event_callback({ "type": "error", "message": str(error_message) }) - logger.warning(f"Command Job ID {job_id} failed: {error_message}") + await publish_tail + logger.info(f"Command Job ID {job_id} failed: {error_message}") return # Format output correctly depending on command type based on their DTO responses @@ -147,11 +256,12 @@ def event_callback(event: Dict[str, Any]): if command_type == "summarize": summary = self._get_result_value(result, "summary") if not self._has_usable_text(summary): - await self._publish_event(event_queue_key, { + event_callback({ "type": "error", "message": "AI service returned an empty summary" }) - logger.warning(f"Command Job ID {job_id} failed: empty summarize result") + await publish_tail + logger.info(f"Command Job ID {job_id} failed: empty summarize result") return final_payload = { @@ -162,11 +272,12 @@ def event_callback(event: Dict[str, Any]): elif command_type == "ask": answer = self._get_result_value(result, "answer") if not self._has_usable_text(answer): - await self._publish_event(event_queue_key, { + event_callback({ "type": "error", "message": "AI service returned an empty answer" }) - logger.warning(f"Command Job ID {job_id} failed: empty ask result") + await publish_tail + logger.info(f"Command Job ID {job_id} failed: empty ask result") return final_payload = { @@ -174,23 +285,33 @@ def event_callback(event: Dict[str, Any]): } event_callback({"type": "final", "result": final_payload}) - + await publish_tail logger.info(f"Command Job ID {job_id} processing completed successfully.") except ValidationError as ve: logger.error(f"Command Job ID {job_id} Validation Error: {ve}") if event_queue_key: - await self._publish_event(event_queue_key, { + event = { "type": "error", "message": f"Input validation error: {str(ve)}" - }) + } + if publish_tail is None: + await self._publish_event(event_queue_key, event) + else: + event_callback(event) + await publish_tail except Exception as e: logger.error(f"Command Job ID {job_id} Unhandled Error: {e}", exc_info=True) if event_queue_key: - await self._publish_event(event_queue_key, { + event = { "type": "error", "message": f"Internal orchestrator command error: {str(e)}" - }) + } + if publish_tail is None: + await self._publish_event(event_queue_key, event) + else: + event_callback(event) + await publish_tail async def _publish_event(self, key: str, event: Dict[str, Any]): """Publish an event back to the job's specific event list. LPUSH (Java uses rightPop).""" @@ -198,9 +319,42 @@ async def _publish_event(self, key: str, event: Dict[str, Any]): if not self._redis: return event_json = json.dumps(event) - await self._redis.lpush(key, event_json) + pipeline = self._redis.pipeline() + pipeline.lpush(key, event_json) + pipeline.expire(key, self.event_ttl_seconds) + await pipeline.execute() + self._record_redis_success("command event publication") except Exception as e: - logger.error(f"Failed to publish event to {key}: {e}") + self._record_redis_failure("command event publication", e) + + def _record_redis_failure(self, operation: str, error: Exception) -> None: + """Emit one actionable diagnostic per continuous Redis outage.""" + channel = self._redis_diagnostic_channel(operation) + if channel not in self._redis_outage_channels: + self._redis_outage_channels.add(channel) + logger.warning( + "Redis unavailable during %s; queue/event delivery is " + "degraded: %s", + operation, + error, + ) + return + logger.debug( + "Redis remains unavailable during %s: %s", + operation, + error, + ) + + def _record_redis_success(self, operation: str) -> None: + channel = self._redis_diagnostic_channel(operation) + if channel not in self._redis_outage_channels: + return + self._redis_outage_channels.discard(channel) + logger.info("Redis connectivity restored during %s", operation) + + @staticmethod + def _redis_diagnostic_channel(operation: str) -> str: + return "read" if operation.endswith("queue read") else "write" @staticmethod def _get_result_value(result: Any, key: str, default: Any = None) -> Any: diff --git a/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py b/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py index 63a4bb07..110ae9d7 100644 --- a/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py @@ -2,7 +2,6 @@ import json import logging import os -import traceback from typing import Dict, Any, Optional import redis.asyncio as redis from pydantic import ValidationError @@ -31,6 +30,8 @@ def __init__(self, review_service: ReviewService): self._redis: Optional[redis.Redis] = None self._task: Optional[asyncio.Task] = None self._consumer_heartbeat_task: Optional[asyncio.Task] = None + self._job_tasks: set[asyncio.Task] = set() + self._redis_outage_channels: set[str] = set() self.consumer_heartbeat_key = "codecrow:analysis:consumer:heartbeat" self.consumer_heartbeat_seconds = max( 1.0, @@ -86,6 +87,16 @@ async def stop(self): await self._consumer_heartbeat_task except asyncio.CancelledError: pass + + # Removing a job from Redis admits durable work. Keep its shared + # Redis/RAG clients alive until every admitted review has finished. + active_jobs = tuple(self._job_tasks) + if active_jobs: + logger.info( + "Waiting for %s admitted review jobs before shutdown", + len(active_jobs), + ) + await asyncio.gather(*active_jobs, return_exceptions=True) if self._redis: await self._redis.aclose() @@ -93,11 +104,19 @@ async def stop(self): async def _publish_consumer_heartbeat(self): if self._redis: - await self._redis.set( - self.consumer_heartbeat_key, - "alive", - ex=self.consumer_heartbeat_ttl_seconds, - ) + try: + await self._redis.set( + self.consumer_heartbeat_key, + "alive", + ex=self.consumer_heartbeat_ttl_seconds, + ) + self._record_redis_success("review consumer heartbeat") + except Exception as error: + self._record_redis_failure( + "review consumer heartbeat", + error, + ) + raise async def _consumer_heartbeat_loop(self): while self.is_running: @@ -106,7 +125,9 @@ async def _consumer_heartbeat_loop(self): except asyncio.CancelledError: raise except Exception: - logger.exception("Failed to publish review consumer heartbeat") + # The transition diagnostic is owned by + # _publish_consumer_heartbeat; keep the loop alive quietly. + pass await asyncio.sleep(self.consumer_heartbeat_seconds) async def is_healthy(self) -> bool: @@ -143,6 +164,7 @@ async def _consume_loop(self): # Block until a job is available or timeout (1 second for graceful shutdown check) result = await self._redis.brpop([self.job_queue_key], timeout=1) + self._record_redis_success("review queue read") if not result: continue @@ -151,16 +173,20 @@ async def _consume_loop(self): logger.debug(f"Received raw job payload from {queue_name}") # Transfer ownership of the reserved permit to the job task. - asyncio.create_task(self._handle_admitted_job(payload_str)) + job_task = asyncio.create_task( + self._handle_admitted_job(payload_str) + ) + self._job_tasks.add(job_task) + job_task.add_done_callback(self._job_tasks.discard) permit_acquired = False except asyncio.CancelledError: break except RedisTimeoutError as error: - logger.warning("Redis review queue read timed out; retrying: %s", error) + self._record_redis_failure("review queue read", error) await asyncio.sleep(1) except Exception as e: - logger.error(f"Error in Redis consume loop: {e}", exc_info=True) + self._record_redis_failure("review queue read", e) await asyncio.sleep(2) # Backoff on error finally: if permit_acquired: @@ -183,6 +209,7 @@ async def _handle_job(self, payload_str: str): job_id = "UNKNOWN" event_queue_key = None publish_tail: Optional[asyncio.Future] = None + terminal_event_type: Optional[str] = None try: payload = json.loads(payload_str) @@ -216,7 +243,19 @@ async def _handle_job(self, payload_str: str): publish_tail.set_result(None) def event_callback(event: Dict[str, Any]): - nonlocal publish_tail + nonlocal publish_tail, terminal_event_type + event_type = event.get("type") + if terminal_event_type is not None: + logger.debug( + "Ignoring review event type=%s after terminal type=%s " + "for job=%s", + event_type, + terminal_event_type, + job_id, + ) + return + if event_type in {"final", "error"}: + terminal_event_type = event_type previous = publish_tail async def publish_after_previous(): @@ -300,5 +339,37 @@ async def _publish_event(self, key: str, event: Dict[str, Any]): pipeline.lpush(key, event_str) pipeline.expire(key, 3600) await pipeline.execute() + self._record_redis_success("review event publication") except Exception as e: - logger.error(f"Failed to publish event to {key}: {e}") + self._record_redis_failure("review event publication", e) + + def _record_redis_failure(self, operation: str, error: Exception) -> None: + """Emit one actionable diagnostic per continuous Redis outage.""" + channel = self._redis_diagnostic_channel(operation) + if channel not in self._redis_outage_channels: + self._redis_outage_channels.add(channel) + logger.warning( + "Redis unavailable during %s; queue/event delivery is " + "degraded: %s", + operation, + error, + ) + return + logger.debug( + "Redis remains unavailable during %s: %s", + operation, + error, + ) + + def _record_redis_success(self, operation: str) -> None: + channel = self._redis_diagnostic_channel(operation) + if channel not in self._redis_outage_channels: + return + self._redis_outage_channels.discard(channel) + logger.info("Redis connectivity restored during %s", operation) + + @staticmethod + def _redis_diagnostic_channel(operation: str) -> str: + # A successful blocking read does not prove that Redis accepts event + # writes (for example during READONLY/OOM states). + return "read" if operation.endswith("queue read") else "write" diff --git a/python-ecosystem/inference-orchestrator/src/service/command/command_service.py b/python-ecosystem/inference-orchestrator/src/service/command/command_service.py index 037409ac..b69cf0f6 100644 --- a/python-ecosystem/inference-orchestrator/src/service/command/command_service.py +++ b/python-ecosystem/inference-orchestrator/src/service/command/command_service.py @@ -20,6 +20,14 @@ logger = logging.getLogger(__name__) +def _rag_response_error(response: Any) -> Optional[str]: + if not isinstance(response, dict) or response.get("status") != "error": + return None + detail = response.get("error") or response.get("detail") or "unknown failure" + status_code = response.get("status_code") + return f"status={status_code} detail={detail}" if status_code else str(detail) + + class CommandService: """Service class for handling CodeCrow commands with AI integration.""" @@ -122,6 +130,7 @@ async def process_summarize( result = self._normalize_summarize_result(result, supports_mermaid=False) if "error" in result: + logger.error("Summarize failed: %s", result["error"]) self._emit_event(event_callback, {"type": "error", "message": result["error"]}) return result @@ -237,6 +246,7 @@ async def process_ask( result = self._normalize_ask_result(result) if "error" in result: + logger.error("Ask failed: %s", result["error"]) self._emit_event(event_callback, {"type": "error", "message": result["error"]}) return result @@ -385,6 +395,19 @@ async def _fetch_rag_context_for_summarize( base_branch=request.get_rag_base_branch(), ) + if rag_error := _rag_response_error(rag_response): + logger.warning( + "Optional RAG context unavailable for summarize; " + "continuing without it: %s", + rag_error, + ) + self._emit_event(event_callback, { + "type": "status", + "state": "rag_skipped", + "message": "Codebase context unavailable; summary generation continues", + }) + return None + if rag_response and rag_response.get("context"): self._emit_event(event_callback, { "type": "status", @@ -739,7 +762,7 @@ async def _execute_summarize( if "error" not in result: return result - logger.warning("Summarize streaming produced an empty final summary; retrying without output_schema") + logger.info("Summarize streaming produced an empty final summary; retrying without output_schema") self._emit_event(event_callback, { "type": "status", "state": "retrying", @@ -756,7 +779,7 @@ async def _execute_summarize( if "error" not in result: return result - logger.warning("Summarize agent retry also produced an empty summary; trying direct LLM fallback") + logger.info("Summarize agent retry also produced an empty summary; trying direct LLM fallback") direct_response = await llm.ainvoke( prompt + "\n\nIf tool calls are unavailable, summarize from the context already provided. " @@ -765,7 +788,7 @@ async def _execute_summarize( return self._coerce_summarize_final_result(direct_response, supports_mermaid) except Exception as e: - logger.warning(f"Summarize streaming failed, retrying without output_schema: {e}", exc_info=True) + logger.info("Summarize streaming failed; retrying without output_schema: %s", e) self._emit_event(event_callback, { "type": "status", "state": "retrying", @@ -789,7 +812,7 @@ async def _execute_summarize( ) return self._coerce_summarize_final_result(direct_response, supports_mermaid) except Exception as retry_error: - logger.error(f"Summarize agent error: {retry_error}", exc_info=True) + logger.debug("Summarize retries exhausted: %s", retry_error, exc_info=True) sanitized_msg = create_user_friendly_error(retry_error) return {"error": sanitized_msg} @@ -966,7 +989,7 @@ async def _execute_ask( if "error" not in result: return result - logger.warning("Ask streaming produced an empty final answer; retrying without output_schema") + logger.info("Ask streaming produced an empty final answer; retrying without output_schema") self._emit_event(event_callback, { "type": "status", "state": "retrying", @@ -983,7 +1006,7 @@ async def _execute_ask( if "error" not in result: return result - logger.warning("Ask agent retry also produced an empty answer; trying direct LLM fallback") + logger.info("Ask agent retry also produced an empty answer; trying direct LLM fallback") direct_response = await llm.ainvoke( prompt + "\n\nIf tool calls are unavailable, answer from the context already provided. " @@ -992,7 +1015,7 @@ async def _execute_ask( return self._coerce_ask_final_result(direct_response) except Exception as e: - logger.error(f"Ask agent error: {e}", exc_info=True) + logger.debug("Ask agent retries exhausted: %s", e, exc_info=True) sanitized_msg = create_user_friendly_error(e) return {"error": sanitized_msg} @@ -1153,7 +1176,7 @@ def _parse_json_response(self, response: str) -> Optional[Dict[str, Any]]: except json.JSONDecodeError: pass - logger.warning(f"Failed to parse JSON from response: {response[:200]}...") + logger.debug("Failed to parse JSON from response: %s...", response[:200]) return None def _extract_json_object(self, text: str) -> Optional[str]: diff --git a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/base_orchestrator.py b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/base_orchestrator.py index d9ba4ee7..70e39df4 100644 --- a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/base_orchestrator.py +++ b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/base_orchestrator.py @@ -3,16 +3,15 @@ Provides shared infrastructure for multi-stage LLM pipelines: - LLM instance management -- RAG indexing / cleanup lifecycle - Smart dependency-aware batching via DependencyGraphBuilder - Diff filtering for per-batch file subsets - Event emission helpers -Subclasses: MultiStageReviewOrchestrator, QaDocOrchestrator +Subclass: QaDocOrchestrator """ import re import logging -from abc import ABC, abstractmethod +from abc import ABC from typing import Dict, Any, List, Optional, Callable, Set from model.enrichment import PrEnrichmentDataDto @@ -55,9 +54,7 @@ class BaseOrchestrator(ABC): Provides: - ``self.llm`` — the LangChain LLM instance - - ``self.rag_client`` — optional RAG client for hybrid queries - ``self.event_callback`` — optional SSE/WS event emitter - - ``index_pr_files()`` / ``cleanup_pr_files()`` — RAG lifecycle - ``build_dependency_batches()`` — smart batching from enrichment data - ``filter_diff_for_files()`` — per-batch diff slicing """ @@ -65,103 +62,10 @@ class BaseOrchestrator(ABC): def __init__( self, llm, - rag_client=None, event_callback: Optional[Callable[[Dict], None]] = None, ): self.llm = llm - self.rag_client = rag_client self.event_callback = event_callback - self._pr_number: Optional[int] = None - self._pr_indexed: bool = False - - # ── RAG lifecycle ──────────────────────────────────────────────── - - async def index_pr_files( - self, - *, - workspace: str, - project: str, - pr_number: int, - branch: str, - enrichment_data: Optional[PrEnrichmentDataDto], - changed_file_paths: Optional[List[str]], - diff: Optional[str] = None, - ) -> None: - """ - Index PR files into RAG for hybrid context queries. - - Uses full file content from enrichment data when available; - falls back to diff hunks otherwise. - """ - if not self.rag_client or not pr_number: - return - - # Build enrichment lookup: path → full file content - enrichment_lookup: Dict[str, str] = {} - if enrichment_data and enrichment_data.fileContents: - for fc in enrichment_data.fileContents: - if fc.content and not fc.skipped: - enrichment_lookup[fc.path] = fc.content - parts = fc.path.split("/", 1) - if len(parts) > 1: - enrichment_lookup[parts[1]] = fc.content - - # Build file list for indexing - files: List[Dict[str, str]] = [] - paths_to_index = changed_file_paths or [] - for path in paths_to_index: - content = enrichment_lookup.get(path, "") - if not content: - # Suffix matching for path variations - for ep, ec in enrichment_lookup.items(): - if path.endswith(ep) or ep.endswith(path): - content = ec - break - if content: - files.append({"path": path, "content": content, "change_type": "MODIFIED"}) - - if not files: - logger.info("No files to index for PR #%s", pr_number) - return - - self._pr_number = pr_number - try: - result = await self.rag_client.index_pr_files( - workspace=workspace, - project=project, - pr_number=pr_number, - branch=branch, - files=files, - ) - if result.get("status") in {"indexed", "reused"}: - self._pr_indexed = True - logger.info( - "%s PR #%s overlay: %s chunks", - "Reused" if result.get("status") == "reused" else "Indexed", - pr_number, - result.get("chunks_indexed", 0), - ) - else: - logger.warning("Failed to index PR files: %s", result) - except Exception as e: - logger.warning("Error indexing PR files: %s", e) - - async def cleanup_pr_files(self, workspace: str, project: str) -> None: - """Delete PR-indexed data (idempotent).""" - if not self._pr_number or not self.rag_client: - return - try: - await self.rag_client.delete_pr_files( - workspace=workspace, - project=project, - pr_number=self._pr_number, - ) - logger.info("Cleaned up PR #%s indexed data", self._pr_number) - except Exception as e: - logger.warning("Failed to cleanup PR files: %s", e) - finally: - self._pr_number = None - self._pr_indexed = False # ── Smart batching ─────────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_orchestrator.py b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_orchestrator.py index 1749f935..659f5adb 100644 --- a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_orchestrator.py +++ b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_orchestrator.py @@ -6,11 +6,12 @@ - Stage 2: Cross-file impact analysis (how changes interact for testing) - Stage 3: Aggregation into polished QA document (or delta update for re-runs) -Extends BaseOrchestrator for shared RAG, batching, and LLM infrastructure. +Extends BaseOrchestrator for shared batching and LLM infrastructure. """ import asyncio import json import logging +import re from typing import Dict, Any, List, Optional, Callable, Set from model.enrichment import PrEnrichmentDataDto @@ -35,6 +36,7 @@ QA_STAGE_3_AGGREGATION_PROMPT, QA_STAGE_3_DELTA_PROMPT, QA_STAGE_3_PREVIOUS_DOC_SECTION, + QA_DOC_TEST_CASES_REPAIR_PROMPT, ) logger = logging.getLogger(__name__) @@ -55,10 +57,9 @@ class QaDocOrchestrator(BaseOrchestrator): def __init__( self, llm, - rag_client=None, event_callback: Optional[Callable[[Dict], None]] = None, ): - super().__init__(llm, rag_client, event_callback) + super().__init__(llm, event_callback) async def run( self, @@ -130,10 +131,6 @@ async def run( changed_file_paths=changed_file_paths or [], previous_documentation=previous_documentation, is_same_pr_rerun=is_same_pr_rerun, - workspace_slug=workspace_slug, - repo_slug=repo_slug, - pr_number=pr_number, - source_branch=source_branch, ) else: logger.info( @@ -151,6 +148,12 @@ async def run( logger.warning("QA doc generation produced empty/short output") return {"documentation_needed": False, "documentation": None} + documentation = await self._ensure_test_cases(documentation, placeholders) + documentation = self._normalize_document_title( + documentation, + placeholders["pr_title"], + ) + # ── Footer with PR tracking ────────────────────────────────── documented_prs = self._extract_documented_prs(previous_documentation) if pr_number: @@ -180,25 +183,9 @@ async def _run_multi_stage( changed_file_paths: List[str], previous_documentation: Optional[str], is_same_pr_rerun: bool, - workspace_slug: Optional[str], - repo_slug: Optional[str], - pr_number: Optional[int], - source_branch: Optional[str], ) -> str: """Execute the 3-stage ULTRATHINKING pipeline.""" try: - # Index files into RAG if available - if self.rag_client and pr_number and workspace_slug and repo_slug: - await self.index_pr_files( - workspace=workspace_slug, - project=repo_slug, - pr_number=pr_number, - branch=source_branch or "unknown", - enrichment_data=enrichment_data, - changed_file_paths=changed_file_paths, - diff=diff, - ) - # For same-PR re-runs, analyze the delta diff — but only if # it contains actual hunks (@@). A delta that is truthy but # header-only (no @@) would starve the whole pipeline of context. @@ -290,9 +277,6 @@ async def _run_multi_stage( placeholders=placeholders, previous_documentation=previous_documentation, ) - finally: - if workspace_slug and repo_slug: - await self.cleanup_pr_files(workspace_slug, repo_slug) # ── Stage 1: Batch Analysis ────────────────────────────────────── @@ -715,6 +699,167 @@ async def _run_single_pass( return content + async def _ensure_test_cases( + self, + documentation: str, + placeholders: Dict[str, str], + ) -> str: + """Guarantee an independently extractable test-case section. + + Normal generation is instructed to emit stable invisible markers. If a + custom template or model response omits them, one focused repair call + generates only the missing section without narrowing the requested test + coverage. + """ + if self._contains_extractable_test_cases(documentation): + return self._normalize_test_case_markers(documentation) + + repair_placeholders = dict(placeholders) + raw_diff = repair_placeholders.get("diff", "") + max_repair_diff = 120_000 + if len(raw_diff) > max_repair_diff: + repair_placeholders["diff"] = ( + raw_diff[:max_repair_diff] + + f"\n\n... (diff truncated — {len(raw_diff)} chars total, " + f"showing first {max_repair_diff})" + ) + + logger.warning("QA doc omitted extractable test cases; running focused repair generation") + prompt = QA_DOC_TEST_CASES_REPAIR_PROMPT.format(**repair_placeholders) + response = await self.llm.ainvoke([ + {"role": "system", "content": QA_DOC_SYSTEM_PROMPT}, + {"role": "user", "content": prompt}, + ]) + generated = self._extract_text(response).strip() + if not generated: + raise ValueError("Test-case generation returned no content") + + start_marker = "" + end_marker = "" + start = generated.find(start_marker) + end = generated.find(end_marker, start + len(start_marker)) if start >= 0 else -1 + if start >= 0 and end >= 0: + test_case_section = generated[start:end + len(end_marker)] + else: + test_case_section = f"{start_marker}\n{generated}\n{end_marker}" + + if not self._contains_extractable_test_cases(test_case_section): + raise ValueError("Test-case generation returned no structured scenarios") + + return self._normalize_test_case_markers( + documentation.rstrip() + "\n\n" + test_case_section.strip() + ) + + @staticmethod + def _normalize_test_case_markers(documentation: str) -> str: + """Keep later peer sections outside the test-case disclosure boundary. + + A model can occasionally place the closing marker at the end of the + document despite the prompt. Move it before the next heading at the + Test Scenarios heading level so sections such as Edge Cases, + Regression Risks, and Environment Notes remain independent. + """ + start_marker = "" + end_marker = "" + start = documentation.find(start_marker) + if start < 0: + return documentation + content_start = start + len(start_marker) + end = documentation.find(end_marker, content_start) + if end < 0: + return documentation + + marked_content = documentation[content_start:end] + test_heading = re.search( + r"(?mi)^\s*(#{2,6})\s+(?:\d+\.\s*)?" + r"Test Scenarios(?:\s+by Area)?\s*$", + marked_content, + ) + if test_heading is None: + return documentation + + heading_level = len(test_heading.group(1)) + following = marked_content[test_heading.end():] + peer_heading = None + for candidate in re.finditer( + r"(?m)^\s*(#{2,6})\s+(.+?)\s*$", + following, + ): + candidate_level = len(candidate.group(1)) + candidate_title = candidate.group(2).strip() + numbered_section = re.match(r"^\d+\.\s+.+$", candidate_title) is not None + known_later_section = re.match( + r"(?i)^(?:Edge Cases(?: and Negative Testing)?|Negative Testing|" + r"Regression Risks|Environment(?: and Setup Notes)?|Setup Notes).*$", + candidate_title, + ) is not None + if ( + candidate_level < heading_level + or ( + candidate_level == heading_level + and (numbered_section or known_later_section) + ) + ): + peer_heading = test_heading.end() + candidate.start() + break + if peer_heading is None: + return documentation + + test_content = marked_content[:peer_heading].strip() + later_sections = marked_content[peer_heading:].strip() + after_marker = documentation[end + len(end_marker):].strip() + + normalized = ( + documentation[:content_start].rstrip() + + "\n" + + test_content + + "\n" + + end_marker + + "\n\n" + + later_sections + ) + if after_marker: + normalized += "\n\n" + after_marker + return normalized.strip() + + @staticmethod + def _contains_extractable_test_cases(documentation: Optional[str]) -> bool: + if not documentation: + return False + start_marker = "" + end_marker = "" + start = documentation.find(start_marker) + if start < 0: + return False + content_start = start + len(start_marker) + end = documentation.find(end_marker, content_start) + if end < 0: + return False + marked_content = documentation[content_start:end] + return re.search( + r"(?mi)^\s*\*\*.+?\*\*\s*\((?:HIGH|MEDIUM|LOW)\)", + marked_content, + ) is not None + + @staticmethod + def _normalize_document_title(documentation: str, fallback_title: str) -> str: + """Replace a leaked empty-title sentinel in the rendered guide heading.""" + return re.sub( + r"(?mi)^(#\s+QA Testing Guide\s*[—–-]\s*)(?:N\s*/?\s*A|None|null)\s*$", + lambda match: f"{match.group(1)}{fallback_title}", + documentation, + count=1, + ) + + @staticmethod + def _display_value(value: Any) -> Optional[str]: + if value is None: + return None + normalized = str(value).strip() + if not normalized or normalized.casefold() in {"n/a", "na", "none", "null"}: + return None + return normalized + # ================================================================== # Shared helpers # ================================================================== @@ -763,14 +908,25 @@ def _build_placeholders( """Build the placeholder dictionary used for prompt formatting.""" task_ctx = task_context_dict or {} effective_language = output_language if output_language and output_language.strip() else "English" + normalized_project_name = self._display_value(project_name) + task_key = self._display_value(task_ctx.get("task_key")) + task_summary = self._display_value(task_ctx.get("task_summary")) + pr_title = ( + self._display_value(pr_metadata.get("prTitle")) + or task_summary + or task_key + or (f"PR #{pr_number}" if pr_number is not None else None) + or normalized_project_name + or "QA documentation" + ) return { - "project_name": project_name or "Unknown", + "project_name": normalized_project_name or "Unknown", "pr_number": str(pr_number) if pr_number else "N/A", - "task_key": task_ctx.get("task_key", "N/A"), - "task_summary": task_ctx.get("task_summary", "N/A"), + "task_key": task_key or "N/A", + "task_summary": task_summary or "N/A", "source_branch": source_branch, "target_branch": target_branch, - "pr_title": pr_metadata.get("prTitle", "N/A"), + "pr_title": pr_title, "pr_description": self._truncate(pr_metadata.get("prDescription", ""), 500), "issues_found": str(issues_found), "files_analyzed": str(files_analyzed), diff --git a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_service.py b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_service.py index beb9e838..51c406bc 100644 --- a/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_service.py +++ b/python-ecosystem/inference-orchestrator/src/service/qa_documentation/qa_doc_service.py @@ -19,7 +19,6 @@ from llm.llm_factory import LLMFactory from model.enrichment import PrEnrichmentDataDto from service.qa_documentation.qa_doc_orchestrator import QaDocOrchestrator -from service.rag.rag_client import RagClient logger = logging.getLogger(__name__) @@ -28,8 +27,8 @@ class QaDocumentationService: """ Generates QA-oriented documentation for completed PR analyses. - Creates the LLM and RAG client, then delegates to QaDocOrchestrator - which runs the 3-stage pipeline or single-pass depending on PR size. + Creates the LLM, then delegates to QaDocOrchestrator which runs the + 3-stage pipeline or single-pass depending on PR size. """ def __init__(self): @@ -37,7 +36,6 @@ def __init__(self): self._ai_provider = os.environ.get("QA_DOC_AI_PROVIDER", os.environ.get("AI_PROVIDER", "openrouter")) self._ai_model = os.environ.get("QA_DOC_AI_MODEL", os.environ.get("AI_MODEL", "google/gemini-2.0-flash")) self._ai_api_key = os.environ.get("QA_DOC_AI_API_KEY", os.environ.get("AI_API_KEY", "")) - self._rag_pipeline_url = os.environ.get("RAG_PIPELINE_URL", "http://rag-pipeline:8020") # ------------------------------------------------------------------ # Public API @@ -88,12 +86,7 @@ async def generate( max_tokens=16_384, # QA docs need room for structured JSON output ) - rag_client = self._create_rag_client() - - orchestrator = QaDocOrchestrator( - llm=llm, - rag_client=rag_client, - ) + orchestrator = QaDocOrchestrator(llm=llm) result = await orchestrator.run( project_name=project_name, @@ -143,12 +136,3 @@ def _create_llm(self, ai_provider=None, ai_model=None, ai_api_key=None, ai_base_ ai_base_url=ai_base_url, max_tokens=max_tokens, ) - - def _create_rag_client(self) -> Optional[RagClient]: - """Create RAG client if the RAG pipeline URL is configured.""" - try: - if self._rag_pipeline_url: - return RagClient(base_url=self._rag_pipeline_url) - except Exception as e: - logger.warning("Failed to create RAG client (non-critical): %s", e) - return None diff --git a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py index 49895c8c..901968b1 100644 --- a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py +++ b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py @@ -83,6 +83,7 @@ def __init__(self, base_url: Optional[str] = None, enabled: Optional[bool] = Non os.environ.get("SERVICE_SECRET") or os.environ.get("CODECROW_RAG_API_SECRET", "") ) + self._cleanup_degraded = False if self.enabled: logger.info(f"RAG client initialized: {self.base_url}") @@ -108,6 +109,18 @@ async def close(self): await self._client.aclose() self._client = None + def _record_cleanup_failure(self, detail: str) -> None: + if not self._cleanup_degraded: + logger.warning("RAG PR cleanup is degraded: %s", detail) + self._cleanup_degraded = True + else: + logger.debug("RAG PR cleanup remains degraded: %s", detail) + + def _record_cleanup_success(self) -> None: + if self._cleanup_degraded: + logger.info("RAG PR cleanup recovered") + self._cleanup_degraded = False + async def get_pr_context( self, workspace: str, @@ -232,7 +245,7 @@ async def get_pr_context( except httpx.HTTPError as e: status_code, detail = _http_error_detail(e) - logger.warning( + logger.debug( "Failed to retrieve PR context from RAG: status=%s detail=%s", status_code or "transport-error", detail, @@ -243,7 +256,7 @@ async def get_pr_context( "error": detail, } except Exception as e: - logger.error(f"Unexpected error querying RAG: {e}") + logger.debug("Unexpected error querying RAG: %s", e, exc_info=True) return { "status": "error", "status_code": None, @@ -308,7 +321,7 @@ async def semantic_search( except httpx.HTTPError as e: status_code, detail = _http_error_detail(e) - logger.warning( + logger.debug( "Semantic search failed: status=%s detail=%s", status_code or "transport-error", detail, @@ -320,7 +333,7 @@ async def semantic_search( "results": [], } except Exception as e: - logger.error(f"Unexpected error in semantic search: {e}") + logger.debug("Unexpected error in semantic search: %s", e, exc_info=True) return { "status": "error", "status_code": None, @@ -497,7 +510,7 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: "revision-bound duplication search failed: " f"{type(e).__name__}: {e}" ) from e - logger.warning(f"Failed duplication search: {e}") + logger.debug("Failed duplication search: %s", e, exc_info=True) return [] async def get_deterministic_context( @@ -600,7 +613,7 @@ async def get_deterministic_context( except httpx.HTTPError as e: status_code, detail = _http_error_detail(e) - logger.warning( + logger.debug( "Failed to retrieve deterministic context: status=%s detail=%s", status_code or "transport-error", detail, @@ -611,7 +624,11 @@ async def get_deterministic_context( "error": detail, } except Exception as e: - logger.error(f"Unexpected error in deterministic RAG query: {e}") + logger.debug( + "Unexpected error in deterministic RAG query: %s", + e, + exc_info=True, + ) return { "status": "error", "status_code": None, @@ -719,7 +736,7 @@ async def index_pr_files( except httpx.HTTPError as e: status_code, detail = _http_error_detail(e) - logger.warning( + logger.debug( "Failed to index PR files: status=%s detail=%s timeout=%.1fs", status_code or "transport-error", detail, @@ -731,7 +748,11 @@ async def index_pr_files( "error": detail, } except Exception as e: - logger.error(f"Unexpected error indexing PR files: {e}") + logger.debug( + "Unexpected error indexing PR files: %s", + e, + exc_info=True, + ) return {"status": "error", "error": str(e)} async def delete_pr_files( @@ -768,13 +789,35 @@ async def delete_pr_files( ) response.raise_for_status() result = response.json() - - logger.info(f"Deleted PR #{pr_number} indexed data") - return result.get("status") == "deleted" + + status = result.get("status") + if status == "deleted": + self._record_cleanup_success() + logger.info("Deleted PR #%s indexed data", pr_number) + return True + if status == "skipped": + self._record_cleanup_success() + logger.info( + "PR #%s indexed-data cleanup was already complete: %s", + pr_number, + result.get("message") or "nothing to delete", + ) + return True + self._record_cleanup_failure( + "PR #%s returned unexpected status %s" + % (pr_number, status or "missing") + ) + return False except httpx.HTTPError as e: - logger.warning(f"Failed to delete PR files: {e}") + status_code, detail = _http_error_detail(e) + self._record_cleanup_failure( + "status=%s detail=%s" + % (status_code or "transport-error", detail) + ) return False except Exception as e: - logger.error(f"Unexpected error deleting PR files: {e}") + self._record_cleanup_failure( + f"unexpected {type(e).__name__}: {e}" + ) return False diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py index 888736fb..eb2a2e73 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py @@ -61,7 +61,6 @@ execute_stage_3_aggregation, _emit_status, _emit_progress, - _emit_error, ) from service.review.plugin_context import ( apply_effective_project_capabilities, @@ -168,6 +167,23 @@ def _resolve_enrichment_content( INTERNAL_PR_INDEX_ENABLED = _env_bool("REVIEW_INTERNAL_PR_INDEX_ENABLED", True) VERIFICATION_ENABLED = _env_bool("REVIEW_VERIFICATION_ENABLED", True) +_REQUEST_RAG_BINDING_FIELDS = ( + "ragCollectionTarget", + "ragBaseGenerationManifestSha256", + "ragPrGenerationFingerprint", + "ragPrOverlayGenerationManifestSha256", + "ragBasePluginFingerprint", + "ragBasePluginDescriptorFingerprint", + "ragBasePluginImplementationFingerprint", + "ragBaseIndexRepresentationFingerprint", +) + + +def _clear_request_rag_bindings(request: ReviewRequestDto) -> None: + """Remove host-provided RAG bindings when the project disables RAG.""" + for field_name in _REQUEST_RAG_BINDING_FIELDS: + setattr(request, field_name, None) + def _review_log_id(request: ReviewRequestDto) -> str: return ( @@ -318,7 +334,11 @@ async def _index_pr_files( as a complete repository artifact. """ self._repository_review_groups = () + self._pr_indexed = False + request.ragPrGenerationFingerprint = None + request.ragPrOverlayGenerationManifestSha256 = None if not request.ragEnabled: + _clear_request_rag_bindings(request) logger.info("PR file indexing skipped because project RAG is disabled") return if not INTERNAL_PR_INDEX_ENABLED: @@ -453,16 +473,21 @@ async def _index_pr_files( request, result.get("effective_project_capabilities"), ) - request.ragBaseGenerationManifestSha256 = ( + base_generation_manifest = ( result.get("base_generation_manifest_sha256") or request.ragBaseGenerationManifestSha256 ) - request.ragPrGenerationFingerprint = result.get( + pr_generation_fingerprint = result.get( "generation_fingerprint" ) - request.ragPrOverlayGenerationManifestSha256 = result.get( + overlay_generation_manifest = result.get( "overlay_generation_manifest_sha256" ) + request.ragBaseGenerationManifestSha256 = ( + base_generation_manifest + ) + request.ragPrGenerationFingerprint = None + request.ragPrOverlayGenerationManifestSha256 = None request.ragBasePluginFingerprint = ( result.get("plugin_fingerprint") or request.ragBasePluginFingerprint @@ -479,7 +504,6 @@ async def _index_pr_files( result.get("index_representation_fingerprint") or request.ragBaseIndexRepresentationFingerprint ) - self._pr_indexed = True self._repository_review_groups = tuple( tuple( path for path in group @@ -488,21 +512,51 @@ async def _index_pr_files( for group in (result.get("review_groups") or ()) if isinstance(group, (list, tuple)) ) - logger.info( - "%s PR #%s overlay: %s chunks, %s partial files, " - "%s repository review groups", - "Reused" if result.get("status") == "reused" else "Indexed", - pr_number, - result.get("chunks_indexed", 0), - len(result.get("partial_files") or ()), - len(self._repository_review_groups), + complete_overlay_binding = all( + isinstance(value, str) and bool(value.strip()) + for value in ( + identity.head_revision, + identity.base_revision, + request.ragCollectionTarget, + base_generation_manifest, + pr_generation_fingerprint, + overlay_generation_manifest, + ) ) + if complete_overlay_binding: + request.ragPrGenerationFingerprint = ( + pr_generation_fingerprint + ) + request.ragPrOverlayGenerationManifestSha256 = ( + overlay_generation_manifest + ) + self._pr_indexed = True + logger.info( + "%s PR #%s overlay: %s chunks, %s partial files, " + "%s repository review groups", + "Reused" if result.get("status") == "reused" else "Indexed", + pr_number, + result.get("chunks_indexed", 0), + len(result.get("partial_files") or ()), + len(self._repository_review_groups), + ) + else: + self._pr_indexed = False + self._repository_review_groups = () + logger.info( + "PR #%s overlay was prepared without a complete generation " + "lease; continuing with target-branch and local evidence", + pr_number, + ) elif result.get("status") == "skipped": logger.info("PR indexing skipped: %s", result) else: status_code = result.get("status_code") detail = result.get("error") or result - logger.warning( + log_unavailable = ( + logger.info if status_code == 409 else logger.warning + ) + log_unavailable( "PR context indexing unavailable%s; continuing review without " "the PR overlay: %s", f" (HTTP {status_code})" if status_code else "", @@ -529,13 +583,20 @@ async def _cleanup_pr_files(self, request: ReviewRequestDto) -> None: return try: - await self.rag_client.delete_pr_files( + deleted = await self.rag_client.delete_pr_files( workspace=request.projectWorkspace, project=request.projectNamespace, pr_number=self._pr_number, collection_target=request.ragCollectionTarget, ) - logger.info(f"Cleaned up PR #{self._pr_number} indexed data") + if deleted: + logger.info("Cleaned up PR #%s indexed data", self._pr_number) + else: + logger.info( + "PR #%s indexed-data cleanup did not complete; the RAG " + "client recorded the failure detail", + self._pr_number, + ) except Exception as e: logger.warning(f"Failed to cleanup PR files: {e}") finally: @@ -889,6 +950,11 @@ async def orchestrate_review( Main entry point for the multi-stage review. Supports both FULL (initial review) and INCREMENTAL (follow-up review) modes. """ + request_rag_client = self.rag_client if request.ragEnabled else None + request_rag_context = rag_context if request.ragEnabled else None + if not request.ragEnabled: + _clear_request_rag_bindings(request) + snapshot_identity = validate_review_snapshot_identity(request) validate_acquired_diff_manifest( request.changedFiles or (), @@ -1052,10 +1118,10 @@ async def orchestrate_review( ) _emit_progress(self.event_callback, 10, stage_0_message) - if not inference_profile.fast_check_enabled and self.rag_client: + if not inference_profile.fast_check_enabled and request_rag_client: stage_2_context_task = asyncio.create_task( prefetch_stage_2_cross_module_context( - self.rag_client, + request_rag_client, request, processed_diff=processed_diff, visible_evidence_by_id=stage_2_visible_evidence_by_id, @@ -1072,8 +1138,8 @@ async def orchestrate_review( with_stage_output_cap(self.llm, "stage_1", inference_profile), request, review_plan, - self.rag_client, - rag_context, + request_rag_client, + request_rag_context, processed_diff, is_incremental, self.max_parallel_stage_1, @@ -1169,7 +1235,7 @@ async def orchestrate_review( file_issues, review_plan, processed_diff=processed_diff, - rag_client=self.rag_client, + rag_client=request_rag_client, fallback_llm=self.llm, prefetched_cross_module_context=prefetched_cross_module_context, visible_evidence_by_id=stage_2_visible_evidence_by_id, @@ -1489,8 +1555,14 @@ async def orchestrate_review( return response except Exception as e: - logger.error(f"Multi-stage review failed: {e}", exc_info=True) - _emit_error(self.event_callback, str(e)) + # ReviewService owns the single terminal diagnostic and error + # event. Logging/emitting here as well duplicated the same failure + # at both the orchestration and transport boundaries. + logger.debug( + "Multi-stage review failed; propagating to ReviewService: %s", + e, + exc_info=True, + ) raise finally: if stage_2_context_task and not stage_2_context_task.done(): diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_0_planning.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_0_planning.py index b2ee7f1f..a43581e6 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_0_planning.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_0_planning.py @@ -141,7 +141,7 @@ async def execute_stage_0_planning( logger.info("Stage 0 planning completed with structured output") return result except Exception as e: - logger.warning(f"Structured output failed for Stage 0: {e}") + logger.debug("Structured output failed for Stage 0: %s", e) else: logger.info("Structured output skipped for Stage 0; using prompt JSON parsing") @@ -150,7 +150,7 @@ async def execute_stage_0_planning( content = extract_llm_response_text(response) return await parse_llm_response(content, ReviewPlan, llm) except Exception as e: - logger.error(f"Stage 0 planning failed, using local fallback plan: {e}") + logger.info("Stage 0 planning unavailable; using local fallback plan: %s", e) return _build_fallback_review_plan(request, processed_diff) diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py index f293e565..ee7f9623 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py @@ -129,6 +129,8 @@ class Stage1PreparedContext: @dataclass class Stage1RagState: """Per-review RAG state shared across Stage 1 batches.""" + context_disabled: bool = False + context_disable_reason: str = "" semantic_disabled: bool = False semantic_failures: int = 0 semantic_disable_reason: str = "" @@ -1562,16 +1564,72 @@ def _is_exact_revision_bound( request: ReviewRequestDto, pr_indexed: bool, ) -> bool: + pr_number = getattr(request, "pullRequestId", None) return bool( pr_indexed - and (request.currentCommitHash or request.commitHash) - and request.baseCommitHash - and request.ragBaseGenerationManifestSha256 - and request.ragPrGenerationFingerprint - and request.ragPrOverlayGenerationManifestSha256 + and isinstance(pr_number, int) + and pr_number > 0 + and all( + isinstance(value, str) and bool(value.strip()) + for value in ( + getattr(request, "currentCommitHash", None) + or getattr(request, "commitHash", None), + getattr(request, "baseCommitHash", None), + getattr(request, "ragCollectionTarget", None), + getattr(request, "ragBaseGenerationManifestSha256", None), + getattr(request, "ragPrGenerationFingerprint", None), + getattr( + request, + "ragPrOverlayGenerationManifestSha256", + None, + ), + ) + ) + ) + + +def _has_exact_base_binding(request: ReviewRequestDto) -> bool: + return all( + isinstance(value, str) and bool(value.strip()) + for value in ( + getattr(request, "baseCommitHash", None), + getattr(request, "ragCollectionTarget", None), + getattr(request, "ragBaseGenerationManifestSha256", None), + ) ) +def _disable_rag_context( + rag_state: Optional[Stage1RagState], + reason: str, +) -> bool: + """Open the optional-context circuit once and report whether it changed.""" + if rag_state is None: + return True + if rag_state.context_disabled: + return False + rag_state.context_disabled = True + rag_state.context_disable_reason = reason + rag_state.semantic_disabled = True + rag_state.semantic_disable_reason = reason + return True + + +def _disable_semantic_rag( + rag_state: Optional[Stage1RagState], + reason: str, +) -> bool: + """Open the semantic-filler circuit once across concurrent batches.""" + if rag_state is None: + return True + if rag_state.semantic_disabled: + return False + rag_state.semantic_failures += 1 + rag_state.semantic_disabled = True + rag_state.semantic_disable_reason = reason + return True + + async def fetch_batch_rag_context( rag_client, request: ReviewRequestDto, @@ -1586,10 +1644,23 @@ async def fetch_batch_rag_context( rag_state: Optional[Stage1RagState] = None, ) -> Optional[Dict[str, Any]]: exact_revision_bound = _is_exact_revision_bound(request, pr_indexed) + exact_base_bound = _has_exact_base_binding(request) + exact_context_bound = exact_revision_bound or exact_base_bound + if rag_state and rag_state.context_disabled: + logger.debug( + "Per-batch RAG context skipped after an earlier optional-context " + "failure: %s", + rag_state.context_disable_reason, + ) + return None if not rag_client: - if exact_revision_bound: - raise RuntimeError( - "revision-bound Stage 1 retrieval requires a RAG client" + if exact_context_bound and _disable_rag_context( + rag_state, + "revision-bound Stage 1 retrieval has no RAG client", + ): + logger.info( + "Optional revision-bound RAG context is unavailable; " + "continuing with local review evidence" ) return None @@ -1600,13 +1671,16 @@ async def fetch_batch_rag_context( base_branch = request.get_rag_base_branch() if not rag_branch: message = "Missing authoritative target branch for Stage 1 RAG retrieval" - logger.warning(message) _capture_deterministic_retrieval_state( {"status": "error", "error": message}, rag_state, ) - if exact_revision_bound: - raise RuntimeError(message) + if _disable_rag_context(rag_state, message): + (logger.info if exact_context_bound else logger.warning)( + "%s; disabling optional RAG context for the remaining " + "Stage 1 batches", + message, + ) return None # Scale top_k based on batch priority to ensure adequate context @@ -1616,29 +1690,34 @@ async def fetch_batch_rag_context( logger.info(f"Fetching per-batch RAG context for {len(batch_file_paths)} files " f"(priority={priority_upper}, top_k={top_k})") - pr_number = request.pullRequestId if pr_indexed else None - all_pr_files = request.changedFiles if pr_indexed else None + pr_number = request.pullRequestId if exact_revision_bound else None + all_pr_files = request.changedFiles if exact_revision_bound else None source_revision = ( request.currentCommitHash or request.commitHash - if pr_indexed + if exact_revision_bound else None ) - base_revision = request.baseCommitHash if pr_indexed else None + base_revision = ( + request.baseCommitHash if exact_context_bound else None + ) base_generation_receipt = ( request.ragBaseGenerationManifestSha256 - if pr_indexed + if exact_context_bound else None ) pr_generation_fingerprint = ( request.ragPrGenerationFingerprint - if pr_indexed + if exact_revision_bound else None ) pr_overlay_generation_manifest_sha256 = ( request.ragPrOverlayGenerationManifestSha256 - if pr_indexed + if exact_revision_bound else None ) + collection_target = ( + request.ragCollectionTarget if exact_context_bound else None + ) context = None @@ -1647,11 +1726,15 @@ async def _fetch_deterministic_context() -> Optional[Dict[str, Any]]: return await rag_client.get_deterministic_context( workspace=request.projectWorkspace, project=request.projectNamespace, - branches=list(dict.fromkeys( - branch - for branch in (rag_branch, base_branch) - if branch - )), + branches=( + [rag_branch] + if exact_context_bound + else list(dict.fromkeys( + branch + for branch in (rag_branch, base_branch) + if branch + )) + ), file_paths=batch_file_paths, limit_per_file=5, pr_number=pr_number, @@ -1666,11 +1749,13 @@ async def _fetch_deterministic_context() -> Optional[Dict[str, Any]]: pr_overlay_generation_manifest_sha256=( pr_overlay_generation_manifest_sha256 ), - collection_target=request.ragCollectionTarget, + collection_target=collection_target, ) except Exception as det_err: - logger.warning("Deterministic RAG lookup failed: %s", det_err) - return {"status": "error", "error": str(det_err)} + return { + "status": "error", + "error": f"{type(det_err).__name__}: {det_err}", + } async def _fetch_semantic_context() -> Optional[Dict[str, Any]]: if not SEMANTIC_RAG_FILLER_ENABLED: @@ -1691,10 +1776,10 @@ async def _fetch_semantic_context() -> Optional[Dict[str, Any]]: pr_title=request.prTitle, pr_description=request.prDescription, top_k=semantic_top_k, - base_branch=base_branch, + base_branch=(base_branch if pr_number else None), pr_number=pr_number, all_pr_changed_files=all_pr_files, - deleted_files=request.deletedFiles or None, + deleted_files=(request.deletedFiles or None) if pr_number else None, source_revision=source_revision, base_revision=base_revision, base_generation_manifest_sha256=base_generation_receipt, @@ -1702,7 +1787,7 @@ async def _fetch_semantic_context() -> Optional[Dict[str, Any]]: pr_overlay_generation_manifest_sha256=( pr_overlay_generation_manifest_sha256 ), - collection_target=request.ragCollectionTarget, + collection_target=collection_target, ) async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: @@ -1743,12 +1828,10 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: repository_generation_manifest_sha256=( base_generation_receipt ), - collection_target=request.ragCollectionTarget, + collection_target=collection_target, ) except Exception as dup_err: - if base_revision or base_generation_receipt: - raise - logger.debug(f"Duplication search skipped: {dup_err}") + logger.debug("Duplication search skipped: %s", dup_err) return None deterministic_task = asyncio.create_task(_fetch_deterministic_context()) @@ -1757,32 +1840,37 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: # 1. Deterministic lookup FIRST — structural deps are highest-value context deterministic_response = await deterministic_task deterministic_error = _rag_response_error(deterministic_response) - if deterministic_error: - logger.warning( - "Deterministic RAG lookup returned a failed state: %s", - deterministic_error, - ) deterministic_chunks = _flatten_deterministic_context(deterministic_response) _capture_deterministic_retrieval_state( deterministic_response, rag_state, ) - if deterministic_error and exact_revision_bound: - raise RuntimeError( - "revision-bound deterministic RAG retrieval failed: " - f"{deterministic_error}" - ) deterministic_retrieval_state = _deterministic_retrieval_state( deterministic_response ) - if ( - exact_revision_bound - and deterministic_retrieval_state != "complete" - ): - raise RuntimeError( - "revision-bound deterministic RAG retrieval is not complete: " - f"{deterministic_retrieval_state}" + context_error = ( + deterministic_error + or ( + f"deterministic retrieval state is {deterministic_retrieval_state}" + if exact_context_bound + and deterministic_retrieval_state != "complete" + else None ) + ) + if context_error: + if duplication_task is not None and not duplication_task.done(): + duplication_task.cancel() + if duplication_task is not None: + await asyncio.gather(duplication_task, return_exceptions=True) + if _disable_rag_context(rag_state, context_error): + (logger.info if exact_context_bound else logger.warning)( + "Optional %sRAG context is unavailable; disabling it for " + "the remaining Stage 1 batches and continuing with local " + "review evidence: %s", + "revision-bound " if exact_context_bound else "", + context_error, + ) + return None if deterministic_chunks: context = {"relevant_code": deterministic_chunks} logger.info( @@ -1805,11 +1893,6 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: ) elif semantic_fill_enabled and rag_state and rag_state.semantic_disabled: logger.info("Semantic RAG filler skipped: %s", rag_state.semantic_disable_reason) - if exact_revision_bound: - raise RuntimeError( - "revision-bound semantic RAG retrieval is disabled after " - f"a prior failure: {rag_state.semantic_disable_reason}" - ) elif semantic_fill_enabled: try: rag_response = await asyncio.wait_for( @@ -1821,38 +1904,21 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: raise RuntimeError(semantic_error) except asyncio.TimeoutError: rag_response = None - if rag_state: - rag_state.semantic_failures += 1 - rag_state.semantic_disabled = True - rag_state.semantic_disable_reason = ( - f"timed out after {SEMANTIC_RAG_TIMEOUT_SECONDS}s" - ) - logger.warning( - "Semantic RAG filler timed out after %ss; disabling for remaining Stage 1 batches", - SEMANTIC_RAG_TIMEOUT_SECONDS, - ) - if exact_revision_bound: - raise RuntimeError( - "revision-bound semantic RAG retrieval timed out" + reason = f"timed out after {SEMANTIC_RAG_TIMEOUT_SECONDS}s" + if _disable_semantic_rag(rag_state, reason): + logger.warning( + "Semantic RAG filler timed out after %ss; disabling " + "for remaining Stage 1 batches", + SEMANTIC_RAG_TIMEOUT_SECONDS, ) except Exception as sem_err: rag_response = None - if rag_state: - rag_state.semantic_failures += 1 - rag_state.semantic_disabled = True - rag_state.semantic_disable_reason = str(sem_err) - logger.warning("Semantic RAG filler failed; disabling for remaining Stage 1 batches: %s", sem_err) - if exact_revision_bound: - raise - - if ( - semantic_fill_enabled - and exact_revision_bound - and rag_response is None - ): - raise RuntimeError( - "revision-bound semantic RAG retrieval returned no response" - ) + if _disable_semantic_rag(rag_state, str(sem_err)): + logger.warning( + "Semantic RAG filler failed; disabling for remaining " + "Stage 1 batches: %s", + sem_err, + ) if semantic_fill > 0 and rag_response: sem_context = _unwrap_rag_context(rag_response) @@ -1945,7 +2011,7 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: f"{rerank_result.original_count}→{rerank_result.reranked_count} chunks)" ) except Exception as rerank_err: - logger.warning(f"Per-batch reranking failed (non-critical): {rerank_err}") + logger.info(f"Per-batch reranking skipped (non-critical): {rerank_err}") return context @@ -1956,9 +2022,12 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: duplication_task.cancel() if duplication_task is not None: await asyncio.gather(duplication_task, return_exceptions=True) - logger.warning(f"Failed to fetch per-batch RAG context: {e}") - if exact_revision_bound: - raise + if _disable_rag_context(rag_state, str(e)): + logger.warning( + "Failed to fetch optional per-batch RAG context; disabling it " + "for the remaining Stage 1 batches: %s", + e, + ) return None @@ -2204,7 +2273,7 @@ async def _run_batch( else: logger.info(f"Batch {batch_num} completed: no issues found") except Exception as exc: - logger.error(f"Error reviewing Stage 1 batch: {exc}") + logger.debug("Stage 1 batch failed; cancelling sibling batches: %s", exc) for task in tasks: if not task.done(): task.cancel() @@ -2268,7 +2337,7 @@ async def _review_batch_with_timing( return result except Exception as e: elapsed = time.time() - start_time - logger.error(f"[Batch {batch_idx}] FAILED after {elapsed:.2f}s: {e}") + logger.debug(f"[Batch {batch_idx}] FAILED after {elapsed:.2f}s: {e}") raise @@ -2401,7 +2470,11 @@ async def review_file_batch( str, tuple[Dict[str, Any], ...] ] = {} - if rag_client or _is_exact_revision_bound(request, pr_indexed): + exact_context_bound = ( + _is_exact_revision_bound(request, pr_indexed) + or _has_exact_base_binding(request) + ) + if rag_client or exact_context_bound: batch_rag_context = await fetch_batch_rag_context( rag_client, request, batch_file_paths, batch_diff_snippets, pr_indexed, llm_reranker=llm_reranker, @@ -2423,15 +2496,25 @@ async def review_file_batch( visible_evidence_by_id=batch_visible_evidence_by_id, ) else: - resolved_fallback_rag_context = await _resolve_fallback_rag_context( - fallback_rag_context + fallback_context_allowed = not ( + exact_context_bound + or (rag_state is not None and rag_state.context_disabled) ) - if resolved_fallback_rag_context: - scoped_fallback_rag_context = _scope_fallback_rag_context_to_batch( - resolved_fallback_rag_context, - batch_file_paths, + if fallback_context_allowed: + resolved_fallback_rag_context = await _resolve_fallback_rag_context( + fallback_rag_context ) + if resolved_fallback_rag_context: + scoped_fallback_rag_context = _scope_fallback_rag_context_to_batch( + resolved_fallback_rag_context, + batch_file_paths, + ) + else: + scoped_fallback_rag_context = None else: + # Global fallback is alias/branch scoped and cannot satisfy an + # immutable generation receipt. It must not be awaited after an + # exact-context failure, because doing so could inject stale code. scoped_fallback_rag_context = None if scoped_fallback_rag_context: @@ -2545,7 +2628,7 @@ async def review_file_batch( return issues if fallback_llm is not None and fallback_llm is not llm: - logger.warning( + logger.info( "Stage 1 batch failed with capped LLM for %s; retrying without output cap", batch_file_paths, ) @@ -2565,7 +2648,7 @@ async def review_file_batch( ) return issues - logger.error( + logger.debug( "Batch review parse failure for %s after capped%s attempts. " "The batch will fail so missing results cannot be published as a clean review.", batch_file_paths, @@ -2689,9 +2772,9 @@ async def _invoke_stage_1_batch_llm( result = await structured_llm.ainvoke(prompt) if result: return _extract_calibrated_issues(result) - logger.warning("Structured output returned empty Stage 1 result for %s (%s)", batch_file_paths, label) + logger.debug("Structured output returned empty Stage 1 result for %s (%s)", batch_file_paths, label) except Exception as e: - logger.warning("Structured output failed for Stage 1 batch %s (%s): %s", batch_file_paths, label, e) + logger.debug("Structured output failed for Stage 1 batch %s (%s): %s", batch_file_paths, label, e) else: logger.info( "Structured output skipped for Stage 1 batch %s (%s); using prompt JSON parsing", @@ -2705,7 +2788,7 @@ async def _invoke_stage_1_batch_llm( data = await parse_llm_response(content, FileReviewBatchOutput, llm) return _extract_calibrated_issues(data) except Exception as parse_err: - logger.warning("Stage 1 batch parse failed for %s (%s): %s", batch_file_paths, label, parse_err) + logger.debug("Stage 1 batch parse failed for %s (%s): %s", batch_file_paths, label, parse_err) return None diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py index 880cb3e7..31e34ce2 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py @@ -136,7 +136,7 @@ async def execute_stage_2_cross_file( return result if fallback_llm is not None and fallback_llm is not llm: - logger.warning("Stage 2 failed with capped LLM; retrying without output cap") + logger.info("Stage 2 failed with capped LLM; retrying without output cap") result = await _invoke_stage_2_llm(fallback_llm, prompt, label="uncapped retry") if result is not None: return result @@ -168,9 +168,9 @@ async def _invoke_stage_2_llm(llm, prompt: str, label: str) -> Optional[CrossFil if result: logger.info("Stage 2 cross-file analysis completed with structured output (%s)", label) return result - logger.warning("Structured output returned empty Stage 2 result (%s)", label) + logger.debug("Structured output returned empty Stage 2 result (%s)", label) except Exception as e: - logger.warning("Structured output failed for Stage 2 (%s): %s", label, e) + logger.debug("Structured output failed for Stage 2 (%s): %s", label, e) else: logger.info("Structured output skipped for Stage 2 (%s); using prompt JSON parsing", label) @@ -179,7 +179,7 @@ async def _invoke_stage_2_llm(llm, prompt: str, label: str) -> Optional[CrossFil content = extract_llm_response_text(response) return await parse_llm_response(content, CrossFileAnalysisResult, llm) except Exception as e: - logger.warning("Stage 2 cross-file analysis failed (%s): %s", label, e) + logger.debug("Stage 2 cross-file analysis failed (%s): %s", label, e) return None @@ -711,7 +711,7 @@ async def _fetch_cross_module_context( ) return "" if not base_revision or not base_generation_receipt: - logger.warning( + logger.info( "Stage 2 cross-module RAG requires both immutable target revision " "and generation receipt" ) @@ -721,7 +721,7 @@ async def _fetch_cross_module_context( rag_branch = request.get_rag_branch() base_branch = request.get_rag_base_branch() if not rag_branch: - logger.warning( + logger.info( "Stage 2 cross-module RAG skipped: missing authoritative target branch" ) return "" @@ -788,7 +788,7 @@ async def _fetch_cross_module_context( return formatted except Exception as e: - logger.warning( + logger.info( "Revision-bound cross-module context unavailable for Stage 2: %s: %s", type(e).__name__, e, diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py index cdb5576b..b4a685d7 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py @@ -106,7 +106,7 @@ def _review_revision(request: ReviewRequestDto) -> str: async def _invoke_stage_3_report(llm, prompt: str, fallback_llm=None) -> Dict[str, Any]: response = await llm.ainvoke(prompt) if _response_finished_by_length(response) and fallback_llm is not None and fallback_llm is not llm: - logger.warning("Stage 3 report hit output cap; retrying without output cap") + logger.info("Stage 3 report hit output cap; retrying without output cap") response = await fallback_llm.ainvoke(prompt) return {"report": extract_llm_response_text(response), "dismissed_issue_ids": []} @@ -449,7 +449,7 @@ async def _stage_3_with_mcp( tool_calls = getattr(response, 'tool_calls', None) if not tool_calls: if _response_finished_by_length(response) and fallback_llm is not None and fallback_llm is not llm: - logger.warning("MCP Stage 3 report hit output cap; retrying without output cap") + logger.info("MCP Stage 3 report hit output cap; retrying without output cap") return await _stage_3_with_mcp( fallback_llm, request, @@ -492,8 +492,8 @@ async def _stage_3_with_mcp( }) except Exception as e: - logger.warning(f"[MCP Stage 3] Iteration {iteration + 1} failed: {e}") + logger.info(f"[MCP Stage 3] Iteration {iteration + 1} failed: {e}") break - logger.warning("[MCP Stage 3] Agentic loop exhausted, falling back to plain call") + logger.info("[MCP Stage 3] Agentic loop exhausted, falling back to plain call") return await _invoke_stage_3_report(llm, prompt, fallback_llm=fallback_llm) diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/verification_agent.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/verification_agent.py index 7593e5cb..69ee559a 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/verification_agent.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/verification_agent.py @@ -1383,7 +1383,8 @@ async def run_verification_agent( result = await _run_verification_tool_loop(llm, prompt) except Exception as exception: failed_batches += 1 - logger.error( + log = logger.info if failed_batches == 1 else logger.debug + log( "Stage 1.5 verification batch %d/%d failed; retaining its " "%d issue(s): %s", batch_index, @@ -1438,7 +1439,7 @@ async def run_verification_agent( ] except Exception as e: - logger.error(f"Stage 1.5 Verification failed: {e}") + logger.info("Stage 1.5 verification unavailable; retaining all issues: %s", e) # Fallback: keep all issues if verification fails final_issues = issues finally: diff --git a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py index 6e384fe6..f7af1dbc 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py @@ -655,6 +655,28 @@ async def _fetch_rag_context( timeout=self.GLOBAL_RAG_QUERY_TIMEOUT_SECONDS, ) + if ( + isinstance(rag_response, dict) + and rag_response.get("status") == "error" + ): + logger.info( + "Global fallback RAG context unavailable; per-batch " + "retrieval remains authoritative: status=%s detail=%s", + rag_response.get("status_code") or "transport-error", + rag_response.get("error") + or rag_response.get("detail") + or "unknown failure", + ) + self._emit_event(event_callback, { + "type": "status", + "state": "rag_skipped", + "message": ( + "Global RAG fallback unavailable; per-batch retrieval " + "remains active" + ), + }) + return None + if rag_response and rag_response.get("context"): context = rag_response.get("context") relevant_code = context.get("relevant_code", []) diff --git a/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_qa_doc.py b/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_qa_doc.py index b7e259f9..c4b870dc 100644 --- a/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_qa_doc.py +++ b/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_qa_doc.py @@ -123,7 +123,20 @@ - NEVER include SQL queries, API endpoints, CLI commands, database operations, or configuration references. - Write everything from the USER's perspective — what screens, buttons, and behaviors to test. - Every scenario must have numbered steps a manual tester can follow. -- Translate all technical changes into user-visible behaviors. If a change is purely internal, say "Verify existing functionality still works" instead of describing the code.""" +- Translate all technical changes into user-visible behaviors. If a change is purely internal, say "Verify existing functionality still works" instead of describing the code. + +The output MUST include test cases even when another template or structure is requested. +Wrap the complete test-case section in these exact invisible markers: + +### Test Scenarios +**Scenario Name** (HIGH) +- **Preconditions:** Required setup +- **Steps:** + 1. Tester action +- **Expected Result:** Observable result + +Use exactly HIGH, MEDIUM, or LOW as each scenario's priority. +Do not place non-test-case content between those markers.""" QA_DOC_BASE_PROMPT = """Generate structured QA documentation for the following PR changes. @@ -170,16 +183,19 @@ - What the user will notice is different - Expected behavior after the change + ### 3. Test Scenarios For each functional area, list test scenarios using this format: -**Scenario Name** (PRIORITY) +**Scenario Name** (HIGH) - **Preconditions:** What needs to be set up before testing - **Steps:** 1. Go to [screen/page] 2. Click [button/link] 3. ... - **Expected Result:** What the tester should see/verify + +Use exactly HIGH, MEDIUM, or LOW as each scenario's priority. ### 4. Edge Cases and Negative Testing Bullet list of boundary conditions and error scenarios to verify, written as user actions. @@ -223,13 +239,27 @@ - Use only: headings (#), bold (**), bullet lists (-), numbered lists (1.) ## Custom Template Instructions -Follow this user-provided template as closely as possible: +Follow this user-provided template as closely as possible for the overview content: --- {custom_template} --- -Generate the QA documentation now, following the custom template above.""" +The custom template MUST NOT remove test cases. After its content, append a complete +test-case section in the exact format below, even when the custom template does not +request scenarios or asks for a different document structure: + + +### Test Scenarios +**Scenario Name** (HIGH) +- **Preconditions:** Required setup +- **Steps:** + 1. Tester action +- **Expected Result:** Observable result + +Use exactly HIGH, MEDIUM, or LOW as each scenario's priority. + +Do not place overview content between the markers. Generate the QA documentation now.""" # --------------------------------------------------------------------------- @@ -424,7 +454,7 @@ - Horizontal rules: --- 5. For test scenarios, use this exact bullet-list format (NOT a table): -**Scenario Name** (PRIORITY) +**Scenario Name** (HIGH) - **Preconditions:** What setup is needed - **Steps:** 1. Navigate to [page/screen] @@ -432,6 +462,8 @@ 3. Verify [result] - **Expected Result:** What the tester should observe +Use exactly HIGH, MEDIUM, or LOW as each scenario's priority. + 6. If a change is purely internal (refactoring, infrastructure, CI/CD), write: "Verify [feature] still works correctly" — do NOT describe the code change itself. ## Instructions @@ -458,11 +490,13 @@ (Skip this section entirely if no acceptance criteria were provided.) + ## 3. Test Scenarios by Area Group scenarios under the functional area heading (### Area Name). List each scenario using the bullet-list format shown above. Order: HIGH priority first, then MEDIUM, then LOW. + ## 4. Edge Cases and Negative Testing Bullet list of unusual conditions to verify: @@ -473,7 +507,7 @@ Areas of the application that were NOT changed but might be affected: - [Feature/screen] — why it might be impacted, how to verify -## 6. Setup and Environment Notes +## 6. Environment and Setup Notes Any special requirements: - Test data needed - Configuration or feature flags to enable @@ -527,12 +561,47 @@ 5. **Marking** new/updated sections with *(updated in latest push)* so testers see what's new 6. **Preserving** all scenarios from the previous guide that are still valid 7. **Stripping any technical references** that may have leaked — replace with user-facing descriptions +8. **Preserving the exact `` and `` markers** around all test scenarios. If the previous guide has no markers, add them. The result must be a COMPLETE, standalone QA testing guide — not just the changes. Generate the updated document now. Remember: NO TABLES, NO FILE NAMES, NO CODE, NO SQL, NO API ENDPOINTS, NO CLI COMMANDS.""" +QA_DOC_TEST_CASES_REPAIR_PROMPT = """Generate ONLY the missing manual test-case section for this PR. +Write all user-facing text in **{output_language}**. Keep the existing breadth and +detail of the QA guidance; do not summarize or reduce scenarios because the change is large. + +PR #{pr_number} in {project_name}: {pr_title} +Task: {task_key} — {task_summary} +Branch: {source_branch} → {target_branch} + +{task_context} + +Analysis summary: +{analysis_summary} + +PR diff: +``` +{diff} +``` + +Return only this structure, with one or more concrete scenarios: + +### Test Scenarios +**Scenario Name** (HIGH) +- **Preconditions:** Required setup +- **Steps:** + 1. Tester action +- **Expected Result:** Observable result + + +Use exactly HIGH, MEDIUM, or LOW as each scenario's priority. + +Never include file names, code symbols, SQL, endpoints, commands, configuration +files, code snippets, or implementation-stack details.""" + + # --------------------------------------------------------------------------- # Update preamble — injected when previous QA documentation already exists # for the same task from earlier PRs. The LLM merges old + new into one doc. diff --git a/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py b/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py index 82352e3f..83b4c312 100644 --- a/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py +++ b/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py @@ -115,6 +115,8 @@ def neutral_request( pullRequestId=42, currentCommitHash=HEAD_REVISION, baseCommitHash=BASE_REVISION, + ragCollectionTarget="cc_workspace_project_main_generation", + ragBaseGenerationManifestSha256=BASE_GENERATION_MANIFEST, changedFiles=paths, rawDiff="\n".join(diffs) + "\n", enrichmentData=PrEnrichmentDataDto(fileContents=contents), diff --git a/python-ecosystem/inference-orchestrator/tests/test_command_queue_consumer.py b/python-ecosystem/inference-orchestrator/tests/test_command_queue_consumer.py index c0c151df..d58079a3 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_command_queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/tests/test_command_queue_consumer.py @@ -8,12 +8,34 @@ from server.command_queue_consumer import CommandQueueConsumer +class FakePipeline: + def __init__(self, redis): + self.redis = redis + self.pending = [] + + def lpush(self, key, value): + self.pending.append(("lpush", key, value)) + return self + + def expire(self, key, ttl): + self.pending.append(("expire", key, ttl)) + return self + + async def execute(self): + for operation, key, value in self.pending: + if operation == "lpush": + self.redis.events.append((key, json.loads(value))) + else: + self.redis.expiries.append((key, value)) + + class FakeRedis: def __init__(self): self.events = [] + self.expiries = [] - async def lpush(self, key, value): - self.events.append((key, json.loads(value))) + def pipeline(self): + return FakePipeline(self) def _ask_request(): @@ -61,20 +83,32 @@ def _consumer(command_service): async def _handle_and_collect_events(consumer, payload): await consumer._handle_job(payload) - await asyncio.sleep(0) - await asyncio.sleep(0) return [event for _, event in consumer._redis.events] @pytest.mark.asyncio(loop_scope="function") async def test_error_result_is_published_as_error_without_final(): command_service = MagicMock() - command_service.process_ask = AsyncMock(return_value={"error": "provider failed"}) + + async def process(_request, callback): + callback({"type": "error", "message": "provider failed"}) + callback({"type": "status", "state": "late_diagnostic"}) + return {"error": "provider failed"} + + command_service.process_ask = AsyncMock(side_effect=process) consumer = _consumer(command_service) events = await _handle_and_collect_events(consumer, _payload("ask", _ask_request())) - assert any(event["type"] == "error" and event["message"] == "provider failed" for event in events) + assert events == [ + { + "type": "status", + "state": "acknowledged", + "message": "Orchestrator picked up ask command from queue", + }, + {"type": "error", "message": "provider failed"}, + ] + assert sum(event["type"] in {"error", "final"} for event in events) == 1 assert not any(event["type"] == "final" for event in events) @@ -105,6 +139,52 @@ async def test_successful_ask_answer_is_published_as_final(): assert not any(event["type"] == "error" for event in events) +@pytest.mark.asyncio(loop_scope="function") +async def test_progress_and_final_events_are_ordered_before_job_returns(): + command_service = MagicMock() + + async def process(_request, callback): + callback({"type": "status", "state": "answering"}) + return {"answer": "42"} + + command_service.process_ask = AsyncMock(side_effect=process) + consumer = _consumer(command_service) + + events = await _handle_and_collect_events( + consumer, + _payload("ask", _ask_request()), + ) + + assert [event.get("state") for event in events[:-1]] == [ + "acknowledged", + "answering", + ] + assert events[-1] == {"type": "final", "result": {"answer": "42"}} + assert consumer._redis.expiries == [ + ("codecrow:analysis:events:job-ask", 3600), + ("codecrow:analysis:events:job-ask", 3600), + ("codecrow:analysis:events:job-ask", 3600), + ] + + +@pytest.mark.asyncio(loop_scope="function") +async def test_command_event_publish_uses_configured_expiry(monkeypatch): + monkeypatch.setenv("COMMAND_EVENT_TTL_SECONDS", "123") + consumer = _consumer(MagicMock()) + + await consumer._publish_event( + "codecrow:analysis:events:job-ttl", + {"type": "status"}, + ) + + assert consumer._redis.events == [ + ("codecrow:analysis:events:job-ttl", {"type": "status"}), + ] + assert consumer._redis.expiries == [ + ("codecrow:analysis:events:job-ttl", 123), + ] + + @pytest.mark.asyncio(loop_scope="function") async def test_empty_summarize_result_is_published_as_error_without_final(): command_service = MagicMock() @@ -125,6 +205,7 @@ async def test_start_uses_blocking_read_safe_redis_timeouts(): command_service = MagicMock() redis_client = MagicMock() redis_client.aclose = AsyncMock() + redis_client.set = AsyncMock() consumer = CommandQueueConsumer(command_service) consumer._consume_loop = AsyncMock() @@ -142,6 +223,26 @@ async def test_start_uses_blocking_read_safe_redis_timeouts(): "socket_timeout": 30, "health_check_interval": 30, } + redis_client.set.assert_awaited_with( + "codecrow:commands:consumer:heartbeat", + "alive", + ex=consumer.consumer_heartbeat_ttl_seconds, + ) + + +@pytest.mark.asyncio(loop_scope="function") +async def test_command_consumer_heartbeat_uses_the_java_supervision_contract(): + consumer = CommandQueueConsumer(MagicMock()) + consumer._redis = MagicMock() + consumer._redis.set = AsyncMock() + + await consumer._publish_consumer_heartbeat() + + consumer._redis.set.assert_awaited_once_with( + "codecrow:commands:consumer:heartbeat", + "alive", + ex=consumer.consumer_heartbeat_ttl_seconds, + ) @pytest.mark.asyncio(loop_scope="function") @@ -166,3 +267,85 @@ async def stop_after_backoff(_seconds): await consumer._consume_loop() warning.assert_called_once() + + +@pytest.mark.asyncio(loop_scope="function") +async def test_worker_capacity_is_reserved_before_command_is_dequeued(): + consumer = CommandQueueConsumer(MagicMock()) + consumer._job_semaphore = asyncio.Semaphore(1) + consumer._redis = MagicMock() + + async def stop_after_dequeue(*_args, **_kwargs): + consumer.is_running = False + return None + + consumer._redis.brpop = AsyncMock(side_effect=stop_after_dequeue) + consumer.is_running = True + await consumer._job_semaphore.acquire() + + consume_task = asyncio.create_task(consumer._consume_loop()) + await asyncio.sleep(0) + await asyncio.sleep(0) + consumer._redis.brpop.assert_not_awaited() + + consumer._job_semaphore.release() + await consume_task + + consumer._redis.brpop.assert_awaited_once_with( + [consumer.job_queue_key], + timeout=1, + ) + assert not consumer._job_semaphore.locked() + + +@pytest.mark.asyncio(loop_scope="function") +async def test_stop_waits_for_admitted_command_before_closing_redis(): + consumer = CommandQueueConsumer(MagicMock()) + consumer._redis = MagicMock() + consumer._redis.aclose = AsyncMock() + started = asyncio.Event() + release = asyncio.Event() + + async def handle_until_released(_payload): + started.set() + await release.wait() + + consumer._handle_job = AsyncMock(side_effect=handle_until_released) + await consumer._job_semaphore.acquire() + job_task = asyncio.create_task(consumer._handle_admitted_job("payload")) + consumer._job_tasks.add(job_task) + job_task.add_done_callback(consumer._job_tasks.discard) + await started.wait() + + stop_task = asyncio.create_task(consumer.stop()) + await asyncio.sleep(0) + + assert not stop_task.done() + consumer._redis.aclose.assert_not_awaited() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + + assert job_task.done() + consumer._redis.aclose.assert_awaited_once_with() + + +def test_redis_outage_diagnostic_is_bounded_until_recovery(): + consumer = CommandQueueConsumer(MagicMock()) + + with ( + patch("server.command_queue_consumer.logger.warning") as warning, + patch("server.command_queue_consumer.logger.debug") as debug, + patch("server.command_queue_consumer.logger.info") as info, + ): + consumer._record_redis_failure("command event publication", RuntimeError("down")) + consumer._record_redis_failure("command event publication", RuntimeError("still down")) + consumer._record_redis_success("command queue read") + consumer._record_redis_success("command event publication") + + warning.assert_called_once() + debug.assert_called_once() + info.assert_called_once_with( + "Redis connectivity restored during %s", + "command event publication", + ) diff --git a/python-ecosystem/inference-orchestrator/tests/test_command_service.py b/python-ecosystem/inference-orchestrator/tests/test_command_service.py index aa8ed4d5..f28aa2b8 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_command_service.py +++ b/python-ecosystem/inference-orchestrator/tests/test_command_service.py @@ -7,6 +7,7 @@ """ import pytest import json +import logging from unittest.mock import AsyncMock, MagicMock, patch from service.command.command_service import CommandService @@ -77,6 +78,42 @@ def test_returns_dict(self, service): assert isinstance(result, dict) +@pytest.mark.asyncio(loop_scope="function") +async def test_summarize_rag_failure_has_one_owner_diagnostic_and_event( + service, + caplog, +): + service.rag_client.get_pr_context = AsyncMock(return_value={ + "status": "error", + "status_code": 503, + "error": "RAG service unavailable", + }) + request = MagicMock( + projectWorkspace="ws", + projectNamespace="project", + pullRequestId=42, + ) + request.get_rag_branch.return_value = "main" + request.get_rag_base_branch.return_value = None + events = [] + + with caplog.at_level( + logging.WARNING, + logger="service.command.command_service", + ): + result = await service._fetch_rag_context_for_summarize( + request, + events.append, + ) + + assert result is None + assert sum( + "Optional RAG context unavailable for summarize" in record.getMessage() + for record in caplog.records + ) == 1 + assert any(event.get("state") == "rag_skipped" for event in events) + + # ── _build_platform_jvm_props ──────────────────────────────────── class TestBuildPlatformJvmProps: diff --git a/python-ecosystem/inference-orchestrator/tests/test_llm_factory.py b/python-ecosystem/inference-orchestrator/tests/test_llm_factory.py index d12c15e4..19618748 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_llm_factory.py +++ b/python-ecosystem/inference-orchestrator/tests/test_llm_factory.py @@ -3,12 +3,12 @@ Covers: LLMFactory._normalize_provider, get_supported_providers, _check_unsupported_gemini_model, create_llm (all providers), - QaDocumentationService._create_llm, _create_rag_client + QaDocumentationService._create_llm and QA orchestration wiring """ import asyncio import pytest -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from langchain_google_genai import ChatGoogleGenerativeAI @@ -488,19 +488,35 @@ def test_create_llm(self): "QA_DOC_AI_API_KEY": "test", "RAG_PIPELINE_URL": "http://rag:8020", }) - def test_create_rag_client(self): + def test_does_not_configure_rag_mutation_client(self): svc = QaDocumentationService() - client = svc._create_rag_client() - # Should not be None when URL is configured - assert client is not None - - @patch.dict("os.environ", { - "QA_DOC_AI_PROVIDER": "openai", - "QA_DOC_AI_MODEL": "gpt-4o", - "QA_DOC_AI_API_KEY": "test", - "RAG_PIPELINE_URL": "", - }) - def test_create_rag_client_no_url(self): + assert not hasattr(svc, "_rag_pipeline_url") + assert not hasattr(svc, "_create_rag_client") + + @pytest.mark.asyncio(loop_scope="function") + @patch("service.qa_documentation.qa_doc_service.QaDocOrchestrator") + async def test_generate_never_wires_a_rag_mutation_client( + self, + orchestrator_type, + ): svc = QaDocumentationService() - client = svc._create_rag_client() - assert client is None + llm = MagicMock() + svc._create_llm = MagicMock(return_value=llm) + orchestrator_type.return_value.run = AsyncMock(return_value={ + "documentation_needed": False, + "documentation": None, + }) + + await svc.generate( + project_id=1, + project_name="project", + pr_number=17, + issues_found=0, + files_analyzed=1, + pr_metadata={}, + template_mode="BASE", + custom_template=None, + task_context=None, + ) + + orchestrator_type.assert_called_once_with(llm=llm) diff --git a/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py index 5e0bf4e7..353af6a0 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py @@ -5,8 +5,10 @@ _deduplicate_previous_issues, _ensure_all_files_planned, _count_files, _convert_cross_file_issues """ +import logging +from unittest.mock import AsyncMock, MagicMock + import pytest -from unittest.mock import MagicMock from service.review.orchestrator.orchestrator import ( MultiStageReviewOrchestrator, @@ -65,6 +67,43 @@ def test_task_evidence_key_falls_back_to_server_built_history(): assert _task_evidence_key(request) == "SHOP-42" +@pytest.mark.asyncio +async def test_pr_cleanup_does_not_report_success_when_delete_returns_false( + caplog, +): + rag_client = MagicMock() + rag_client.delete_pr_files = AsyncMock(return_value=False) + orchestrator = MultiStageReviewOrchestrator( + llm=MagicMock(), + mcp_client=None, + rag_client=rag_client, + ) + orchestrator._pr_number = 42 + orchestrator._pr_indexed = True + request = MagicMock( + projectWorkspace="workspace", + projectNamespace="project", + ragCollectionTarget="cc_workspace_project_main_generation", + ) + + with caplog.at_level( + logging.INFO, + logger="service.review.orchestrator.orchestrator", + ): + await orchestrator._cleanup_pr_files(request) + + rag_client.delete_pr_files.assert_awaited_once_with( + workspace="workspace", + project="project", + pr_number=42, + collection_target="cc_workspace_project_main_generation", + ) + assert "Cleaned up PR #42 indexed data" not in caplog.text + assert "PR #42 indexed-data cleanup did not complete" in caplog.text + assert orchestrator._pr_number is None + assert orchestrator._pr_indexed is False + + # ── _filter_diff_for_files ────────────────────────────────────── class TestFilterDiffForFiles: diff --git a/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py b/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py index b834b212..42c91f30 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py +++ b/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py @@ -992,6 +992,69 @@ async def test_project_disabled_rag_skips_pr_overlay(): assert rag.index_requests == [] +@pytest.mark.asyncio +async def test_project_disabled_rag_clears_bindings_and_never_queries_client(): + request = _request().model_copy(update={ + "ragEnabled": False, + "ragPrGenerationFingerprint": "sha256:" + "a" * 64, + "ragPrOverlayGenerationManifestSha256": "b" * 64, + "ragBasePluginFingerprint": "sha256:" + "c" * 64, + "ragBasePluginDescriptorFingerprint": "sha256:" + "d" * 64, + "ragBasePluginImplementationFingerprint": "sha256:" + "e" * 64, + "ragBaseIndexRepresentationFingerprint": "sha256:" + "f" * 64, + # Force Stage 2 even for the small neutral fixture. + "taskContext": {"task_key": "SHOP-42"}, + }) + rag = DeterministicRagSpy() + rag.index_pr_files = AsyncMock( + side_effect=AssertionError("disabled RAG must not index") + ) + rag.get_deterministic_context = AsyncMock( + side_effect=AssertionError("disabled RAG must not retrieve") + ) + rag.get_pr_context = AsyncMock( + side_effect=AssertionError("disabled RAG must not retrieve") + ) + rag.search_for_duplicates = AsyncMock( + side_effect=AssertionError("disabled RAG must not query duplicates") + ) + session = PromptCaptureSession(request=request) + orchestrator = MultiStageReviewOrchestrator( + llm=PromptCaptureLLM(session), + mcp_client=None, + rag_client=rag, + ) + + result = await orchestrator.orchestrate_review( + request, + rag_context={"relevant_code": ["DISABLED_GLOBAL_RAG_SENTINEL"]}, + processed_diff=DiffProcessor().process(request.rawDiff), + ) + + assert result["issues"] == [] + rag.index_pr_files.assert_not_awaited() + rag.get_deterministic_context.assert_not_awaited() + rag.get_pr_context.assert_not_awaited() + rag.search_for_duplicates.assert_not_awaited() + assert all( + getattr(request, field_name) is None + for field_name in ( + "ragCollectionTarget", + "ragBaseGenerationManifestSha256", + "ragPrGenerationFingerprint", + "ragPrOverlayGenerationManifestSha256", + "ragBasePluginFingerprint", + "ragBasePluginDescriptorFingerprint", + "ragBasePluginImplementationFingerprint", + "ragBaseIndexRepresentationFingerprint", + ) + ) + assert all( + "DISABLED_GLOBAL_RAG_SENTINEL" not in prompt["renderedPrompt"] + for prompt in session.prompts + ) + + @pytest.mark.asyncio async def test_pr_overlay_receives_one_exact_snapshot_identity(): rag = DeterministicRagSpy() @@ -1044,6 +1107,42 @@ async def reject_overlay(**kwargs): assert rag.index_requests assert orchestrator._pr_indexed is False + assert request.ragCollectionTarget == "cc_workspace_project_main_generation" + assert request.ragBaseGenerationManifestSha256 == "3" * 64 + assert request.ragPrGenerationFingerprint is None + assert request.ragPrOverlayGenerationManifestSha256 is None + + +@pytest.mark.asyncio +async def test_incomplete_pr_overlay_receipt_degrades_to_exact_base_binding(): + rag = DeterministicRagSpy() + complete_index = rag.index_pr_files + + async def omit_overlay_manifest(**kwargs): + result = await complete_index(**kwargs) + result.pop("overlay_generation_manifest_sha256") + result["review_groups"] = [["src/file_0.py"]] + return result + + rag.index_pr_files = omit_overlay_manifest + request = _request() + orchestrator = MultiStageReviewOrchestrator( + llm=object(), + mcp_client=None, + rag_client=rag, + ) + + await orchestrator._index_pr_files( + request, + DiffProcessor().process(request.rawDiff), + ) + + assert orchestrator._pr_indexed is False + assert orchestrator._repository_review_groups == () + assert request.ragCollectionTarget == "cc_workspace_project_main_generation" + assert request.ragBaseGenerationManifestSha256 == "3" * 64 + assert request.ragPrGenerationFingerprint is None + assert request.ragPrOverlayGenerationManifestSha256 is None @pytest.mark.asyncio diff --git a/python-ecosystem/inference-orchestrator/tests/test_qa_documentation.py b/python-ecosystem/inference-orchestrator/tests/test_qa_documentation.py index 481b51cf..19a454e8 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_qa_documentation.py +++ b/python-ecosystem/inference-orchestrator/tests/test_qa_documentation.py @@ -7,9 +7,10 @@ BaseOrchestrator._simple_batch, filter_diff_for_files, get_file_content_from_enrichment, build_enrichment_lookup """ -import pytest import json -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock + +import pytest from service.qa_documentation.qa_doc_orchestrator import QaDocOrchestrator from service.qa_documentation.base_orchestrator import ( @@ -18,6 +19,7 @@ emit_progress, emit_error, ) +from utils.prompts.constants_qa_doc import QA_DOC_CUSTOM_PROMPT # ── emit_status / emit_progress / emit_error ───────────────────── @@ -85,6 +87,13 @@ def test_items_have_file_info_and_priority(self): assert item["file_info"].path == "a.py" assert item["priority"] == "MEDIUM" + def test_qa_orchestrator_has_no_rag_mutation_lifecycle(self): + orchestrator = QaDocOrchestrator(llm=MagicMock()) + + assert not hasattr(orchestrator, "rag_client") + assert not hasattr(orchestrator, "index_pr_files") + assert not hasattr(orchestrator, "cleanup_pr_files") + # ── BaseOrchestrator.filter_diff_for_files ─────────────────────── @@ -209,6 +218,122 @@ def test_compact_format(self): assert " " not in result # Compact separators +class TestIndependentTestCases: + def test_custom_template_cannot_remove_test_cases(self): + assert "custom template MUST NOT remove test cases" in QA_DOC_CUSTOM_PROMPT + assert "" in QA_DOC_CUSTOM_PROMPT + assert "" in QA_DOC_CUSTOM_PROMPT + + def test_detects_only_marked_structured_scenarios(self): + valid = """ + + **Checkout succeeds** (HIGH) + - **Expected Result:** Confirmation appears + + """ + assert QaDocOrchestrator._contains_extractable_test_cases(valid) + assert not QaDocOrchestrator._contains_extractable_test_cases( + "**Checkout succeeds** (HIGH)" + ) + assert not QaDocOrchestrator._contains_extractable_test_cases(""" + + + **Scenario outside the disclosure boundary** (HIGH) + """) + assert not QaDocOrchestrator._contains_extractable_test_cases(""" + + **Scenario between reversed markers** (HIGH) + + """) + + def test_moves_an_overbroad_end_marker_before_later_numbered_sections(self): + documentation = """ + ### 1. Change Summary + Checkout changed. + + + ### 3. Test Scenarios + ### Checkout + **Checkout succeeds** (HIGH) + - **Expected Result:** Confirmation appears + + ### 4. Edge Cases and Negative Testing + - Try an expired card. + + ### 5. Regression Risks + - Existing card payments. + + ### 6. Environment and Setup Notes + - Use the QA environment. + + """ + + normalized = QaDocOrchestrator._normalize_test_case_markers(documentation) + + assert normalized.index("") < normalized.index( + "### 4. Edge Cases and Negative Testing" + ) + assert normalized.index("") > normalized.index( + "**Checkout succeeds** (HIGH)" + ) + assert "### 5. Regression Risks" in normalized + assert "### 6. Environment and Setup Notes" in normalized + + @pytest.mark.asyncio(loop_scope="function") + async def test_repairs_a_custom_document_that_omits_test_cases(self): + response = MagicMock(content=""" + + ### Test Scenarios + **Checkout succeeds** (HIGH) + - **Expected Result:** Confirmation appears + + """) + llm = MagicMock() + llm.ainvoke = AsyncMock(return_value=response) + orchestrator = QaDocOrchestrator(llm=llm) + placeholders = { + "output_language": "English", + "pr_number": "12", + "project_name": "Project", + "pr_title": "Checkout", + "task_key": "QA-12", + "task_summary": "Test checkout", + "source_branch": "feature", + "target_branch": "main", + "task_context": "", + "analysis_summary": "Checkout behavior changed", + "diff": "+ changed behavior", + } + + repaired = await orchestrator._ensure_test_cases("# Custom QA summary", placeholders) + + assert repaired.startswith("# Custom QA summary") + assert "" in repaired + assert "**Checkout succeeds** (HIGH)" in repaired + + @pytest.mark.asyncio(loop_scope="function") + async def test_rejects_a_repair_without_a_structured_scenario(self): + llm = MagicMock() + llm.ainvoke = AsyncMock(return_value=MagicMock(content="General testing notes")) + orchestrator = QaDocOrchestrator(llm=llm) + placeholders = { + "output_language": "English", + "pr_number": "12", + "project_name": "Project", + "pr_title": "Checkout", + "task_key": "QA-12", + "task_summary": "Test checkout", + "source_branch": "feature", + "target_branch": "main", + "task_context": "", + "analysis_summary": "Checkout behavior changed", + "diff": "+ changed behavior", + } + + with pytest.raises(ValueError, match="no structured scenarios"): + await orchestrator._ensure_test_cases("# Custom QA summary", placeholders) + + # ── QaDocOrchestrator._build_placeholders ──────────────────────── class TestBuildPlaceholders: @@ -250,8 +375,57 @@ def test_defaults_when_empty(self): ) assert result["project_name"] == "Unknown" assert result["pr_number"] == "N/A" + assert result["pr_title"] == "QA documentation" assert result["diff"] == "No diff available." + def test_title_falls_back_to_task_summary_before_task_key(self): + orch = QaDocOrchestrator(llm=MagicMock()) + result = orch._build_placeholders( + project_name="TestProject", + pr_number=42, + issues_found=0, + files_analyzed=0, + pr_metadata={"prTitle": "N/A"}, + task_context_dict={ + "task_key": "JIRA-123", + "task_summary": "Support split shipments", + }, + task_context_block="", + diff="d", + ) + + assert result["pr_title"] == "Support split shipments" + + def test_title_falls_back_to_pr_number_without_task_metadata(self): + orch = QaDocOrchestrator(llm=MagicMock()) + result = orch._build_placeholders( + project_name=None, + pr_number=42, + issues_found=0, + files_analyzed=0, + pr_metadata={"prTitle": " "}, + task_context_dict=None, + task_context_block="", + diff="d", + ) + + assert result["pr_title"] == "PR #42" + + def test_replaces_na_in_the_rendered_guide_title(self): + documentation = """# QA Testing Guide — N/A + +## 1. What Changed +Checkout behavior changed. +""" + + normalized = QaDocOrchestrator._normalize_document_title( + documentation, + "Support split shipments", + ) + + assert normalized.startswith("# QA Testing Guide — Support split shipments") + assert "— N/A" not in normalized + def test_custom_language(self): orch = QaDocOrchestrator(llm=MagicMock()) result = orch._build_placeholders( diff --git a/python-ecosystem/inference-orchestrator/tests/test_queue_consumer.py b/python-ecosystem/inference-orchestrator/tests/test_queue_consumer.py index 027e737f..746dd613 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/tests/test_queue_consumer.py @@ -112,6 +112,39 @@ async def process(_request, _callback): assert events[-1]["type"] == "final" +@pytest.mark.asyncio(loop_scope="function") +async def test_service_error_is_the_only_terminal_review_event(): + review_service = MagicMock() + + async def process(_request, callback): + callback({"type": "error", "message": "provider failed"}) + callback({"type": "status", "state": "late_diagnostic"}) + return { + "result": { + "status": "error", + "message": "provider failed", + }, + } + + review_service.process_review_request = AsyncMock(side_effect=process) + consumer = RedisQueueConsumer(review_service) + consumer._redis = FakeRedis() + + with patch("server.queue_consumer.ReviewRequestDto", return_value=MagicMock()): + await consumer._handle_job(_payload()) + + events = [event for _, event in consumer._redis.events] + assert events == [ + { + "type": "status", + "state": "acknowledged", + "message": "Orchestrator picked up job from queue", + }, + {"type": "error", "message": "provider failed"}, + ] + assert sum(event["type"] in {"error", "final"} for event in events) == 1 + + @pytest.mark.asyncio(loop_scope="function") async def test_neutral_mixed_language_dry_run_traverses_queue_handler( monkeypatch, @@ -291,3 +324,56 @@ async def stop_after_dequeue(*_args, **_kwargs): timeout=1, ) assert not consumer._job_semaphore.locked() + + +@pytest.mark.asyncio(loop_scope="function") +async def test_stop_waits_for_admitted_review_before_closing_redis(): + consumer = RedisQueueConsumer(MagicMock()) + consumer._redis = MagicMock() + consumer._redis.aclose = AsyncMock() + started = asyncio.Event() + release = asyncio.Event() + + async def handle_until_released(_payload): + started.set() + await release.wait() + + consumer._handle_job = AsyncMock(side_effect=handle_until_released) + await consumer._job_semaphore.acquire() + job_task = asyncio.create_task(consumer._handle_admitted_job("payload")) + consumer._job_tasks.add(job_task) + job_task.add_done_callback(consumer._job_tasks.discard) + await started.wait() + + stop_task = asyncio.create_task(consumer.stop()) + await asyncio.sleep(0) + + assert not stop_task.done() + consumer._redis.aclose.assert_not_awaited() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + + assert job_task.done() + consumer._redis.aclose.assert_awaited_once_with() + + +def test_redis_outage_diagnostic_is_bounded_until_recovery(): + consumer = RedisQueueConsumer(MagicMock()) + + with ( + patch("server.queue_consumer.logger.warning") as warning, + patch("server.queue_consumer.logger.debug") as debug, + patch("server.queue_consumer.logger.info") as info, + ): + consumer._record_redis_failure("review event publication", RuntimeError("down")) + consumer._record_redis_failure("review consumer heartbeat", RuntimeError("still down")) + consumer._record_redis_success("review queue read") + consumer._record_redis_success("review consumer heartbeat") + + warning.assert_called_once() + debug.assert_called_once() + info.assert_called_once_with( + "Redis connectivity restored during %s", + "review consumer heartbeat", + ) diff --git a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py index 510cac40..ec264777 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py +++ b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py @@ -1,6 +1,8 @@ """ Unit tests for service.rag.rag_client — RagClient (all async methods). """ +import logging + import pytest import httpx import respx @@ -185,6 +187,19 @@ async def test_delete_pr_files_uses_exact_generation_target(self): ] == "cc_w1_p2_branch_generation" await c.close() + @pytest.mark.asyncio(loop_scope="function") + @respx.mock + async def test_delete_pr_files_treats_missing_collection_as_idempotent(self): + respx.delete("http://rag:8001/index/pr-files/ws/proj/1").mock( + return_value=httpx.Response( + 200, + json={"status": "skipped", "message": "Collection does not exist"}, + ) + ) + c = RagClient(base_url="http://rag:8001", enabled=True) + assert await c.delete_pr_files("ws", "proj", 1) is True + await c.close() + # ── Error handling ─────────────────────────────────────────── @@ -204,7 +219,7 @@ async def test_get_pr_context_http_error(self): @pytest.mark.asyncio(loop_scope="function") @respx.mock - async def test_get_pr_context_preserves_reindex_409_detail(self): + async def test_get_pr_context_preserves_reindex_409_detail(self, caplog): respx.post("http://rag:8001/query/pr-context").mock( return_value=httpx.Response( 409, @@ -214,12 +229,18 @@ async def test_get_pr_context_preserves_reindex_409_detail(self): ) ) c = RagClient(base_url="http://rag:8001", enabled=True) - r = await c.get_pr_context("ws", "proj", "main", ["a.py"]) + with caplog.at_level(logging.DEBUG, logger="service.rag.rag_client"): + r = await c.get_pr_context("ws", "proj", "main", ["a.py"]) assert r == { "status": "error", "status_code": 409, "error": "branch 'main' requires a full reindex", } + assert not any( + record.levelno >= logging.WARNING + for record in caplog.records + if record.name == "service.rag.rag_client" + ) await c.close() @pytest.mark.asyncio(loop_scope="function") @@ -256,7 +277,40 @@ async def test_deterministic_context_error(self): @pytest.mark.asyncio(loop_scope="function") @respx.mock - async def test_index_pr_files_error(self): + async def test_deterministic_context_409_is_returned_without_client_warning( + self, + caplog, + ): + respx.post("http://rag:8001/query/deterministic").mock( + return_value=httpx.Response( + 409, + json={"detail": "requested PR overlay generation is unavailable"}, + ) + ) + c = RagClient(base_url="http://rag:8001", enabled=True) + with caplog.at_level(logging.DEBUG, logger="service.rag.rag_client"): + r = await c.get_deterministic_context( + "ws", + "proj", + ["main"], + ["a.py"], + ) + + assert r == { + "status": "error", + "status_code": 409, + "error": "requested PR overlay generation is unavailable", + } + assert not any( + record.levelno >= logging.WARNING + for record in caplog.records + if record.name == "service.rag.rag_client" + ) + await c.close() + + @pytest.mark.asyncio(loop_scope="function") + @respx.mock + async def test_index_pr_files_error(self, caplog): respx.post("http://rag:8001/index/pr-files").mock( return_value=httpx.Response( 409, @@ -264,20 +318,58 @@ async def test_index_pr_files_error(self): ) ) c = RagClient(base_url="http://rag:8001", enabled=True) - r = await c.index_pr_files("ws", "proj", 1, "main", [{"path": "a.py", "content": "x", "change_type": "M"}]) + with caplog.at_level(logging.DEBUG, logger="service.rag.rag_client"): + r = await c.index_pr_files("ws", "proj", 1, "main", [{"path": "a.py", "content": "x", "change_type": "M"}]) assert r["status"] == "error" assert r["status_code"] == 409 assert r["error"] == "target branch is missing plugin snapshots" + assert not any( + record.levelno >= logging.WARNING + for record in caplog.records + if record.name == "service.rag.rag_client" + ) await c.close() @pytest.mark.asyncio(loop_scope="function") @respx.mock - async def test_delete_pr_files_error(self): + async def test_delete_pr_files_error(self, caplog): respx.delete("http://rag:8001/index/pr-files/ws/proj/1").mock( - return_value=httpx.Response(500) + return_value=httpx.Response( + 500, + json={"detail": "cleanup backend timed out"}, + ) ) c = RagClient(base_url="http://rag:8001", enabled=True) - assert await c.delete_pr_files("ws", "proj", 1) is False + with caplog.at_level(logging.WARNING, logger="service.rag.rag_client"): + assert await c.delete_pr_files("ws", "proj", 1) is False + assert "status=500 detail=cleanup backend timed out" in caplog.text + await c.close() + + @pytest.mark.asyncio(loop_scope="function") + @respx.mock + async def test_delete_pr_failure_logs_only_outage_transitions(self, caplog): + route = respx.delete( + "http://rag:8001/index/pr-files/ws/proj/1" + ).mock(side_effect=[ + httpx.Response(500, json={"detail": "timed out"}), + httpx.Response(500, json={"detail": "timed out"}), + httpx.Response(200, json={"status": "deleted"}), + ]) + c = RagClient(base_url="http://rag:8001", enabled=True) + + with caplog.at_level(logging.DEBUG, logger="service.rag.rag_client"): + assert await c.delete_pr_files("ws", "proj", 1) is False + assert await c.delete_pr_files("ws", "proj", 1) is False + assert await c.delete_pr_files("ws", "proj", 1) is True + + assert route.call_count == 3 + warnings = [ + record for record in caplog.records + if record.levelno == logging.WARNING + and "RAG PR cleanup is degraded" in record.getMessage() + ] + assert len(warnings) == 1 + assert "RAG PR cleanup recovered" in caplog.text await c.close() diff --git a/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py index 405c33c6..ec007212 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py @@ -5,7 +5,7 @@ _create_mcp_client """ import pytest -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from service.review.review_service import ( ReviewService, @@ -166,6 +166,38 @@ def test_non_pr_request_can_use_global_fallback(self): assert allow_unbound_global_rag_fallback(request) is True + @pytest.mark.asyncio(loop_scope="function") + async def test_failed_global_fallback_emits_reduced_context_status( + self, + service, + ): + service.rag_client = MagicMock(enabled=True) + service.rag_client.get_pr_context = AsyncMock(return_value={ + "status": "error", + "status_code": 503, + "error": "RAG service unavailable", + }) + service.rag_cache.get.return_value = None + request = MagicMock( + ragEnabled=True, + projectId=1, + pullRequestId=None, + projectWorkspace="ws", + projectNamespace="project", + changedFiles=["src/a.py"], + diffSnippets=[], + prTitle="Change A", + prDescription=None, + ) + request.get_rag_branch.return_value = "main" + request.get_rag_base_branch.return_value = None + events = [] + + result = await service._fetch_rag_context(request, events.append) + + assert result is None + assert any(event.get("state") == "rag_skipped" for event in events) + # ── _create_llm ────────────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py b/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py index 461da71f..270dac9f 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py @@ -8,6 +8,7 @@ """ import pytest import asyncio +import logging import time from unittest.mock import MagicMock, patch, AsyncMock @@ -39,6 +40,7 @@ _build_duplication_queries_from_diff, _scope_deterministic_to_diff, _extract_calibrated_issues, + _invoke_stage_1_batch_llm, create_smart_batches_wrapper, ) from model.multi_stage import ( @@ -54,6 +56,27 @@ # ── chunk_files ────────────────────────────────────────────────── +@pytest.mark.asyncio(loop_scope="function") +async def test_batch_llm_attempt_details_do_not_duplicate_owner_warning(caplog): + class FailingLlm: + def with_structured_output(self, _schema): + return self + + async def ainvoke(self, _prompt): + raise RuntimeError("provider unavailable") + + with caplog.at_level(logging.DEBUG): + result = await _invoke_stage_1_batch_llm( + FailingLlm(), "prompt", ["src/a.py"], "capped" + ) + + assert result is None + assert not [ + record for record in caplog.records + if record.levelno >= logging.WARNING + ] + + class TestChunkFiles: def _make_groups(self, paths_per_group): groups = [] @@ -1151,6 +1174,7 @@ def _exact_request(self): request.currentCommitHash = "a" * 40 request.commitHash = "a" * 40 request.baseCommitHash = "b" * 40 + request.ragCollectionTarget = "cc_ws_proj_main_generation" request.ragBaseGenerationManifestSha256 = "c" * 64 request.ragPrGenerationFingerprint = "d" * 64 request.ragPrOverlayGenerationManifestSha256 = "e" * 64 @@ -1161,6 +1185,35 @@ def _exact_request(self): request.previousCodeAnalysisIssues = [] return request + @staticmethod + def _batch(path="src/a.py"): + return [{ + "file": ReviewFile( + path=path, + focus_areas=["general"], + risk_level="MEDIUM", + ), + "priority": "MEDIUM", + }] + + class _FallbackAwaitProbe: + def __init__(self): + self.awaited = False + + def __await__(self): + self.awaited = True + + async def resolve(): + return { + "relevant_code": [{ + "text": "STALE_FALLBACK_SENTINEL", + "metadata": {"path": "src/a.py"}, + "score": 1.0, + }] + } + + return resolve().__await__() + @pytest.mark.asyncio(loop_scope="function") async def test_missing_target_branch_does_not_query_an_invented_branch(self): request = self._request() @@ -1182,6 +1235,7 @@ async def test_missing_target_branch_does_not_query_an_invented_branch(self): assert result is None assert state.deterministic_retrieval_states == ["failed"] + assert state.context_disabled is True rag.get_deterministic_context.assert_not_awaited() rag.get_pr_context.assert_not_awaited() rag.search_for_duplicates.assert_not_awaited() @@ -1373,49 +1427,126 @@ async def search_for_duplicates(self, **kwargs): assert rag.semantic_calls == 1 @pytest.mark.asyncio(loop_scope="function") - async def test_deterministic_failure_is_recorded_and_semantic_filler_survives(self): + async def test_concurrent_semantic_failures_emit_one_owner_warning( + self, + caplog, + ): + release = asyncio.Event() + class Rag: + def __init__(self): + self.semantic_calls = 0 + async def get_deterministic_context(self, **kwargs): + return {"context": {"chunks": [], "related_definitions": {}}} + + async def get_pr_context(self, **kwargs): + self.semantic_calls += 1 + await release.wait() return { "status": "error", - "status_code": 500, - "error": "exact retrieval failed", + "status_code": 503, + "error": "RAG service unavailable", } - async def get_pr_context(self, **kwargs): + async def search_for_duplicates(self, **kwargs): + return [] + + rag = Rag() + state = Stage1RagState() + with caplog.at_level( + logging.WARNING, + logger="service.review.orchestrator.stage_1_file_review", + ): + tasks = [ + asyncio.create_task(fetch_batch_rag_context( + rag, + self._request(), + [file_path], + ["changed line"], + batch_priority="MEDIUM", + rag_state=state, + )) + for file_path in ("src/a.py", "src/b.py") + ] + while rag.semantic_calls < 2: + await asyncio.sleep(0) + release.set() + assert await asyncio.gather(*tasks) == [None, None] + + assert state.semantic_disabled is True + assert state.semantic_failures == 1 + owner_warnings = [ + record for record in caplog.records + if record.levelno == logging.WARNING + and "Semantic RAG filler failed" in record.getMessage() + ] + assert len(owner_warnings) == 1 + + @pytest.mark.asyncio(loop_scope="function") + async def test_deterministic_failure_opens_one_review_context_circuit( + self, + caplog, + ): + class Rag: + def __init__(self): + self.deterministic_calls = 0 + self.semantic_calls = 0 + + async def get_deterministic_context(self, **kwargs): + self.deterministic_calls += 1 return { - "context": { - "relevant_code": [ - { - "text": "class Client {}", - "metadata": {"path": "src/client.py"}, - } - ] - } + "status": "error", + "status_code": 500, + "error": "RAG service unavailable", } + async def get_pr_context(self, **kwargs): + self.semantic_calls += 1 + raise AssertionError("semantic retrieval must not run") + async def search_for_duplicates(self, **kwargs): return [] + rag = Rag() state = Stage1RagState() - result = await fetch_batch_rag_context( - Rag(), - self._request(), - ["src/a.py"], - ["changed line"], - batch_priority="MEDIUM", - rag_state=state, - ) + with caplog.at_level( + logging.WARNING, + logger="service.review.orchestrator.stage_1_file_review", + ): + first = await fetch_batch_rag_context( + rag, + self._request(), + ["src/a.py"], + ["changed line"], + batch_priority="MEDIUM", + rag_state=state, + ) + second = await fetch_batch_rag_context( + rag, + self._request(), + ["src/b.py"], + ["changed line"], + batch_priority="MEDIUM", + rag_state=state, + ) - assert result is not None - assert [chunk["text"] for chunk in result["relevant_code"]] == [ - "class Client {}" - ] + assert first is None + assert second is None + assert rag.deterministic_calls == 1 + assert rag.semantic_calls == 0 assert state.deterministic_retrieval_states == ["failed"] - assert state.semantic_disabled is False + assert state.context_disabled is True + assert state.semantic_disabled is True + owner_warnings = [ + record for record in caplog.records + if record.levelno == logging.WARNING + and "Optional RAG context is unavailable" in record.getMessage() + ] + assert len(owner_warnings) == 1 @pytest.mark.asyncio(loop_scope="function") - async def test_exact_deterministic_failure_prevents_review_model_call(self): + async def test_exact_deterministic_failure_fails_open_to_review_model(self): class Rag: async def get_deterministic_context(self, **kwargs): return { @@ -1430,72 +1561,129 @@ async def get_pr_context(self, **kwargs): async def search_for_duplicates(self, **kwargs): return [] - batch = [{ - "file": ReviewFile( - path="src/a.py", - focus_areas=["general"], - risk_level="MEDIUM", - ), - "priority": "MEDIUM", - }] + fallback = self._FallbackAwaitProbe() with patch( "service.review.orchestrator.stage_1_file_review." "_invoke_stage_1_batch_llm", new_callable=AsyncMock, ) as invoke: - with pytest.raises( - RuntimeError, - match="revision-bound deterministic RAG retrieval failed", - ): - await review_file_batch( - MagicMock(), - self._exact_request(), - batch, - rag_client=Rag(), - prepared_context=Stage1PreparedContext(), - pr_indexed=True, - rag_state=Stage1RagState(), - ) + invoke.return_value = [] + state = Stage1RagState() + result = await review_file_batch( + MagicMock(), + self._exact_request(), + self._batch(), + rag_client=Rag(), + prepared_context=Stage1PreparedContext(), + fallback_rag_context=fallback, + pr_indexed=True, + rag_state=state, + ) - invoke.assert_not_awaited() + assert result == [] + assert state.context_disabled is True + assert fallback.awaited is False + assert "STALE_FALLBACK_SENTINEL" not in invoke.await_args.args[1] + invoke.assert_awaited_once() @pytest.mark.asyncio(loop_scope="function") - async def test_exact_missing_rag_client_prevents_review_model_call(self): - batch = [{ - "file": ReviewFile( - path="src/a.py", - focus_areas=["general"], - risk_level="MEDIUM", - ), - "priority": "MEDIUM", - }] + async def test_exact_missing_rag_client_fails_open_to_review_model(self): + fallback = self._FallbackAwaitProbe() with patch( "service.review.orchestrator.stage_1_file_review." "_invoke_stage_1_batch_llm", new_callable=AsyncMock, ) as invoke: - with pytest.raises( - RuntimeError, - match="requires a RAG client", - ): - await review_file_batch( - MagicMock(), - self._exact_request(), - batch, - rag_client=None, - prepared_context=Stage1PreparedContext(), - pr_indexed=True, - rag_state=Stage1RagState(), - ) + invoke.return_value = [] + state = Stage1RagState() + result = await review_file_batch( + MagicMock(), + self._exact_request(), + self._batch(), + rag_client=None, + prepared_context=Stage1PreparedContext(), + fallback_rag_context=fallback, + pr_indexed=True, + rag_state=state, + ) + + assert result == [] + assert state.context_disabled is True + assert fallback.awaited is False + assert "STALE_FALLBACK_SENTINEL" not in invoke.await_args.args[1] + invoke.assert_awaited_once() + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_base_binding_never_consumes_unbound_fallback(self): + request = self._exact_request() + request.ragPrGenerationFingerprint = None + request.ragPrOverlayGenerationManifestSha256 = None + fallback = self._FallbackAwaitProbe() + + with patch( + "service.review.orchestrator.stage_1_file_review." + "_invoke_stage_1_batch_llm", + new_callable=AsyncMock, + return_value=[], + ) as invoke: + state = Stage1RagState() + result = await review_file_batch( + MagicMock(), + request, + self._batch(), + rag_client=None, + prepared_context=Stage1PreparedContext(), + fallback_rag_context=fallback, + pr_indexed=False, + rag_state=state, + ) + + assert result == [] + assert state.context_disabled is True + assert fallback.awaited is False + assert "STALE_FALLBACK_SENTINEL" not in invoke.await_args.args[1] + + @pytest.mark.asyncio(loop_scope="function") + async def test_legacy_unbound_review_still_uses_scoped_fallback(self): + request = self._request() + request.rawDiff = "" + request.deltaDiff = None + request.taskContext = None + request.projectRules = [] + request.previousCodeAnalysisIssues = [] + fallback = self._FallbackAwaitProbe() + + with patch( + "service.review.orchestrator.stage_1_file_review." + "_invoke_stage_1_batch_llm", + new_callable=AsyncMock, + return_value=[], + ) as invoke: + result = await review_file_batch( + MagicMock(), + request, + self._batch(), + rag_client=None, + prepared_context=Stage1PreparedContext(), + fallback_rag_context=fallback, + pr_indexed=False, + rag_state=Stage1RagState(), + ) - invoke.assert_not_awaited() + assert result == [] + assert fallback.awaited is True + assert "STALE_FALLBACK_SENTINEL" in invoke.await_args.args[1] @pytest.mark.asyncio(loop_scope="function") - async def test_exact_partial_deterministic_state_fails_closed(self): + async def test_exact_partial_deterministic_state_fails_open_once(self): class Rag: + def __init__(self): + self.deterministic_calls = 0 + async def get_deterministic_context(self, **kwargs): + self.deterministic_calls += 1 return { "context": { "chunks": [{"text": "partial context"}], @@ -1506,21 +1694,33 @@ async def get_deterministic_context(self, **kwargs): async def search_for_duplicates(self, **kwargs): return [] - with pytest.raises( - RuntimeError, - match="deterministic RAG retrieval is not complete: partial", - ): - await fetch_batch_rag_context( - Rag(), - self._exact_request(), - ["src/a.py"], - ["changed line"], - pr_indexed=True, - rag_state=Stage1RagState(), - ) + rag = Rag() + state = Stage1RagState() + first = await fetch_batch_rag_context( + rag, + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=state, + ) + second = await fetch_batch_rag_context( + rag, + self._exact_request(), + ["src/b.py"], + ["changed line"], + pr_indexed=True, + rag_state=state, + ) + + assert first is None + assert second is None + assert rag.deterministic_calls == 1 + assert state.context_disabled is True + assert state.deterministic_retrieval_states == ["partial"] @pytest.mark.asyncio(loop_scope="function") - async def test_exact_semantic_transport_failure_fails_closed(self): + async def test_exact_semantic_transport_failure_keeps_review_available(self): class Rag: async def get_deterministic_context(self, **kwargs): return { @@ -1540,15 +1740,105 @@ async def get_pr_context(self, **kwargs): async def search_for_duplicates(self, **kwargs): return [] - with pytest.raises(RuntimeError, match="semantic backend unavailable"): - await fetch_batch_rag_context( - Rag(), - self._exact_request(), - ["src/a.py"], - ["changed line"], - pr_indexed=True, - rag_state=Stage1RagState(), - ) + state = Stage1RagState() + result = await fetch_batch_rag_context( + Rag(), + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=state, + ) + + assert result is None + assert state.context_disabled is False + assert state.semantic_disabled is True + assert state.semantic_disable_reason == "semantic backend unavailable" + + @pytest.mark.asyncio(loop_scope="function") + async def test_degraded_overlay_uses_only_exact_base_binding( + self, + monkeypatch, + ): + import service.review.orchestrator.stage_1_file_review as stage1 + + monkeypatch.setattr(stage1, "SEMANTIC_RAG_FILLER_ENABLED", True) + + class Rag: + def __init__(self): + self.deterministic_request = None + self.semantic_request = None + + async def get_deterministic_context(self, **kwargs): + self.deterministic_request = kwargs + return { + "context": { + "chunks": [], + "_metadata": {"retrieval_state": "complete"}, + } + } + + async def get_pr_context(self, **kwargs): + self.semantic_request = kwargs + return {"context": {"relevant_code": []}} + + async def search_for_duplicates(self, **kwargs): + return [] + + request = self._exact_request() + request.ragPrGenerationFingerprint = None + request.ragPrOverlayGenerationManifestSha256 = None + rag = Rag() + + result = await fetch_batch_rag_context( + rag, + request, + ["src/a.py"], + ["changed line"], + pr_indexed=False, + rag_state=Stage1RagState(), + ) + + assert result == {"relevant_code": []} + assert rag.deterministic_request["branches"] == ["feature"] + assert rag.deterministic_request["base_revision"] == "b" * 40 + assert ( + rag.deterministic_request["base_generation_manifest_sha256"] + == "c" * 64 + ) + assert ( + rag.deterministic_request["collection_target"] + == "cc_ws_proj_main_generation" + ) + assert rag.deterministic_request["pr_number"] is None + assert rag.deterministic_request["pr_changed_files"] is None + assert rag.deterministic_request["source_revision"] is None + assert rag.deterministic_request["pr_generation_fingerprint"] is None + assert ( + rag.deterministic_request[ + "pr_overlay_generation_manifest_sha256" + ] + is None + ) + assert rag.semantic_request["base_branch"] is None + assert rag.semantic_request["pr_number"] is None + assert rag.semantic_request["all_pr_changed_files"] is None + assert rag.semantic_request["deleted_files"] is None + assert rag.semantic_request["source_revision"] is None + assert rag.semantic_request["base_revision"] == "b" * 40 + assert ( + rag.semantic_request["base_generation_manifest_sha256"] + == "c" * 64 + ) + assert ( + rag.semantic_request["collection_target"] + == "cc_ws_proj_main_generation" + ) + assert rag.semantic_request["pr_generation_fingerprint"] is None + assert ( + rag.semantic_request["pr_overlay_generation_manifest_sha256"] + is None + ) @pytest.mark.asyncio(loop_scope="function") async def test_exact_bound_success_allows_intentional_semantic_disable( diff --git a/python-ecosystem/inference-orchestrator/tests/test_streaming_router_lifecycle.py b/python-ecosystem/inference-orchestrator/tests/test_streaming_router_lifecycle.py new file mode 100644 index 00000000..30275822 --- /dev/null +++ b/python-ecosystem/inference-orchestrator/tests/test_streaming_router_lifecycle.py @@ -0,0 +1,135 @@ +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from api.routers.commands import ask_endpoint, summarize_endpoint +from api.routers.review import review_endpoint + + +def _streaming_request(service_name, service): + state = SimpleNamespace(**{service_name: service}) + return SimpleNamespace( + headers={"accept": "application/x-ndjson"}, + app=SimpleNamespace(state=state), + ) + + +async def _assert_disconnect_cancels_runner(response, cancelled): + stream = response.body_iterator + queued = await anext(stream) + assert '"state": "queued"' in queued + + progress = await anext(stream) + assert '"state": "working"' in progress + + await stream.aclose() + await asyncio.wait_for(cancelled.wait(), timeout=1) + + +async def _collect_stream_events(response): + return [json.loads(line) async for line in response.body_iterator] + + +@pytest.mark.asyncio(loop_scope="function") +async def test_review_stream_disconnect_cancels_service_runner(): + cancelled = asyncio.Event() + service = MagicMock() + + async def process(_request, event_callback): + event_callback({"type": "status", "state": "working"}) + try: + await asyncio.Future() + finally: + cancelled.set() + + service.process_review_request = AsyncMock(side_effect=process) + response = await review_endpoint( + MagicMock(), + _streaming_request("review_service", service), + ) + + await _assert_disconnect_cancels_runner(response, cancelled) + + +@pytest.mark.asyncio(loop_scope="function") +async def test_review_stream_emits_one_terminal_service_error(): + service = MagicMock() + + async def process(_request, event_callback): + event_callback({"type": "error", "message": "provider failed"}) + event_callback({"type": "status", "state": "late_diagnostic"}) + return {"result": {"status": "error", "message": "provider failed"}} + + service.process_review_request = AsyncMock(side_effect=process) + response = await review_endpoint( + MagicMock(), + _streaming_request("review_service", service), + ) + + events = await _collect_stream_events(response) + assert events[-1] == {"type": "error", "message": "provider failed"} + assert sum(event["type"] in {"error", "final"} for event in events) == 1 + + +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.parametrize( + ("endpoint", "method_name"), + ( + (summarize_endpoint, "process_summarize"), + (ask_endpoint, "process_ask"), + ), +) +async def test_command_stream_disconnect_cancels_service_runner( + endpoint, + method_name, +): + cancelled = asyncio.Event() + service = MagicMock() + + async def process(_request, event_callback): + event_callback({"type": "status", "state": "working"}) + try: + await asyncio.Future() + finally: + cancelled.set() + + setattr(service, method_name, AsyncMock(side_effect=process)) + response = await endpoint( + MagicMock(), + _streaming_request("command_service", service), + ) + + await _assert_disconnect_cancels_runner(response, cancelled) + + +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.parametrize( + ("endpoint", "method_name"), + ( + (summarize_endpoint, "process_summarize"), + (ask_endpoint, "process_ask"), + ), +) +async def test_command_stream_emits_one_terminal_service_error( + endpoint, + method_name, +): + service = MagicMock() + + async def process(_request, event_callback): + event_callback({"type": "error", "message": "provider failed"}) + event_callback({"type": "status", "state": "late_diagnostic"}) + return {"error": "provider failed"} + + setattr(service, method_name, AsyncMock(side_effect=process)) + response = await endpoint( + MagicMock(), + _streaming_request("command_service", service), + ) + + events = await _collect_stream_events(response) + assert events[-1] == {"type": "error", "message": "provider failed"} + assert sum(event["type"] in {"error", "final"} for event in events) == 1 diff --git a/python-ecosystem/rag-pipeline/integration/test_pr_endpoints.py b/python-ecosystem/rag-pipeline/integration/test_pr_endpoints.py index 3a7a6f3e..2ef1d126 100644 --- a/python-ecosystem/rag-pipeline/integration/test_pr_endpoints.py +++ b/python-ecosystem/rag-pipeline/integration/test_pr_endpoints.py @@ -263,7 +263,10 @@ async def test_delete_pr_files_success(self, client, auth_headers, rag_app): import rag_pipeline.api.api as api_module im = api_module.index_manager im._get_project_collection_name.return_value = "col_ws_proj" - im._collection_manager.collection_exists.return_value = True + im._collection_manager.resolve_collection_target.side_effect = None + im._collection_manager.resolve_collection_target.return_value = ( + "col_ws_proj" + ) im.qdrant_client.delete.return_value = None resp = await client.delete( @@ -281,7 +284,8 @@ async def test_delete_pr_files_collection_not_found(self, client, auth_headers, import rag_pipeline.api.api as api_module im = api_module.index_manager im._get_project_collection_name.return_value = "col_ws_proj" - im._collection_manager.collection_exists.return_value = False + im._collection_manager.resolve_collection_target.side_effect = None + im._collection_manager.resolve_collection_target.return_value = None resp = await client.delete( "/index/pr-files/ws/proj/99", @@ -291,12 +295,22 @@ async def test_delete_pr_files_collection_not_found(self, client, auth_headers, body = resp.json() assert body["status"] == "skipped" + # The application singleton is session-scoped in this integration + # suite. Restore a normal resolver so this absence case cannot leak + # into the following Qdrant-error scenario. + im._collection_manager.resolve_collection_target.side_effect = ( + lambda collection_name: collection_name + ) + async def test_delete_pr_files_qdrant_error(self, client, auth_headers, rag_app): """Qdrant error → 500.""" import rag_pipeline.api.api as api_module im = api_module.index_manager im._get_project_collection_name.return_value = "col_ws_proj" - im._collection_manager.collection_exists.return_value = True + im._collection_manager.resolve_collection_target.side_effect = None + im._collection_manager.resolve_collection_target.return_value = ( + "col_ws_proj" + ) im.qdrant_client.delete.side_effect = RuntimeError("qdrant down") resp = await client.delete( diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/api.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/api.py index fcaccc42..a7a3c12c 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/api.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/api.py @@ -47,18 +47,34 @@ def _pending_janitor_interval_seconds() -> int: async def _pending_collection_janitor(manager: RAGIndexManager) -> None: """Periodically remove only expired, unowned pending collections.""" interval = _pending_janitor_interval_seconds() + unavailable = False while True: try: cleaned = await asyncio.to_thread( manager.cleanup_expired_pending_collections ) + if unavailable: + logger.info("Pending collection janitor recovered") + unavailable = False if cleaned: logger.info("Pending collection janitor removed %s collections", cleaned) except asyncio.CancelledError: raise - except Exception: + except Exception as exception: # Cleanup is auxiliary: retain uncertain collections and keep serving. - logger.exception("Pending collection janitor failed") + if not unavailable: + logger.warning( + "Pending collection janitor unavailable; retaining " + "uncertain collections: %s", + exception, + exc_info=True, + ) + unavailable = True + else: + logger.debug( + "Pending collection janitor still unavailable: %s", + exception, + ) await asyncio.sleep(interval) @@ -77,6 +93,17 @@ async def lifespan(app: FastAPI): config, plugin_catalog=index_manager.plugin_catalog, ) + from .routers.index import ( + cleanup_orphaned_index_repository_stream_workspaces, + ) + cleaned_stream_workspaces = ( + cleanup_orphaned_index_repository_stream_workspaces() + ) + if cleaned_stream_workspaces: + logger.info( + "Removed %s orphaned RAG HTTP index workspaces", + cleaned_stream_workspaces, + ) # Initialize and start the Redis Queue Consumer from ..server.rag_queue_consumer import RAGQueueConsumer @@ -98,10 +125,18 @@ async def lifespan(app: FastAPI): pass if hasattr(app.state, 'rag_queue_consumer'): await app.state.rag_queue_consumer.stop() + # HTTP streaming requests run synchronous indexing in dedicated workers. + # A disconnected response task can be gone before that call returns, so + # drain the independently tracked workers before closing shared embedding + # and Qdrant clients. + from .routers.index import drain_index_repository_stream_workers + await drain_index_repository_stream_workers() if hasattr(index_manager, 'embed_model') and hasattr(index_manager.embed_model, 'close'): index_manager.embed_model.close() if hasattr(query_service, 'embed_model') and hasattr(query_service.embed_model, 'close'): query_service.embed_model.close() + if query_service is not None: + query_service.close() if index_manager is not None: index_manager.close() logger.info("RAG Pipeline API shutdown complete") diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py index 21922f54..22028ac7 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py @@ -44,10 +44,12 @@ class IndexRequest(BaseModel): pattern=r"^[0-9a-f]{64}$", ) collection_target: Optional[str] = Field(default=None, min_length=1) + reuse_collection_target: Optional[str] = Field(default=None, min_length=1) publish_branch_alias: bool = False publish_legacy_project_alias: bool = False preserve_other_branches: bool = False cleanup_repo_path: bool = False + transfer_repo_ownership: bool = False include_patterns: Optional[List[str]] = None exclude_patterns: Optional[List[str]] = None @@ -142,6 +144,10 @@ class GenerationAliasPublicationRequest(BaseModel): branch: str commit: str collection_target: str = Field(min_length=1) + generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) publish_branch_alias: bool = True publish_legacy_project_alias: bool = False diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py index aeb8f473..96b325ef 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py @@ -1,10 +1,16 @@ """Index and branch management endpoints.""" import asyncio +import fcntl import json import logging -from queue import Empty, Queue -from threading import Thread -from typing import List +import os +import shutil +import time +from queue import Empty, Full, Queue +from pathlib import Path +from threading import Lock, Thread, current_thread +from typing import BinaryIO, Callable, List +from uuid import uuid4 from fastapi import APIRouter, HTTPException, BackgroundTasks, Query from fastapi.responses import StreamingResponse @@ -26,6 +32,223 @@ router = APIRouter(tags=["index"]) +class _IndexStreamWorkerRegistry: + """Track synchronous HTTP-stream indexing beyond request cancellation.""" + + def __init__(self) -> None: + self._workers: set[Thread] = set() + self._lock = Lock() + + def start(self, target: Callable[[], None]) -> Thread: + """Admit and start one worker before exposing its response stream.""" + + def run_tracked() -> None: + try: + target() + finally: + with self._lock: + self._workers.discard(current_thread()) + + worker = Thread( + target=run_tracked, + name="rag-index-progress", + # Graceful shutdown drains these workers explicitly. Keeping them + # non-daemon also prevents interpreter teardown from closing + # shared clients while an admitted index operation is still using + # them. + daemon=False, + ) + with self._lock: + self._workers.add(worker) + try: + worker.start() + except BaseException: + with self._lock: + self._workers.discard(worker) + raise + return worker + + async def wait_for(self, worker: Thread) -> None: + """Wait for a worker, deferring cancellation until it has returned.""" + cancellation: asyncio.CancelledError | None = None + while worker.is_alive(): + try: + await asyncio.sleep(0.05) + except asyncio.CancelledError as exception: + # A disconnected streaming client must not unwind its server + # handler while the admitted worker can still read the + # caller-owned repository snapshot. + cancellation = cancellation or exception + worker.join() + if cancellation is not None: + raise cancellation + + async def drain(self) -> None: + """Wait for every currently admitted stream worker.""" + announced = False + while True: + with self._lock: + active_workers = tuple(self._workers) + if not active_workers: + return + if not announced: + logger.info( + "Waiting for %s HTTP RAG indexing workers before shutdown", + len(active_workers), + ) + announced = True + for worker in active_workers: + await self.wait_for(worker) + + @property + def active_count(self) -> int: + with self._lock: + return len(self._workers) + +_index_stream_workers = _IndexStreamWorkerRegistry() +_OWNED_STREAM_DIRECTORY_PREFIX = "codecrow-rag-owned-stream-" +_DEFAULT_OWNED_STREAM_ORPHAN_AGE_SECONDS = 24 * 60 * 60 + + +async def drain_index_repository_stream_workers() -> None: + """Drain admitted HTTP index operations before shared clients close.""" + await _index_stream_workers.drain() + + +def cleanup_orphaned_index_repository_stream_workspaces( + max_age_seconds: int = _DEFAULT_OWNED_STREAM_ORPHAN_AGE_SECONDS, +) -> int: + """Remove only old RAG-owned workspaces left by an earlier process.""" + allowed_root = Path(os.environ.get("ALLOWED_REPO_ROOT", "/tmp")).resolve() + if not allowed_root.is_dir(): + return 0 + cutoff = time.time() - max(0, max_age_seconds) + cleaned = 0 + for candidate in allowed_root.iterdir(): + if ( + not candidate.name.startswith(_OWNED_STREAM_DIRECTORY_PREFIX) + or candidate.is_symlink() + or not candidate.is_dir() + ): + continue + try: + if candidate.stat().st_mtime > cutoff: + continue + lock_path = allowed_root / f".{candidate.name}.lock" + lock_descriptor = os.open( + lock_path, + os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, + 0o600, + ) + lock_file = os.fdopen(lock_descriptor, "a+b") + try: + try: + fcntl.flock( + lock_file.fileno(), + fcntl.LOCK_EX | fcntl.LOCK_NB, + ) + except BlockingIOError: + # Another RAG process still owns this workspace. Its age + # alone is never authority to disrupt an active worker. + continue + shutil.rmtree(candidate) + cleaned += 1 + finally: + lock_file.close() + if not candidate.exists(): + lock_path.unlink(missing_ok=True) + except FileNotFoundError: + continue + except Exception: + logger.exception( + "Failed to remove orphaned RAG HTTP index workspace: %s", + candidate, + ) + return cleaned + + +def _coalesce_stream_progress( + events: Queue[dict], + event: dict, +) -> None: + """Retain only the latest undelivered optional progress event.""" + while True: + try: + events.put_nowait(event) + return + except Full: + try: + events.get_nowait() + except Empty: + continue + + +def _take_stream_repository_ownership( + repo_path: str, +) -> tuple[Path, BinaryIO]: + """Atomically move one Java-owned snapshot into the RAG namespace.""" + allowed_root = Path(os.environ.get("ALLOWED_REPO_ROOT", "/tmp")).resolve() + source = Path(repo_path).resolve() + if ( + source.parent != allowed_root + or not source.name.startswith("codecrow-rag-branch-generation-") + or not source.is_dir() + ): + raise ValueError( + "stream repository ownership transfer requires an existing " + "codecrow-rag-branch-generation-* directory directly under " + f"{allowed_root}" + ) + owned = allowed_root / f"{_OWNED_STREAM_DIRECTORY_PREFIX}{uuid4().hex}" + lock_path = allowed_root / f".{owned.name}.lock" + lock_file = None + try: + lock_descriptor = os.open( + lock_path, + os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, + 0o600, + ) + lock_file = os.fdopen(lock_descriptor, "a+b") + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BaseException: + if lock_file is not None: + lock_file.close() + lock_path.unlink(missing_ok=True) + raise + # Both services mount the same temporary volume. A rename in that volume + # is atomic: after this point Java may safely clean the now-absent source + # path even if the admission SSE is lost, while RAG alone owns `owned`. + try: + source.rename(owned) + except BaseException: + lock_file.close() + lock_path.unlink(missing_ok=True) + raise + return owned, lock_file + + +def _remove_owned_stream_repository( + repo_path: Path, + lock_file: BinaryIO, +) -> None: + removed = False + try: + shutil.rmtree(repo_path) + removed = True + except FileNotFoundError: + removed = True + except Exception: + logger.exception( + "Failed to remove RAG-owned HTTP index workspace: %s", + repo_path, + ) + finally: + lock_path = repo_path.parent / f".{repo_path.name}.lock" + lock_file.close() + if removed: + lock_path.unlink(missing_ok=True) + + def _get_singletons(): """Get lifecycle-managed singletons from the api module.""" from ..api import config, index_manager @@ -101,6 +324,11 @@ def index_repository(request: IndexRequest, background_tasks: BackgroundTasks): optional_generation_args["source_tree_sha256"] = source_tree_sha256 if isinstance(collection_target, str) and collection_target: optional_generation_args["collection_target"] = collection_target + reuse_collection_target = getattr(request, "reuse_collection_target", None) + if isinstance(reuse_collection_target, str) and reuse_collection_target: + optional_generation_args["reuse_collection_target"] = ( + reuse_collection_target + ) if getattr(request, "publish_branch_alias", False) is True: optional_generation_args["publish_branch_alias"] = True if getattr(request, "publish_legacy_project_alias", False) is True: @@ -139,73 +367,153 @@ def index_repository_stream(request: IndexRequest): delivery a prerequisite for a successful snapshot. """ _, index_manager = _get_singletons() + progress_events: Queue[dict] = Queue(maxsize=1) + terminal_events: Queue[tuple[str, object]] = Queue(maxsize=1) + repository_ownership_transferred = ( + getattr(request, "transfer_repo_ownership", False) is True + ) + index_repo_path = Path(request.repo_path) + repository_ownership_lock: BinaryIO | None = None + if repository_ownership_transferred: + try: + ( + index_repo_path, + repository_ownership_lock, + ) = _take_stream_repository_ownership(request.repo_path) + except ValueError as exception: + raise HTTPException(status_code=400, detail=str(exception)) + except OSError as exception: + raise HTTPException( + status_code=503, + detail=( + "RAG could not take repository snapshot ownership: " + f"{exception}" + ), + ) - async def event_stream(): - events: Queue[tuple[str, object]] = Queue() - - def progress(event: dict) -> None: - events.put(("progress", event)) + def progress(event: dict) -> None: + _coalesce_stream_progress(progress_events, event) - def run_index() -> None: - try: - optional_generation_args = {} - if request.source_tree_sha256: - optional_generation_args["source_tree_sha256"] = ( - request.source_tree_sha256 - ) - if request.collection_target: - optional_generation_args["collection_target"] = ( - request.collection_target - ) - if getattr(request, "publish_branch_alias", False) is True: - optional_generation_args["publish_branch_alias"] = True - if getattr(request, "publish_legacy_project_alias", False) is True: - optional_generation_args["publish_legacy_project_alias"] = True - stats = index_manager.index_repository( - repo_path=request.repo_path, - workspace=request.workspace, - project=request.project, - branch=request.branch, - commit=request.commit, - preserve_other_branches=request.preserve_other_branches, - include_patterns=request.include_patterns, - exclude_patterns=request.exclude_patterns, - progress_callback=progress, - **optional_generation_args, + def run_index() -> None: + try: + optional_generation_args = {} + if request.source_tree_sha256: + optional_generation_args["source_tree_sha256"] = ( + request.source_tree_sha256 + ) + if request.collection_target: + optional_generation_args["collection_target"] = ( + request.collection_target + ) + if request.reuse_collection_target: + optional_generation_args["reuse_collection_target"] = ( + request.reuse_collection_target + ) + if getattr(request, "publish_branch_alias", False) is True: + optional_generation_args["publish_branch_alias"] = True + if getattr(request, "publish_legacy_project_alias", False) is True: + optional_generation_args["publish_legacy_project_alias"] = True + stats = index_manager.index_repository( + repo_path=str(index_repo_path), + workspace=request.workspace, + project=request.project, + branch=request.branch, + commit=request.commit, + preserve_other_branches=request.preserve_other_branches, + include_patterns=request.include_patterns, + exclude_patterns=request.exclude_patterns, + progress_callback=progress, + **optional_generation_args, + ) + terminal_events.put( + ("complete", stats.model_dump(mode="json")) + ) + except Exception as exception: + # The terminal event gives the Java job owner complete context and + # that owner emits the rate-bounded diagnostic. Avoid logging the + # same failure again at this transport layer. + logger.debug( + "RAG repository stream worker failed: %s", exception, + exc_info=True, + ) + terminal_events.put(("error", {"message": str(exception)})) + finally: + if repository_ownership_transferred: + _remove_owned_stream_repository( + index_repo_path, + repository_ownership_lock, ) - events.put(("complete", stats.model_dump(mode="json"))) - except Exception as exception: - logger.error("Error indexing repository with progress: %s", exception) - events.put(("error", {"message": str(exception)})) - worker = Thread( - target=run_index, - name="rag-index-progress", - daemon=True, - ) - worker.start() - while True: + try: + # Admission happens before successful response headers. Ownership is + # already represented by an atomic rename, so a lost admission event + # cannot make the Java caller delete the path this worker is reading. + worker = _index_stream_workers.start(run_index) + except BaseException: + if repository_ownership_transferred: + _remove_owned_stream_repository( + index_repo_path, + repository_ownership_lock, + ) + raise + + async def event_stream(): + try: + if repository_ownership_transferred: + admitted = { + "type": "admitted", + "repositoryOwnershipTransferred": True, + } + yield f"data: {json.dumps(admitted)}\n\n" + while True: + try: + payload = progress_events.get_nowait() + event_type = "progress" + except Empty: + try: + event_type, payload = terminal_events.get_nowait() + except Empty: + if not worker.is_alive(): + # run_index publishes a terminal event before it + # returns. This guards an unexpected BaseException + # in the worker without leaving the stream open. + event_type = "error" + payload = { + "message": ( + "RAG indexing worker stopped without a " + "terminal result" + ) + } + else: + # Polling thread-safe queues avoids nesting a + # blocking consumer inside Starlette's thread pool. + await asyncio.sleep(0.05) + continue + if event_type == "progress": + event = {"type": "progress", **payload} + elif event_type == "complete": + event = {"type": "complete", "result": payload} + else: + event = {"type": "error", **payload} + yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n" + if event_type in {"complete", "error"}: + break + finally: + # StreamingResponse closes this async generator when its client + # disconnects. Defer handler teardown until the independently + # admitted synchronous operation has returned. + worker_drain = asyncio.create_task( + _index_stream_workers.wait_for(worker) + ) try: - event_type, payload = events.get_nowait() - except Empty: - # Polling a thread-safe queue avoids nesting a blocking queue - # consumer inside Starlette's thread pool. The short wait keeps - # the event loop responsive and progress delivery prompt. - await asyncio.sleep(0.05) - continue - if event_type == "progress": - event = {"type": "progress", **payload} - elif event_type == "complete": - event = {"type": "complete", "result": payload} - else: - event = {"type": "error", **payload} - yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n" - if event_type in {"complete", "error"}: - # The terminal event is scheduled just before the producer - # returns. Join that final unwind so a short-lived consumer - # cannot finish while its producer thread is still active. - worker.join(timeout=1.0) - break + await asyncio.shield(worker_drain) + except asyncio.CancelledError: + # Cancellation can be delivered before an awaited coroutine + # executes its own cancellation handler. Shield admission + # draining as a separate task, then re-raise only after it has + # completed. + await worker_drain + raise return StreamingResponse(event_stream(), media_type="text/event-stream") @@ -348,6 +656,7 @@ def publish_generation_aliases(request: GenerationAliasPublicationRequest): branch=request.branch, commit=request.commit, collection_target=request.collection_target, + generation_manifest_sha256=request.generation_manifest_sha256, publish_branch_alias=request.publish_branch_alias, publish_legacy_project_alias=request.publish_legacy_project_alias, ) @@ -359,7 +668,10 @@ def publish_generation_aliases(request: GenerationAliasPublicationRequest): except MutationCoordinationUnavailable as e: raise HTTPException(status_code=503, detail=str(e)) except Exception as e: - logger.error("Error publishing readable generation aliases: %s", e) + # Alias repair is optional and retried by the registry owner, which + # emits the contextual transition alert. Avoid a fixed-delay ERROR + # stream here while preserving the HTTP failure for that caller. + logger.info("Readable generation alias publication failed: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -387,12 +699,22 @@ def delete_branch( project: str, branch: str, collection_target: str | None = Query(default=None), + generation_revision: str | None = Query(default=None, min_length=1), + generation_manifest_sha256: str | None = Query( + default=None, + pattern=r"^[0-9a-f]{64}$", + ), ): """Delete all points for a specific branch from the project collection.""" _, index_manager = _get_singletons() try: success = index_manager.delete_branch( - workspace, project, branch, collection_target=collection_target + workspace, + project, + branch, + collection_target=collection_target, + generation_revision=generation_revision, + generation_manifest_sha256=generation_manifest_sha256, ) if success: return { diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py index b86625e4..32dbd678 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py @@ -800,10 +800,10 @@ def index_pr_files(request: PRIndexRequest): except HTTPException: raise except IncrementalIndexPreconditionError as e: - logger.warning("Rejected PR indexing against invalid repository state: %s", e) + logger.info("Rejected PR indexing against invalid repository state: %s", e) raise HTTPException(status_code=409, detail=str(e)) except ValueError as e: - logger.warning(f"Invalid request for PR indexing: {e}") + logger.info(f"Invalid request for PR indexing: {e}") raise HTTPException(status_code=400, detail=str(e)) except MutationLeaseUnavailable as e: raise HTTPException(status_code=409, detail=str(e)) @@ -826,6 +826,8 @@ def delete_pr_files( ): """Delete all indexed points for a specific PR.""" index_manager = _get_index_manager() + if not isinstance(collection_target, str) or not collection_target.strip(): + collection_target = None try: with index_manager.pr_overlay_mutation( workspace, @@ -837,13 +839,25 @@ def delete_pr_files( collection_target or index_manager._get_project_collection_name(workspace, project) ) - - if not index_manager._collection_manager.collection_exists(collection_name): + physical_collection = ( + index_manager._collection_manager.resolve_collection_target( + collection_name + ) + ) + if physical_collection is None: return {"status": "skipped", "message": "Collection does not exist"} + # Existing exact generations may predate the PR filter indexes. + # Repair them before the acknowledged filter delete. Keeping + # wait=True is correctness-critical: releasing the same-PR lease + # before Qdrant applies the delete could erase a subsequent rerun. + index_manager._collection_manager.ensure_payload_indexes( + physical_collection + ) + lease.assert_owned() index_manager.qdrant_client.delete( - collection_name=collection_name, + collection_name=physical_collection, points_selector=Filter( must=[ FieldCondition(key="workspace", match=MatchValue(value=workspace)), @@ -851,15 +865,20 @@ def delete_pr_files( FieldCondition(key="pr", match=MatchValue(value=True)), FieldCondition(key="pr_number", match=MatchValue(value=pr_number)), ] - ) + ), + wait=True, ) - logger.info(f"Deleted PR #{pr_number} points from {collection_name}") + logger.info( + "Deleted PR #%s points from %s", + pr_number, + physical_collection, + ) return { "status": "deleted", "pr_number": pr_number, - "collection": collection_name + "collection": physical_collection } except MutationLeaseUnavailable as e: @@ -867,5 +886,5 @@ def delete_pr_files( except MutationCoordinationUnavailable as e: raise HTTPException(status_code=503, detail=str(e)) except Exception as e: - logger.error(f"Error deleting PR files: {e}") + logger.debug("PR file deletion failed: %s", e, exc_info=True) raise HTTPException(status_code=500, detail=str(e)) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py index 562d5b9e..ce72bd8c 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py @@ -287,12 +287,11 @@ def acquire( def is_operation_active(self, token: str) -> bool: if not self.enabled or self._client is None: return False - try: - return bool(self._client.exists(f"codecrow:rag:operation:{token}")) - except Exception: - # Cleanup is auxiliary and must fail open by retaining collections. - logger.warning("Could not verify pending-collection operation %s", token) - return True + # Cleanup is auxiliary and must retain uncertain collections. Let an + # unavailable ownership store escape to the lifecycle janitor instead + # of logging once per candidate; that owner stops the pass and emits + # one transition-bounded outage/recovery diagnostic. + return bool(self._client.exists(f"codecrow:rag:operation:{token}")) def close(self) -> None: if self._client is not None: @@ -322,6 +321,7 @@ def __init__( health_check_interval=30, ) self._disabled_until = 0.0 + self._distributed_unavailable = False self._state_lock = threading.Lock() self._key = "codecrow:rag:openrouter:index:permits" @@ -337,8 +337,8 @@ def permit(self) -> Iterator[None]: if distributed: try: self._client.zrem(self._key, token) - except Exception: - logger.warning("Could not release distributed OpenRouter permit") + except Exception as exception: + self._record_distributed_failure(exception) self._local.release() def _acquire_distributed(self, token: str) -> bool: @@ -346,6 +346,7 @@ def _acquire_distributed(self, token: str) -> bool: if time.monotonic() < self._disabled_until: return False deadline = time.monotonic() + self.acquire_timeout_seconds + wait_reported = False while True: now = time.time() try: @@ -360,23 +361,40 @@ def _acquire_distributed(self, token: str) -> bool: self.permit_seconds, ) if acquired: + self._record_distributed_recovery() return True except Exception as exception: with self._state_lock: self._disabled_until = time.monotonic() + 60 - logger.warning( - "Distributed OpenRouter capacity limit unavailable; " - "using process-local cap for 60s: %s", - exception, - ) + self._record_distributed_failure(exception) return False if time.monotonic() >= deadline: - logger.warning( - "Still waiting for distributed OpenRouter capacity after %.1fs", + log = logger.info if not wait_reported else logger.debug + log( + "Waiting for distributed OpenRouter capacity after %.1fs", self.acquire_timeout_seconds, ) + wait_reported = True deadline = time.monotonic() + self.acquire_timeout_seconds time.sleep(0.05) + def _record_distributed_failure(self, exception: BaseException) -> None: + with self._state_lock: + first_failure = not self._distributed_unavailable + self._distributed_unavailable = True + log = logger.warning if first_failure else logger.debug + log( + "Distributed OpenRouter capacity limit unavailable; using " + "process-local cap: %s", + exception, + ) + + def _record_distributed_recovery(self) -> None: + with self._state_lock: + recovered = self._distributed_unavailable + self._distributed_unavailable = False + if recovered: + logger.info("Distributed OpenRouter capacity limit recovered") + def close(self) -> None: self._client.close() diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py index 75c5977b..5bc2c805 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py @@ -6,6 +6,7 @@ import json import math import re +import uuid from collections.abc import Iterable, Mapping, Sequence from typing import Any @@ -377,3 +378,13 @@ def build_generation_manifest_node( def is_sha256_hex(value: object) -> bool: """Return whether ``value`` is one canonical lower-case SHA-256 digest.""" return isinstance(value, str) and _SHA256_RE.fullmatch(value) is not None + + +def generation_manifest_point_id( + workspace: str, + project: str, + branch: str, +) -> str: + """Return the deterministic storage ID of a repository generation seal.""" + key = f"{workspace}:{project}:{branch}:{GENERATION_MANIFEST_PATH}:0" + return str(uuid.uuid5(uuid.NAMESPACE_DNS, key)) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py index f2a43d84..6c893b5e 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py @@ -7,12 +7,16 @@ import logging import os import re +import threading import time import uuid from typing import Callable, Mapping, Optional, List from qdrant_client import QdrantClient -from qdrant_client.http.exceptions import UnexpectedResponse +from qdrant_client.http.exceptions import ( + ResponseHandlingException, + UnexpectedResponse, +) from qdrant_client.models import ( Distance, VectorParams, CreateAlias, DeleteAlias, CreateAliasOperation, DeleteAliasOperation, @@ -29,6 +33,9 @@ def __init__(self, client: QdrantClient, embedding_dim: int): self.client = client self.embedding_dim = embedding_dim self.vectors_on_disk = os.environ.get("QDRANT_VECTORS_ON_DISK", "true").lower() == "true" + self._payload_indexes_ensured: set[str] = set() + self._payload_indexes_in_progress: set[str] = set() + self._payload_index_condition = threading.Condition() def ensure_collection_exists(self, collection_name: str) -> None: """Ensure Qdrant collection exists with proper configuration. @@ -37,6 +44,9 @@ def ensure_collection_exists(self, collection_name: str) -> None: """ if self.alias_exists(collection_name): logger.info(f"Collection name {collection_name} is an alias, using existing aliased collection") + physical = self.resolve_collection_target(collection_name) + if physical is not None: + self.ensure_payload_indexes(physical) return collections = self.client.get_collections().collections @@ -53,9 +63,10 @@ def ensure_collection_exists(self, collection_name: str) -> None: "Collection %s was created concurrently; using it", collection_name, ) - self._ensure_payload_indexes(collection_name) + self.ensure_payload_indexes(collection_name) else: logger.info(f"Collection {collection_name} already exists") + self.ensure_payload_indexes(collection_name) def create_pending_collection( self, @@ -79,6 +90,8 @@ def create_pending_collection( ) logger.info(f"Creating pending collection: {pending_name}") if self._create_collection(pending_name): + # Pending names are deliberately unique and short lived; do + # not retain every generation in the process-wide cache. self._ensure_payload_indexes(pending_name) return pending_name logger.warning( @@ -115,36 +128,136 @@ def _physical_collection_exists(self, collection_name: str) -> bool: collections = self.client.get_collections().collections return any(collection.name == collection_name for collection in collections) - def _ensure_payload_indexes(self, collection_name: str) -> None: - """Create payload indexes for efficient filtering on common fields.""" - fields = ( - "path", - "branch", - "architecture_paths", - "architecture_group", - "snapshot_plugin", - "snapshot_kind", + @staticmethod + def _payload_index_specs(): + """Fields used by bounded tenant, revision, and PR filters.""" + return ( + ("path", PayloadSchemaType.KEYWORD), + ("branch", PayloadSchemaType.KEYWORD), + ("workspace", PayloadSchemaType.KEYWORD), + ("project", PayloadSchemaType.KEYWORD), + ("commit", PayloadSchemaType.KEYWORD), + ("architecture_paths", PayloadSchemaType.KEYWORD), + ("architecture_group", PayloadSchemaType.KEYWORD), + ("snapshot_plugin", PayloadSchemaType.KEYWORD), + ("snapshot_kind", PayloadSchemaType.KEYWORD), + ("pr", PayloadSchemaType.BOOL), + ("pr_number", PayloadSchemaType.INTEGER), + ("repository_generation_manifest", PayloadSchemaType.BOOL), + ("generation_manifest_sha256", PayloadSchemaType.KEYWORD), ) - for field_name in fields: + + def ensure_payload_indexes(self, collection_name: str) -> None: + """Repair required indexes once per physical collection and process. + + Collections created before a field was introduced are repaired on + first use after restart. Failures remain fail-open for the current + operation and retry on a later request instead of being cached. + """ + with self._payload_index_condition: + while collection_name in self._payload_indexes_in_progress: + self._payload_index_condition.wait() + if collection_name in self._payload_indexes_ensured: + return + self._payload_indexes_in_progress.add(collection_name) + + successful = False + try: + successful = self._ensure_payload_indexes(collection_name) + finally: + with self._payload_index_condition: + self._payload_indexes_in_progress.discard(collection_name) + if successful: + self._payload_indexes_ensured.add(collection_name) + self._payload_index_condition.notify_all() + + def _existing_payload_index_types( + self, + collection_name: str, + ) -> Optional[dict]: + """Read schemas once so existing fields need no write request.""" + try: + payload_schema = getattr( + self.client.get_collection(collection_name), + "payload_schema", + {}, + ) + except Exception as exception: + logger.warning( + "Deferring payload index repair on %s because its schema " + "could not be inspected: %s", + collection_name, + exception, + ) + return None + if not isinstance(payload_schema, Mapping): + return {} + return { + field_name: getattr(index_info, "data_type", index_info) + for field_name, index_info in payload_schema.items() + } + + def _ensure_payload_indexes(self, collection_name: str) -> bool: + """Create payload indexes for efficient filtering on common fields.""" + successful = True + failures = [] + existing_types = self._existing_payload_index_types(collection_name) + if existing_types is None: + return False + for field_name, field_schema in self._payload_index_specs(): + if existing_types.get(field_name) == field_schema: + continue try: self.client.create_payload_index( collection_name=collection_name, field_name=field_name, - field_schema=PayloadSchemaType.KEYWORD, + field_schema=field_schema, + wait=True, ) + except ResponseHandlingException as exception: + successful = False + failures.append((field_name, exception)) + break + except UnexpectedResponse as exception: + successful = False + failures.append((field_name, exception)) + if exception.status_code in {401, 403, 404, 408, 429} or ( + exception.status_code >= 500 + ): + break except Exception as exception: - logger.warning( - "Failed to create payload index %s on %s: %s", - field_name, - collection_name, - exception, - ) - logger.info(f"Payload indexes ensured for {collection_name}") + successful = False + failures.append((field_name, exception)) + if successful: + logger.info("Payload indexes ensured for %s", collection_name) + else: + first_field, first_exception = failures[0] + logger.warning( + "Payload index repair failed for %s field(s) on %s; first " + "failure was %s: %s", + len(failures), + collection_name, + first_field, + first_exception, + ) + logger.info( + "Payload index repair remains incomplete for %s; it will be " + "retried on a later use", + collection_name, + ) + return successful def delete_collection(self, collection_name: str) -> bool: """Delete a collection.""" try: self.client.delete_collection(collection_name) + # A direct/legacy name can be recreated in the same process. Wait + # for a concurrent repair to finish, then invalidate its receipt + # so the replacement collection receives every required index. + with self._payload_index_condition: + while collection_name in self._payload_indexes_in_progress: + self._payload_index_condition.wait() + self._payload_indexes_ensured.discard(collection_name) logger.info(f"Deleted collection: {collection_name}") return True except Exception as e: @@ -338,13 +451,14 @@ def cleanup_expired_pending_collections( int(os.getenv("RAG_PENDING_COLLECTION_MAX_AGE_SECONDS", "21600")), ) now = int(time.time()) - try: - aliased_targets = { - alias.collection_name for alias in self.client.get_aliases().aliases - } - except Exception: - logger.warning("Pending collection janitor could not read aliases") - return 0 + # Alias membership is a safety precondition: an unavailable Qdrant + # response must escape to the lifecycle janitor, which owns the + # transition-bounded outage/recovery diagnostic. Returning zero here + # would make an outage look like a healthy empty cleanup and emit one + # warning on every scheduled pass. + aliased_targets = { + alias.collection_name for alias in self.client.get_aliases().aliases + } pattern = re.compile( r"_pending_(\d{10})_([a-fA-F0-9]{8,32})_[a-fA-F0-9]{8}$" diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py index da808628..05bcddab 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py @@ -432,7 +432,11 @@ def estimate_repository_size( ) is FileDisposition.FULL ] file_count = len(file_list) - logger.info(f"Found {file_count} files for estimation") + logger.info( + "RAG capacity scan found %s repository files " + "(this is not the LLM review scope)", + file_count, + ) if file_count == 0: return 0, 0 @@ -497,6 +501,7 @@ def index_repository( source_tree=None, seal_generation: bool = False, publication_aliases: Optional[List[str]] = None, + reuse_collection_name: Optional[str] = None, operation_id: Optional[str] = None, activation_guard: Optional[Callable[[], None]] = None, progress_callback: Optional[Callable[[dict], None]] = None, @@ -571,6 +576,7 @@ def report_progress( actual_old_collection = None if old_collection_exists: actual_old_collection = self.collection_manager.resolve_alias(alias_name) or alias_name + vector_reuse_collection = reuse_collection_name or actual_old_collection # Get file list repository_file_list = list( @@ -851,7 +857,7 @@ def report_progress( workspace, project, branch, - reuse_collection_name=actual_old_collection, + reuse_collection_name=vector_reuse_collection, operation_id=operation_id, metrics=embedding_metrics, ) @@ -996,7 +1002,7 @@ def report_progress( workspace, project, branch, - reuse_collection_name=actual_old_collection, + reuse_collection_name=vector_reuse_collection, operation_id=operation_id, metrics=embedding_metrics, ) @@ -1751,21 +1757,30 @@ def _apply_change_set( commit=revision, ) - if repository_analysis_plugins: - documents_by_path = { - document.metadata["path"]: document for document in documents - } - missing = sorted( - path for path in active_updated_paths - if path not in documents_by_path + documents_by_path = { + document.metadata["path"]: document for document in documents + } + missing = sorted( + path for path in active_updated_paths + if path not in documents_by_path + ) + if missing: + logger.info( + "Quarantining %s changed repository files that could not " + "be loaded during incremental indexing: %s", + len(missing), + ", ".join(missing[:10]), ) - if missing: - logger.warning( - "Quarantining %s changed repository files that could not " - "be loaded during incremental indexing: %s", - len(missing), - ", ".join(missing[:10]), - ) + + if repository_analysis_plugins: + # A loader omission is not a deletion. Lock files, binary files, + # oversized files, and locally excluded files can all be present + # in the authoritative checkout while intentionally producing no + # Document. Feeding those paths to repository plugins as + # tombstones corrupts their restored state; dereferencing the + # absent document also used to raise a raw KeyError (for example, + # ``composer.lock``). Only successfully loaded updates and + # provider-declared deletions participate in the plugin overlay. artifacts = tuple(sorted(( *( FileArtifact( @@ -1773,10 +1788,11 @@ def _apply_change_set( content=documents_by_path[path].text, ) for path in active_updated_paths + if path in documents_by_path ), *( FileArtifact(path=path, content="", deleted=True) - for path in (*active_deleted_paths, *missing) + for path in active_deleted_paths ), ), key=lambda artifact: artifact.path)) handle = self.plugin_runtime.start_repository_analysis( @@ -1821,8 +1837,21 @@ def _apply_change_set( for node in semantic_nodes: node.metadata.update(identity_metadata) + # Preserve the last usable semantic and repository-graph + # representations for updates which the loader quarantined. In + # copy-on-write generations this also keeps the already copied points + # intact. Real deletions and loaded updates still replace their prior + # representations normally. Repository facts intentionally continue + # to use the authoritative provider change set above. + replacement_paths = sorted({ + *active_deleted_paths, + *( + path for path in active_updated_paths + if path in documents_by_path + ), + }) old_path_records = self._records_for_values( - collection_name, branch, "path", paths + collection_name, branch, "path", replacement_paths ) old_semantic_records = [ record for record in old_path_records @@ -1838,7 +1867,10 @@ def _apply_change_set( if analysis is not None: impacted_old_nodes = self._records_for_values( - collection_name, branch, "architecture_paths", paths + collection_name, + branch, + "architecture_paths", + replacement_paths, ) impacted_old_nodes = [ record for record in impacted_old_nodes @@ -1848,7 +1880,10 @@ def _apply_change_set( architecture_group_from_payload(record.payload or {}) for record in impacted_old_nodes } - groups = old_groups | affected_architecture_groups(analysis, paths) + groups = old_groups | affected_architecture_groups( + analysis, + replacement_paths, + ) group_ids = {architecture_group_id(group) for group in groups} old_group_nodes = self._records_for_values( collection_name, branch, "architecture_group", group_ids @@ -1866,7 +1901,7 @@ def _apply_change_set( self.representation_fingerprint, groups=groups, ) - related_paths = set(paths) + related_paths = set(replacement_paths) for record in old_group_nodes: related_paths.update( (record.payload or {}).get("architecture_paths") or () diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py index ffce9bdc..24b23fc6 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py @@ -63,6 +63,13 @@ def read_repository_revision_preflight(*args, **kwargs): ) +def read_repository_generation_manifest_receipt(*args, **kwargs): + """Patchable boundary around bounded readable-alias verification.""" + return revision_preflight.read_repository_generation_manifest_receipt( + *args, **kwargs + ) + + def _config_int(config, name: str, default: int) -> int: value = getattr(config, name, default) if isinstance(value, bool) or not isinstance(value, (int, str)): @@ -162,6 +169,7 @@ def __init__(self, config: RAGConfig): self.qdrant_client = QdrantClient( url=config.qdrant_url, api_key=config.qdrant_api_key or None, + timeout=_config_int(config, "qdrant_timeout_seconds", 30), ) logger.info(f"Connected to Qdrant at {config.qdrant_url}") @@ -325,6 +333,7 @@ def index_repository( exclude_patterns: Optional[List[str]] = None, source_tree_sha256: Optional[str] = None, collection_target: Optional[str] = None, + reuse_collection_target: Optional[str] = None, publish_branch_alias: bool = False, publish_legacy_project_alias: bool = False, progress_callback: Optional[Callable[[dict], None]] = None, @@ -355,6 +364,18 @@ def index_repository( publish_branch_alias, publish_legacy_project_alias, ) + reuse_collection_name = None + if reuse_collection_target: + reuse_collection_name = ( + self._collection_manager.resolve_collection_target( + reuse_collection_target + ) + ) + if reuse_collection_name is None: + logger.info( + "Prior RAG generation is unavailable for vector reuse; " + "embedding the target snapshot normally" + ) with self._mutation_coordinator.acquire( workspace, project, @@ -376,6 +397,7 @@ def index_repository( source_tree=source_tree, seal_generation=collection_target is not None, publication_aliases=publication_aliases, + reuse_collection_name=reuse_collection_name, operation_id=lease.token, activation_guard=lease.assert_owned, progress_callback=progress_callback, @@ -455,6 +477,7 @@ def publish_generation_aliases( branch: str, commit: str, collection_target: str, + generation_manifest_sha256: Optional[str] = None, publish_branch_alias: bool = True, publish_legacy_project_alias: bool = False, ) -> List[str]: @@ -486,11 +509,14 @@ def publish_generation_aliases( raise IncrementalIndexPreconditionError( "repository generation is unavailable for alias publication" ) - receipt = read_repository_revision_preflight( + receipt = read_repository_generation_manifest_receipt( self.qdrant_client, physical, + workspace, + project, branch, commit, + generation_manifest_sha256, ) if receipt is None or any(receipt.get(key) != value for key, value in ( ("workspace", workspace), @@ -907,11 +933,18 @@ def delete_branch( project: str, branch: str, collection_target: Optional[str] = None, + generation_revision: Optional[str] = None, + generation_manifest_sha256: Optional[str] = None, ) -> bool: """Delete all points for a specific branch from the project collection.""" if collection_target: return self.delete_collection_target( - workspace, project, branch, collection_target + workspace, + project, + branch, + collection_target, + generation_revision, + generation_manifest_sha256, ) with self._mutation_coordinator.acquire( workspace, @@ -921,7 +954,10 @@ def delete_branch( collection_name = self._get_project_collection_name(workspace, project) if not self._collection_manager.collection_exists(collection_name): if not self._collection_manager.alias_exists(collection_name): - logger.warning(f"Collection {collection_name} does not exist") + logger.info( + "Branch cleanup is already complete; collection %s does not exist", + collection_name, + ) return False lease.assert_owned() @@ -933,8 +969,14 @@ def delete_collection_target( project: str, branch: str, collection_target: str, + generation_revision: Optional[str], + generation_manifest_sha256: Optional[str], ) -> bool: - """Delete one exact generation after proving its tenant ownership.""" + """Delete one registry-selected generation after O(1) seal proof.""" + if not generation_revision or not generation_manifest_sha256: + raise IncrementalIndexPreconditionError( + "exact generation deletion requires its revision and manifest receipt" + ) with self._mutation_coordinator.acquire( workspace, project, "delete-generation" ) as lease: @@ -943,32 +985,18 @@ def delete_collection_target( ) if physical is None: return False - offset = None - point_count = 0 - while True: - points, offset = self.qdrant_client.scroll( - collection_name=physical, - limit=256, - offset=offset, - with_payload=["workspace", "project", "branch"], - with_vectors=False, - ) - point_count += len(points) - for point in points: - payload = point.payload or {} - if any(payload.get(key) != value for key, value in ( - ("workspace", workspace), - ("project", project), - ("branch", branch), - )): - raise IncrementalIndexPreconditionError( - "collection target does not belong to the requested tenant branch" - ) - if offset is None: - break - if point_count == 0: + receipt = read_repository_generation_manifest_receipt( + self.qdrant_client, + physical, + workspace, + project, + branch, + generation_revision, + generation_manifest_sha256, + ) + if receipt is None: raise IncrementalIndexPreconditionError( - "empty collection target cannot be ownership-verified" + "collection target does not match the registry generation receipt" ) lease.assert_owned() if self._collection_manager.alias_exists(collection_target): @@ -1064,7 +1092,10 @@ def pr_overlay_mutation( ) def close(self) -> None: - self._mutation_coordinator.close() + try: + self._mutation_coordinator.close() + finally: + self.qdrant_client.close() # Statistics diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py index 96825c99..672c4746 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py @@ -16,6 +16,7 @@ canonical_index_selection_policy, compute_index_selection_policy_sha256, compute_generation_members_digest, + generation_manifest_point_id, generation_manifest_content, is_sha256_hex, verified_generation_member, @@ -305,6 +306,119 @@ def _validate_generation_manifest( } +def read_repository_generation_manifest_receipt( + client, + collection_name: str, + workspace: str, + project: str, + branch: str, + commit: str, + generation_manifest_sha256: str | None = None, +): + """Validate one registry-selected immutable seal without scanning members. + + Readable aliases are non-authoritative operator conveniences. The Java + registry has already accepted the digest returned by the full generation + build, so alias repair only needs to prove that the selected physical + target still contains that exact coordinate-bound manifest. Exact review + preflight continues to verify every member and vector separately. + """ + if ( + generation_manifest_sha256 is not None + and not is_sha256_hex(generation_manifest_sha256) + ): + raise IncrementalIndexPreconditionError( + "repository generation manifest receipt is invalid" + ) + records = client.retrieve( + collection_name=collection_name, + ids=[generation_manifest_point_id(workspace, project, branch)], + with_payload=True, + with_vectors=False, + ) + if len(records) != 1: + return None + payload = records[0].payload or {} + stored_manifest_sha256 = payload.get("generation_manifest_sha256") + if any(payload.get(key) != value for key, value in ( + ("workspace", workspace), + ("project", project), + ("branch", branch), + ("commit", commit), + )): + return None + if ( + payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is not True + or payload.get("generation_schema") != GENERATION_SCHEMA + or payload.get("path") != GENERATION_MANIFEST_PATH + or not is_sha256_hex(stored_manifest_sha256) + or ( + generation_manifest_sha256 is not None + and stored_manifest_sha256 != generation_manifest_sha256 + ) + ): + return None + + # Recompute the manifest digest from its complete coordinate/membership + # metadata. This remains O(1): member contents are intentionally not read + # on the optional alias-repair path. + expected_count = payload.get("generation_member_count") + members_sha256 = payload.get("generation_members_sha256") + source_tree_sha256 = payload.get("source_tree_sha256") + include_patterns = payload.get("index_include_patterns") + exclude_patterns = payload.get("index_exclude_patterns") + selection_digest = payload.get("index_selection_policy_sha256") + if ( + type(expected_count) is not int + or expected_count < 1 + or not is_sha256_hex(members_sha256) + or not is_sha256_hex(source_tree_sha256) + or not isinstance(include_patterns, list) + or not isinstance(exclude_patterns, list) + or not is_sha256_hex(selection_digest) + ): + return None + try: + policy = canonical_index_selection_policy( + include_patterns, + exclude_patterns, + ) + except GenerationManifestError: + return None + if ( + include_patterns != policy["includePatterns"] + or exclude_patterns != policy["excludePatterns"] + or selection_digest != compute_index_selection_policy_sha256( + include_patterns, + exclude_patterns, + ) + ): + return None + content = generation_manifest_content( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=expected_count, + members_sha256=members_sha256, + source_tree_sha256=source_tree_sha256, + index_include_patterns=include_patterns, + index_exclude_patterns=exclude_patterns, + index_selection_policy_sha256=selection_digest, + ) + if hashlib.sha256(content.encode("utf-8")).hexdigest() != stored_manifest_sha256: + return None + if payload.get("text", payload.get("_node_content")) != content: + return None + return { + "workspace": workspace, + "project": project, + "branch": branch, + "commit": commit, + "generation_manifest_sha256": stored_manifest_sha256, + } + + def _require_unmixed_branch_revision( client, collection_name: str, diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py index 163b39aa..7b886e29 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py @@ -63,6 +63,10 @@ class RAGConfig(BaseModel): qdrant_url: str = Field(default_factory=lambda: os.getenv("QDRANT_URL", "http://qdrant:6333")) qdrant_api_key: str = Field(default_factory=lambda: os.getenv("QDRANT_API_KEY", "")) qdrant_collection_prefix: str = Field(default_factory=lambda: os.getenv("QDRANT_COLLECTION_PREFIX", "codecrow")) + qdrant_timeout_seconds: int = Field( + default_factory=lambda: int(os.getenv("QDRANT_TIMEOUT_SECONDS", "30")), + ge=1, + ) # Embedding provider selection: "ollama" (local) or "openrouter" (cloud) embedding_provider: str = Field(default_factory=lambda: os.getenv("EMBEDDING_PROVIDER", "ollama")) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py index 9926393a..9f23775e 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py @@ -1,8 +1,10 @@ import asyncio +from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor import json import logging import os import shutil +import threading import time from pathlib import Path from typing import Dict, Any, Optional @@ -29,6 +31,11 @@ def __init__(self, index_manager: RAGIndexManager): self.is_running = False self._redis: Optional[redis.Redis] = None self._task: Optional[asyncio.Task] = None + self._job_tasks: set[asyncio.Task] = set() + self._index_workers: set[ConcurrentFuture] = set() + self._index_workers_lock = threading.Lock() + self._queue_read_unavailable = False + self._event_delivery_unavailable = False max_concurrent = int(os.environ.get("MAX_CONCURRENT_RAG_JOBS", "2")) self._job_semaphore = asyncio.Semaphore(max_concurrent) self.heartbeat_seconds = max( @@ -63,6 +70,24 @@ async def stop(self): await self._task except asyncio.CancelledError: pass + + # A dequeued job is already admitted durable work. Let every admitted + # index call finish before API shutdown closes its embedding and + # Qdrant clients. Deployment supervisors may still enforce their + # outer termination deadline, but this service must not manufacture a + # client-use-after-close race itself. + active_jobs = tuple(self._job_tasks) + if active_jobs: + logger.info( + "Waiting for %s admitted RAG indexing jobs before shutdown", + len(active_jobs), + ) + await asyncio.gather(*active_jobs, return_exceptions=True) + + # A job task can be canceled independently while its synchronous + # indexing call is still running. Wait for that admitted durable work + # before the application closes shared embedding and Qdrant clients. + await self._wait_for_index_workers() if self._redis: await self._redis.aclose() @@ -83,21 +108,26 @@ async def _consume_loop(self): break result = await self._redis.brpop([self.job_queue_key], timeout=1) + self._record_queue_read_recovery() if not result: continue queue_name, payload_str = result logger.debug(f"Received raw RAG job payload from {queue_name}") - asyncio.create_task(self._handle_admitted_job(payload_str)) + job_task = asyncio.create_task( + self._handle_admitted_job(payload_str) + ) + self._job_tasks.add(job_task) + job_task.add_done_callback(self._job_tasks.discard) permit_acquired = False except asyncio.CancelledError: break except RedisTimeoutError as error: - logger.warning("Redis RAG queue read timed out; retrying: %s", error) + self._record_queue_read_failure(error) await asyncio.sleep(1) except Exception as e: - logger.error(f"Error in RAG Queue consume loop: {e}", exc_info=True) + self._record_queue_read_failure(e) await asyncio.sleep(2) finally: if permit_acquired: @@ -119,7 +149,11 @@ async def _handle_job(self, payload_str: str): """Process a single RAG job popped from the queue.""" job_id = "UNKNOWN" event_queue_key = None - indexing_future = None + index_worker_future = None + index_executor = None + progress_queue = None + progress_publisher_task = None + progress_delivery_available = False try: payload = json.loads(payload_str) @@ -158,63 +192,115 @@ async def _handle_job(self, payload_str: str): # index_manager.index_repository is synchronous, so we run it in an executor loop = asyncio.get_running_loop() progress_delivery_available = True - progress_futures = [] + # Progress is observability, so cap producer pressure and retain + # only the latest pending status when Redis publication is slower + # than indexing. + progress_queue = asyncio.Queue(maxsize=1) + + async def publish_progress_events() -> None: + while True: + event = await progress_queue.get() + try: + if event is None: + return + await self._publish_event(event_queue_key, event) + finally: + progress_queue.task_done() + + progress_publisher_task = asyncio.create_task( + publish_progress_events() + ) def publish_progress(event: Dict[str, Any]) -> None: - nonlocal progress_delivery_available if not progress_delivery_available: return payload = {"type": "status", **event} - future = asyncio.run_coroutine_threadsafe( - self._publish_event(event_queue_key, payload), - loop, - ) - # Never block the indexing executor waiting for its own event - # loop. Drain these publications before the terminal event so - # ordering is retained and delivery remains fail-open. - progress_futures.append(future) + try: + # Transfer plain data to the event-loop thread. Creating + # cross-loop coroutine futures here used to retain the + # default executor thread after a job completed and could + # hang process/test-loop shutdown. + loop.call_soon_threadsafe( + self._coalesce_progress_event, + progress_queue, + payload, + ) + except RuntimeError: + # The service is shutting down. Progress is auxiliary and + # must never keep the indexing worker alive. + return - indexing_future = loop.run_in_executor( - None, - lambda: self.index_manager.index_repository( - repo_path=request_dto.repo_path, - workspace=request_dto.workspace, - project=request_dto.project, - branch=request_dto.branch, - commit=request_dto.commit, - source_tree_sha256=request_dto.source_tree_sha256, - preserve_other_branches=request_dto.preserve_other_branches, - include_patterns=request_dto.include_patterns, - exclude_patterns=request_dto.exclude_patterns, - collection_target=request_dto.collection_target, - progress_callback=publish_progress, - ) + # Keep long-lived indexing work out of asyncio's process-wide + # default executor. A per-job executor has an explicit lifecycle, + # so completed progress jobs cannot leave idle threads attached to + # an event loop and block graceful shutdown. + index_executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="rag-index", ) - while True: - done, _ = await asyncio.wait( - {indexing_future}, - timeout=self.heartbeat_seconds, - return_when=asyncio.FIRST_COMPLETED, - ) - if done: - break - await self._publish_event(event_queue_key, { - "type": "status", - "state": "processing", - "message": "RAG indexing is still processing", - }) - result_obj = await indexing_future - for progress_future in progress_futures: + def run_index_job(): try: - await asyncio.wrap_future(progress_future) - except Exception as exception: - progress_delivery_available = False - logger.warning( - "Could not publish RAG progress for job %s: %s", - job_id, - exception, + return self.index_manager.index_repository( + repo_path=request_dto.repo_path, + workspace=request_dto.workspace, + project=request_dto.project, + branch=request_dto.branch, + commit=request_dto.commit, + source_tree_sha256=request_dto.source_tree_sha256, + preserve_other_branches=request_dto.preserve_other_branches, + include_patterns=request_dto.include_patterns, + exclude_patterns=request_dto.exclude_patterns, + collection_target=request_dto.collection_target, + progress_callback=publish_progress, ) + finally: + # Cleanup belongs to the synchronous worker lifecycle, so + # cancellation can never delete a checkout while that same + # worker is still indexing it. + if request_dto.cleanup_repo_path: + self._cleanup_owned_repository_path( + request_dto.repo_path + ) + + index_worker_future = index_executor.submit(run_index_job) + self._track_index_worker(index_worker_future) + next_heartbeat = loop.time() + self.heartbeat_seconds + while not index_worker_future.done(): + # Poll the thread-safe concurrent future rather than relying + # on a worker-thread callback to wake the asyncio selector. + # The latter can be retained until a long heartbeat timeout + # on some Python/event-loop combinations during shutdown. + await asyncio.sleep( + min(0.1, max(0.0, next_heartbeat - loop.time())) + ) + if ( + not index_worker_future.done() + and loop.time() >= next_heartbeat + ): + await self._publish_event(event_queue_key, { + "type": "status", + "state": "processing", + "message": "RAG indexing is still processing", + }) + next_heartbeat = loop.time() + self.heartbeat_seconds + + try: + result_obj = index_worker_future.result() + finally: + # The worker has returned, so all progress callbacks have + # already been enqueued ahead of this sentinel. Drain them + # before the final/error event to preserve event ordering. + progress_delivery_available = False + # A callback queued from the worker can be behind this task's + # timer wakeup in the loop-ready queue. Yield once so every + # already-scheduled transfer reaches the bounded queue before + # it is joined and closed. + await asyncio.sleep(0) + await progress_queue.join() + progress_queue.put_nowait(None) + await progress_publisher_task + progress_publisher_task = None # Serialize the IndexStats result to a dictionary result = result_obj.dict() if hasattr(result_obj, "dict") else result_obj.model_dump() @@ -237,28 +323,140 @@ def publish_progress(event: Dict[str, Any]) -> None: "message": f"Internal RAG pipeline error: {str(e)}" }) finally: - if "request_dto" in locals() and request_dto.cleanup_repo_path: - if indexing_future is None or indexing_future.done(): - await asyncio.to_thread( - self._cleanup_owned_repository_path, - request_dto.repo_path, - ) + progress_delivery_available = False + if progress_publisher_task is not None: + if index_worker_future is not None and not index_worker_future.done(): + progress_publisher_task.cancel() + try: + await progress_publisher_task + except asyncio.CancelledError: + pass else: - logger.warning( - "Preserving RAG job workspace because indexing is still active: %s", - request_dto.repo_path, - ) + await asyncio.sleep(0) + await progress_queue.join() + progress_queue.put_nowait(None) + await progress_publisher_task + worker_finished = ( + index_worker_future is None or index_worker_future.done() + ) + if ( + not worker_finished + and "request_dto" in locals() + and request_dto.cleanup_repo_path + ): + logger.info( + "Deferring owned RAG workspace cleanup until active " + "indexing returns: %s", + request_dto.repo_path, + ) + if index_executor is not None: + # wait=False still marks the executor closed; a live + # synchronous call exits its worker as soon as it returns. + # Completed calls are joined before the job task returns. + index_executor.shutdown( + wait=worker_finished, + cancel_futures=False, + ) + + def _track_index_worker(self, worker_future: ConcurrentFuture) -> None: + """Track synchronous indexing independently of its asyncio job task.""" + with self._index_workers_lock: + self._index_workers.add(worker_future) + worker_future.add_done_callback(self._retire_index_worker) + + def _retire_index_worker(self, worker_future: ConcurrentFuture) -> None: + with self._index_workers_lock: + self._index_workers.discard(worker_future) + + async def _wait_for_index_workers(self) -> None: + announced = False + while True: + with self._index_workers_lock: + active_count = len(self._index_workers) + if active_count == 0: + return + if not announced: + logger.info( + "Waiting for %s active RAG indexing workers before shutdown", + active_count, + ) + announced = True + await asyncio.sleep(0.01) + + @staticmethod + def _coalesce_progress_event( + progress_queue: asyncio.Queue, + event: Optional[Dict[str, Any]], + ) -> None: + """Keep at most the latest pending progress event or sentinel.""" + if progress_queue.full(): + try: + pending = progress_queue.get_nowait() + progress_queue.task_done() + if pending is None: + # Once the publisher is closing, late auxiliary progress + # must not replace its sentinel and strand the task. + progress_queue.put_nowait(None) + return + except asyncio.QueueEmpty: + pass + progress_queue.put_nowait(event) - async def _publish_event(self, key: str, event: Dict[str, Any]): + async def _publish_event(self, key: str, event: Dict[str, Any]) -> bool: """Publish an event back to the job's specific event list. LPUSH (Java uses rightPop).""" + if not self._redis: + return False try: - if not self._redis: - return event_json = json.dumps(event) - await self._redis.lpush(key, event_json) - await self._redis.expire(key, self.event_ttl_seconds) + # The event and its retention policy are one Redis transaction. + # A process/network failure cannot commit LPUSH while losing the + # EXPIRE and leave an orphaned event list indefinitely. + async with self._redis.pipeline(transaction=True) as pipeline: + pipeline.lpush(key, event_json) + pipeline.expire(key, self.event_ttl_seconds) + await pipeline.execute() + if self._event_delivery_unavailable: + logger.info("Redis RAG event delivery recovered") + self._event_delivery_unavailable = False + return True except Exception as e: - logger.error(f"Failed to publish event to {key}: {e}") + event_type = event.get("type", "unknown") + if not self._event_delivery_unavailable: + logger.warning( + "Redis RAG event delivery unavailable; events remain " + "fail-open (key=%s type=%s): %s", + key, + event_type, + e, + ) + self._event_delivery_unavailable = True + else: + logger.debug( + "Redis RAG event delivery still unavailable " + "(key=%s type=%s): %s", + key, + event_type, + e, + ) + return False + + def _record_queue_read_failure(self, error: BaseException) -> None: + if not self._queue_read_unavailable: + logger.warning( + "Redis RAG queue read unavailable; retrying: %s", + error, + ) + self._queue_read_unavailable = True + else: + logger.debug( + "Redis RAG queue read still unavailable; retrying: %s", + error, + ) + + def _record_queue_read_recovery(self) -> None: + if self._queue_read_unavailable: + logger.info("Redis RAG queue read recovered") + self._queue_read_unavailable = False @staticmethod def _cleanup_owned_repository_path(repo_path: str) -> None: diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py index a2c8b192..bc4b6a33 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py @@ -41,9 +41,15 @@ def __init__(self, config: RAGConfig, plugin_catalog=None): index_representation_fingerprint(config) ) self._observed_branch_cache: set[tuple[str, str]] = set() + qdrant_timeout = getattr(config, "qdrant_timeout_seconds", 30) + if isinstance(qdrant_timeout, bool) or not isinstance( + qdrant_timeout, int + ): + qdrant_timeout = 30 self.qdrant_client = QdrantClient( url=config.qdrant_url, api_key=config.qdrant_api_key or None, + timeout=qdrant_timeout, ) embed_info = get_embedding_model_info(config) @@ -82,19 +88,30 @@ def _accept_stored_points(self, points: List[Any]) -> List[Any]: def _collection_or_alias_exists(self, name: str) -> bool: """Check if a collection or alias with the given name exists.""" try: - collections = [c.name for c in self.qdrant_client.get_collections().collections] + collections = [ + c.name for c in self.qdrant_client.get_collections().collections + ] if name in collections: return True aliases = self.qdrant_client.get_aliases() - if any(a.alias_name == name for a in aliases.aliases): - return True - - return False - except Exception as e: - logger.warning(f"Error checking collection/alias existence: {e}") + return any(a.alias_name == name for a in aliases.aliases) + except Exception as exception: + # Exact callers convert absence to their revision-bound 409; legacy + # optional callers return empty context. The HTTP owner records the + # one contextual degraded diagnostic, so this shared probe stays + # below warning level. + logger.debug( + "Collection/alias probe unavailable for %s: %s", + name, + exception, + ) return False + def close(self) -> None: + """Release the query-side Qdrant transport.""" + self.qdrant_client.close() + def _get_project_collection_name(self, workspace: str, project: str) -> str: """Generate the legacy shared collection name for a project.""" namespace = make_project_namespace(workspace, project) diff --git a/python-ecosystem/rag-pipeline/tests/test_api_app.py b/python-ecosystem/rag-pipeline/tests/test_api_app.py index 9c51a015..0aa54a7e 100644 --- a/python-ecosystem/rag-pipeline/tests/test_api_app.py +++ b/python-ecosystem/rag-pipeline/tests/test_api_app.py @@ -1,9 +1,11 @@ """ Tests for rag_pipeline.api.api — App creation, middleware, lifespan. """ +import asyncio import logging import os import pytest +from types import SimpleNamespace from unittest.mock import patch, MagicMock, AsyncMock @@ -108,6 +110,62 @@ def test_app_exists(self): assert app is not None assert app.title == "CodeCrow RAG API" + @pytest.mark.asyncio + async def test_shutdown_drains_http_index_workers_before_clients_close(self): + import rag_pipeline.api.api as api_module + + order = [] + manager = MagicMock() + manager.embed_model.close.side_effect = lambda: order.append( + "manager-embed-close" + ) + manager.close.side_effect = lambda: order.append("manager-close") + query_service = MagicMock() + query_service.embed_model.close.side_effect = lambda: order.append( + "query-embed-close" + ) + query_service.close.side_effect = lambda: order.append("query-close") + queue_consumer = MagicMock() + queue_consumer.start = AsyncMock() + queue_consumer.stop = AsyncMock() + drain_workers = AsyncMock(side_effect=lambda: order.append("drain")) + test_app = SimpleNamespace(state=SimpleNamespace()) + + with ( + patch.object(api_module, "RAGConfig", return_value=MagicMock()), + patch.object( + api_module, + "RAGIndexManager", + return_value=manager, + ), + patch.object( + api_module, + "RAGQueryService", + return_value=query_service, + ), + patch( + "rag_pipeline.server.rag_queue_consumer.RAGQueueConsumer", + return_value=queue_consumer, + ), + patch( + "rag_pipeline.api.routers.index." + "drain_index_repository_stream_workers", + drain_workers, + ), + ): + async with api_module.lifespan(test_app): + pass + + queue_consumer.stop.assert_awaited_once() + drain_workers.assert_awaited_once() + assert order == [ + "drain", + "manager-embed-close", + "query-embed-close", + "query-close", + "manager-close", + ] + class TestPendingCollectionJanitor: @@ -137,3 +195,42 @@ def test_malformed_interval_falls_back_to_default(self, caplog): assert _pending_janitor_interval_seconds() == 3600 assert "Invalid RAG_PENDING_JANITOR_INTERVAL_SECONDS" in caplog.text + + @pytest.mark.asyncio + async def test_outage_logs_once_and_reports_recovery(self, caplog): + from rag_pipeline.api.api import _pending_collection_janitor + + cleanup_attempt = AsyncMock(side_effect=[ + RuntimeError("qdrant unavailable"), + RuntimeError("qdrant unavailable"), + 0, + asyncio.CancelledError(), + ]) + with ( + patch( + "rag_pipeline.api.api.asyncio.to_thread", + cleanup_attempt, + ), + patch( + "rag_pipeline.api.api.asyncio.sleep", + AsyncMock(return_value=None), + ), + caplog.at_level(logging.DEBUG), + ): + with pytest.raises(asyncio.CancelledError): + await _pending_collection_janitor(MagicMock()) + + janitor_records = [ + record + for record in caplog.records + if "Pending collection janitor" in record.getMessage() + ] + assert sum( + record.levelno == logging.WARNING for record in janitor_records + ) == 1 + assert not any( + record.levelno >= logging.ERROR for record in janitor_records + ) + assert any( + "recovered" in record.getMessage() for record in janitor_records + ) diff --git a/python-ecosystem/rag-pipeline/tests/test_api_models.py b/python-ecosystem/rag-pipeline/tests/test_api_models.py index 330ecdcf..bddc31d5 100644 --- a/python-ecosystem/rag-pipeline/tests/test_api_models.py +++ b/python-ecosystem/rag-pipeline/tests/test_api_models.py @@ -22,6 +22,7 @@ ApplyChangesRequest, UpdateFilesRequest, CleanupStaleBranchesRequest, + GenerationAliasPublicationRequest, VectorGraphRequest, VectorNodeRequest, ) @@ -41,6 +42,7 @@ def test_valid_path(self): assert req.workspace == "ws" assert req.preserve_other_branches is False assert req.cleanup_repo_path is False + assert req.transfer_repo_ownership is False @patch.dict(os.environ, {"ALLOWED_REPO_ROOT": "/tmp"}) def test_other_branch_preservation_requires_explicit_opt_in(self): @@ -66,6 +68,18 @@ def test_queue_consumer_cleanup_requires_explicit_opt_in(self): ) assert req.cleanup_repo_path is True + @patch.dict(os.environ, {"ALLOWED_REPO_ROOT": "/tmp"}) + def test_stream_repository_ownership_requires_explicit_opt_in(self): + req = IndexRequest( + repo_path="/tmp/codecrow-rag-branch-generation-owned", + workspace="ws", + project="proj", + branch="main", + commit="abc123", + transfer_repo_ownership=True, + ) + assert req.transfer_repo_ownership is True + @patch.dict(os.environ, {"ALLOWED_REPO_ROOT": "/tmp"}) def test_path_traversal_rejected(self): with pytest.raises(ValueError, match="Path must be under"): @@ -294,6 +308,39 @@ def test_construction(self): assert req.branch == "feature/old" +class TestGenerationAliasPublicationRequest: + + def test_accepts_legacy_caller_without_registry_manifest_receipt(self): + legacy_request = GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="target", + ) + assert legacy_request.generation_manifest_sha256 is None + + request = GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="target", + generation_manifest_sha256="b" * 64, + ) + assert request.generation_manifest_sha256 == "b" * 64 + + with pytest.raises(ValueError): + GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="target", + generation_manifest_sha256="invalid", + ) + + class TestCleanupStaleBranches: def test_requires_authoritative_branches(self): diff --git a/python-ecosystem/rag-pipeline/tests/test_config.py b/python-ecosystem/rag-pipeline/tests/test_config.py index 0164d8a0..b3d4be90 100644 --- a/python-ecosystem/rag-pipeline/tests/test_config.py +++ b/python-ecosystem/rag-pipeline/tests/test_config.py @@ -64,6 +64,11 @@ def test_default_values(self): assert config.retrieval_top_k == 10 assert config.similarity_threshold == 0.7 assert config.max_file_size_bytes == 1024 * 1024 + assert config.qdrant_timeout_seconds == 30 + + def test_qdrant_timeout_is_configurable(self): + with patch.dict(os.environ, {"QDRANT_TIMEOUT_SECONDS": "45"}): + assert RAGConfig().qdrant_timeout_seconds == 45 def test_auto_detect_embedding_dim_ollama(self): config = RAGConfig(embedding_provider="ollama", ollama_model="all-minilm", embedding_dim=0) diff --git a/python-ecosystem/rag-pipeline/tests/test_coordination.py b/python-ecosystem/rag-pipeline/tests/test_coordination.py index 2d560346..816b460d 100644 --- a/python-ecosystem/rag-pipeline/tests/test_coordination.py +++ b/python-ecosystem/rag-pipeline/tests/test_coordination.py @@ -1,4 +1,5 @@ from types import SimpleNamespace +import logging from unittest.mock import MagicMock, patch import pytest @@ -137,6 +138,40 @@ def test_openrouter_capacity_limiter_falls_back_locally_when_redis_is_unavailabl pool._local.release() +def test_openrouter_capacity_outage_logs_only_transitions(caplog): + pool = RedisPermitPool( + "redis://unused", + 2, + permit_seconds=60, + acquire_timeout_seconds=0.1, + ) + pool._client = MagicMock() + pool._client.eval.side_effect = [ + RuntimeError("redis unavailable"), + RuntimeError("redis unavailable"), + True, + ] + + with caplog.at_level(logging.DEBUG): + pool._disabled_until = 0 + with pool.permit(): + pass + pool._disabled_until = 0 + with pool.permit(): + pass + pool._disabled_until = 0 + with pool.permit(): + pass + + warnings = [ + record for record in caplog.records + if record.levelno == logging.WARNING + and "capacity limit unavailable" in record.getMessage() + ] + assert len(warnings) == 1 + assert "capacity limit recovered" in caplog.text + + def test_pending_janitor_keeps_live_and_aliased_collections_and_deletes_expired(): client = MagicMock() client.get_aliases.return_value.aliases = [ @@ -166,3 +201,26 @@ def test_pending_janitor_keeps_live_and_aliased_collections_and_deletes_expired( client.delete_collection.assert_called_once_with( "base_pending_1000000000_eeeeeeee_ffffffff" ) + + +def test_pending_janitor_propagates_alias_read_failure_to_lifecycle_owner(): + client = MagicMock() + client.get_aliases.side_effect = RuntimeError("qdrant unavailable") + manager = CollectionManager(client, 3) + + with pytest.raises(RuntimeError, match="qdrant unavailable"): + manager.cleanup_expired_pending_collections( + is_operation_active=lambda _token: False, + min_age_seconds=300, + ) + + client.get_collections.assert_not_called() + client.delete_collection.assert_not_called() + + +def test_pending_janitor_operation_check_propagates_redis_failure(): + coordinator = _coordinator() + coordinator._client.exists.side_effect = RuntimeError("redis unavailable") + + with pytest.raises(RuntimeError, match="redis unavailable"): + coordinator.is_operation_active("aaaaaaaa") diff --git a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py index f7c22c9e..f57d08f9 100644 --- a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py +++ b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py @@ -17,6 +17,7 @@ build_generation_manifest_node, collect_generation_members, compute_generation_members_digest, + generation_manifest_point_id, ) from rag_pipeline.core.index_manager.collection_manager import CollectionManager from rag_pipeline.core.index_manager.manager import RAGIndexManager @@ -123,6 +124,16 @@ def _sealed_source(client, point_ops, collection): ) == (1, 0) +def _manifest_digest(client, collection): + records = client.retrieve( + collection_name=collection, + ids=[generation_manifest_point_id("ws", "project", "develop")], + with_payload=True, + with_vectors=False, + ) + return records[0].payload["generation_manifest_sha256"] + + def test_copy_on_write_advance_keeps_source_and_seals_target_revision(tmp_path): client = QdrantClient(":memory:") source_physical = "develop_source_physical" @@ -275,10 +286,57 @@ def test_exact_generation_delete_verifies_tenant_coordinates(): with pytest.raises( IncrementalIndexPreconditionError, - match="does not belong", + match="does not match the registry generation receipt", ): manager.delete_collection_target( - "ws", "project", "develop", "foreign_generation" + "ws", + "project", + "develop", + "foreign_generation", + SOURCE_COMMIT, + "1" * 64, ) assert manager._collection_manager.collection_exists("foreign_generation") + + +def test_exact_generation_delete_allows_tenant_owned_pr_overlay_points(): + client = QdrantClient(":memory:") + client.create_collection( + collection_name="review_generation", + vectors_config=VectorParams(size=4, distance=Distance.COSINE), + ) + point_ops = PointOperations(client, _Embedding(), embedding_dim=4) + _sealed_source(client, point_ops, "review_generation") + point_ops.process_and_upsert_chunks( + [TextNode(text="pull request change", metadata={ + "workspace": "ws", + "project": "project", + "branch": "feature/review", + "path": "src/Changed.java", + "pr": True, + "pr_number": 42, + })], + "review_generation", + "ws", + "project", + "feature/review", + ) + manifest_digest = _manifest_digest(client, "review_generation") + client.scroll = MagicMock( + side_effect=AssertionError("exact generation deletion must remain O(1)") + ) + manager = object.__new__(RAGIndexManager) + manager.qdrant_client = client + manager._collection_manager = CollectionManager(client, 4) + manager._mutation_coordinator = _Coordinator() + + assert manager.delete_collection_target( + "ws", + "project", + "develop", + "review_generation", + SOURCE_COMMIT, + manifest_digest, + ) is True + assert not manager._collection_manager.collection_exists("review_generation") diff --git a/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py b/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py index 7473cdff..7ff852d8 100644 --- a/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py +++ b/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py @@ -7,7 +7,15 @@ from qdrant_client import QdrantClient from qdrant_client.models import Distance, VectorParams -from codecrow_plugins import FileArtifact, PluginRuntime, ProjectSelector, build_repository_facts +from codecrow_plugins import ( + FileArtifact, + FileDisposition, + PluginRuntime, + ProjectSelector, + RepositoryAnalysis, + RepositorySnapshot, + build_repository_facts, +) from codecrow_plugins.bootstrap import discover_builtin_plugins from rag_pipeline.core.index_manager.collection_manager import CollectionManager from rag_pipeline.core.index_manager.indexer import FileOperations, RepositoryIndexer @@ -686,6 +694,100 @@ def test_incremental_update_rejects_missing_repository_analysis_snapshots( point_ops.prepare_chunks_for_embedding.assert_not_called() +def test_excluded_composer_lock_update_is_quarantined_not_deleted( + tmp_path, + monkeypatch, + caplog, +): + """A present excluded lockfile must not become a plugin tombstone.""" + _write_repository(tmp_path, "Acme\\Checkout\\Model\\Cart") + lock_path = tmp_path / "composer.lock" + lock_path.write_text('{"packages": []}', encoding="utf-8") + catalog = discover_builtin_plugins() + selector = ProjectSelector(catalog.registry) + facts = _repository_facts(tmp_path, "base", catalog) + capabilities = selector.select(facts) + implementation_fingerprint = catalog.implementation_fingerprint( + capabilities.repository_plugins + ) + + from rag_pipeline.core import repository_overlay + + monkeypatch.setattr( + repository_overlay, + "load_repository_facts", + lambda *_args: ( + facts, + capabilities.repository_plugins, + capabilities.fingerprint, + capabilities.descriptor_fingerprint, + implementation_fingerprint, + ), + ) + monkeypatch.setattr( + repository_overlay, + "load_repository_snapshots", + lambda *_args: ( + (RepositorySnapshot("magento", "state", "{}"),), + capabilities.repository_plugins, + capabilities.fingerprint, + capabilities.descriptor_fingerprint, + implementation_fingerprint, + ), + ) + monkeypatch.setattr( + "rag_pipeline.core.index_manager.indexer.observe_branch_representation", + lambda *_args, **_kwargs: None, + ) + + handle = MagicMock() + handle.finish.return_value = (RepositoryAnalysis(), ()) + runtime = MagicMock() + runtime.repository_analysis_plugins.return_value = ("magento",) + runtime.file_disposition.return_value = FileDisposition.FULL + runtime.start_repository_analysis.return_value = handle + client = MagicMock() + client.scroll.return_value = ([], None) + operations = FileOperations( + client, + MagicMock(), + MagicMock(), + MagicMock(), + MagicMock(), + DocumentLoader(SimpleNamespace( + excluded_patterns=("*.lock",), + max_file_size_bytes=1_000_000, + )), + plugin_catalog=catalog, + plugin_runtime=runtime, + plugin_selector=selector, + ) + operations._records_for_values = MagicMock(return_value=[]) + operations._replace_points = MagicMock(return_value=1) + + operations.apply_changes( + ["composer.lock"], + [], + str(tmp_path), + "ws", + "project", + "main", + "changed", + "repository", + ) + + handle.ingest.assert_called_once_with(()) + assert all( + "composer.lock" not in set(call.args[3]) + for call in operations._records_for_values.call_args_list + ) + operations._replace_points.assert_called_once() + assert not any( + record.levelno >= 30 and "composer.lock" in record.getMessage() + for record in caplog.records + ) + + def test_generic_change_set_accepts_older_build_and_updates_one_generation( tmp_path, ): diff --git a/python-ecosystem/rag-pipeline/tests/test_index_manager.py b/python-ecosystem/rag-pipeline/tests/test_index_manager.py index aba24d6d..9823655b 100644 --- a/python-ecosystem/rag-pipeline/tests/test_index_manager.py +++ b/python-ecosystem/rag-pipeline/tests/test_index_manager.py @@ -5,7 +5,10 @@ import pytest import uuid from httpx import Headers -from qdrant_client.http.exceptions import UnexpectedResponse +from qdrant_client.http.exceptions import ( + ResponseHandlingException, + UnexpectedResponse, +) from unittest.mock import patch, MagicMock, PropertyMock from datetime import datetime @@ -126,20 +129,103 @@ def test_atomic_assign_aliases_replaces_all_requested_aliases_in_one_call(self): ] assert len(operations) == 3 - def test_payload_index_failure_does_not_skip_remaining_indexes(self): + def test_payload_index_failure_does_not_skip_remaining_indexes( + self, + caplog, + ): cm = self._make() cm.client.create_payload_index.side_effect = [ RuntimeError("path index already exists"), - True, - True, - True, - True, - True, + *([True] * 12), ] cm._ensure_payload_indexes("test_coll") - assert cm.client.create_payload_index.call_count == 6 + assert cm.client.create_payload_index.call_count == 13 + warnings = [ + record for record in caplog.records + if record.levelname == "WARNING" + ] + assert len(warnings) == 1 + assert "failed for 1 field(s)" in warnings[0].getMessage() + + def test_payload_index_failures_emit_one_aggregate_warning(self, caplog): + cm = self._make() + cm.client.create_payload_index.side_effect = RuntimeError("unsupported") + + assert cm._ensure_payload_indexes("test_coll") is False + + assert cm.client.create_payload_index.call_count == 13 + warnings = [ + record for record in caplog.records + if record.levelname == "WARNING" + ] + assert len(warnings) == 1 + assert "failed for 13 field(s)" in warnings[0].getMessage() + + def test_payload_index_repair_defers_when_schema_inspection_times_out(self): + cm = self._make() + cm.client.get_collection.side_effect = ResponseHandlingException( + TimeoutError("timed out") + ) + + assert cm._ensure_payload_indexes("test_coll") is False + cm.client.create_payload_index.assert_not_called() + + def test_payload_index_repair_stops_after_transport_failure(self): + cm = self._make() + cm.client.create_payload_index.side_effect = ( + ResponseHandlingException(TimeoutError("timed out")) + ) + + assert cm._ensure_payload_indexes("test_coll") is False + assert cm.client.create_payload_index.call_count == 1 + + def test_existing_collection_repairs_payload_indexes_once(self): + cm = self._make() + collection = MagicMock(name="test_coll") + collection.name = "test_coll" + cm.client.get_collections.return_value.collections = [collection] + cm.alias_exists = MagicMock(return_value=False) + + cm.ensure_collection_exists("test_coll") + cm.ensure_collection_exists("test_coll") + + assert cm.client.create_payload_index.call_count == 13 + + def test_payload_index_repair_only_creates_missing_schemas(self): + from qdrant_client.models import PayloadSchemaType + + cm = self._make() + existing = MagicMock() + existing.data_type = PayloadSchemaType.KEYWORD + cm.client.get_collection.return_value.payload_schema = { + "workspace": existing, + } + + cm._ensure_payload_indexes("test_coll") + + fields = { + call.kwargs["field_name"] + for call in cm.client.create_payload_index.call_args_list + } + assert "workspace" not in fields + assert "project" in fields + + def test_pr_payload_indexes_use_filter_compatible_schemas(self): + from qdrant_client.models import PayloadSchemaType + + cm = self._make() + cm._ensure_payload_indexes("test_coll") + schemas = { + call.kwargs["field_name"]: call.kwargs["field_schema"] + for call in cm.client.create_payload_index.call_args_list + } + + assert schemas["workspace"] is PayloadSchemaType.KEYWORD + assert schemas["project"] is PayloadSchemaType.KEYWORD + assert schemas["pr"] is PayloadSchemaType.BOOL + assert schemas["pr_number"] is PayloadSchemaType.INTEGER def test_delete_collection(self): cm = self._make() @@ -147,6 +233,20 @@ def test_delete_collection(self): assert result is True cm.client.delete_collection.assert_called_once_with("test_coll") + def test_delete_then_recreate_repairs_payload_indexes_again(self): + cm = self._make() + cm._payload_indexes_ensured.add("test_coll") + + assert cm.delete_collection("test_coll") is True + + collection = MagicMock() + collection.name = "test_coll" + cm.client.get_collections.return_value.collections = [collection] + cm.alias_exists = MagicMock(return_value=False) + cm.ensure_collection_exists("test_coll") + + assert cm.client.create_payload_index.call_count == 13 + def test_delete_collection_failure(self): cm = self._make() cm.client.delete_collection.side_effect = Exception("fail") @@ -399,6 +499,23 @@ def test_init(self, MockQdrant, mock_info, mock_create): mgr = RAGIndexManager(self._mock_config()) assert mgr.qdrant_client is not None assert mgr.embed_model is mock_embed + MockQdrant.assert_called_once_with( + url="http://localhost:6333", + api_key=None, + timeout=30, + ) + + def test_close_releases_coordinator_and_qdrant_client(self): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + manager._mutation_coordinator = MagicMock() + manager.qdrant_client = MagicMock() + + manager.close() + + manager._mutation_coordinator.close.assert_called_once_with() + manager.qdrant_client.close.assert_called_once_with() @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") @@ -464,6 +581,51 @@ def test_readable_alias_publication_requires_immutable_generation_target(self): publish_branch_alias=True, ) + @patch( + "rag_pipeline.core.index_manager.manager.verify_repository_source_tree" + ) + def test_exact_snapshot_forwards_resolved_prior_generation_for_vector_reuse( + self, + verify_source_tree, + ): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + manager._collection_manager = MagicMock() + manager._collection_manager.resolve_collection_target.return_value = ( + "prior-generation-physical" + ) + manager._indexer = MagicMock() + manager._indexer.index_repository.return_value = MagicMock() + manager._mutation_coordinator = MagicMock() + lease = MagicMock(token="operation-token") + lease.assert_owned = MagicMock() + manager._mutation_coordinator.acquire.return_value.__enter__.return_value = ( + lease + ) + manager._publication_aliases = MagicMock(return_value=[]) + manager._publication_scope = MagicMock(return_value=None) + source_tree = MagicMock(tree_sha256="f" * 64) + verify_source_tree.return_value = source_tree + + manager.index_repository( + repo_path="/tmp/repository", + workspace="workspace", + project="project", + branch="main", + commit="a" * 40, + source_tree_sha256="f" * 64, + collection_target="new-generation", + reuse_collection_target="prior-generation", + ) + + manager._collection_manager.resolve_collection_target.assert_called_once_with( + "prior-generation" + ) + assert manager._indexer.index_repository.call_args.kwargs[ + "reuse_collection_name" + ] == "prior-generation-physical" + @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") @patch("rag_pipeline.core.index_manager.manager.QdrantClient") diff --git a/python-ecosystem/rag-pipeline/tests/test_rag_queue_consumer.py b/python-ecosystem/rag-pipeline/tests/test_rag_queue_consumer.py index c4152ac2..6469cee4 100644 --- a/python-ecosystem/rag-pipeline/tests/test_rag_queue_consumer.py +++ b/python-ecosystem/rag-pipeline/tests/test_rag_queue_consumer.py @@ -1,6 +1,8 @@ import asyncio import json -from unittest.mock import AsyncMock, Mock, patch +import logging +import threading +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -12,16 +14,37 @@ def model_dump(self): return {"document_count": 1, "chunk_count": 1} +def _redis_with_transactional_pipeline(): + redis_client = AsyncMock() + pipeline = MagicMock() + pipeline.lpush.return_value = pipeline + pipeline.expire.return_value = pipeline + pipeline.execute = AsyncMock(return_value=[1, True]) + pipeline.__aenter__ = AsyncMock(return_value=pipeline) + pipeline.__aexit__ = AsyncMock(return_value=None) + redis_client.pipeline = Mock(return_value=pipeline) + return redis_client, pipeline + + @pytest.mark.asyncio async def test_active_indexing_emits_heartbeats_and_refreshes_event_ttl(tmp_path): owned_repo = tmp_path / "codecrow-rag-owned" owned_repo.mkdir() manager = Mock() + indexing_started = threading.Event() + finish_indexing = threading.Event() + + def index_until_released(**_kwargs): + indexing_started.set() + finish_indexing.wait(timeout=2) + return _Stats() + + manager.index_repository.side_effect = index_until_released consumer = RAGQueueConsumer(manager) consumer.heartbeat_seconds = 0.01 consumer.event_ttl_seconds = 123 - consumer._redis = AsyncMock() + consumer._redis, event_pipeline = _redis_with_transactional_pipeline() payload = json.dumps({ "job_id": "job-1", @@ -35,22 +58,21 @@ async def test_active_indexing_emits_heartbeats_and_refreshes_event_ttl(tmp_path }, }) - loop = asyncio.get_running_loop() - indexing_future = loop.create_future() - with patch.object(loop, "run_in_executor", return_value=indexing_future): - task = asyncio.create_task(consumer._handle_job(payload)) - await asyncio.sleep(0.04) - indexing_future.set_result(_Stats()) - await task + task = asyncio.create_task(consumer._handle_job(payload)) + while not indexing_started.is_set(): + await asyncio.sleep(0) + await asyncio.sleep(0.04) + finish_indexing.set() + await task events = [ json.loads(call.args[1]) - for call in consumer._redis.lpush.await_args_list + for call in event_pipeline.lpush.call_args_list ] assert any(event.get("state") == "processing" for event in events) assert events[-1]["type"] == "final" - assert consumer._redis.expire.await_count == len(events) - consumer._redis.expire.assert_awaited_with( + assert event_pipeline.expire.call_count == len(events) + event_pipeline.expire.assert_called_with( "codecrow:analysis:events:job-1", 123 ) @@ -73,7 +95,7 @@ def index_with_progress(**kwargs): manager.index_repository.side_effect = index_with_progress consumer = RAGQueueConsumer(manager) - consumer._redis = AsyncMock() + consumer._redis, event_pipeline = _redis_with_transactional_pipeline() payload = json.dumps({ "job_id": "job-progress", "request": { @@ -88,9 +110,14 @@ def index_with_progress(**kwargs): await consumer._handle_job(payload) + assert not any( + thread.is_alive() and thread.name.startswith("rag-index") + for thread in threading.enumerate() + ) + events = [ json.loads(call.args[1]) - for call in consumer._redis.lpush.await_args_list + for call in event_pipeline.lpush.call_args_list ] assert any( event.get("stage") == "indexing" and event.get("progress") == 40 @@ -108,7 +135,7 @@ async def test_consumer_removes_only_explicitly_owned_workspace(tmp_path): manager = Mock() manager.index_repository.return_value = _Stats() consumer = RAGQueueConsumer(manager) - consumer._redis = AsyncMock() + consumer._redis, _ = _redis_with_transactional_pipeline() payload = json.dumps({ "job_id": "job-2", @@ -122,20 +149,7 @@ async def test_consumer_removes_only_explicitly_owned_workspace(tmp_path): }, }) - loop = asyncio.get_running_loop() - - def run_inline(_executor, function): - result = loop.create_future() - try: - result.set_result(function()) - except Exception as error: - result.set_exception(error) - return result - - with ( - patch.object(loop, "run_in_executor", side_effect=run_inline), - patch.dict("os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)}), - ): + with patch.dict("os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)}): await consumer._handle_job(payload) assert not owned_repo.exists() @@ -151,6 +165,199 @@ def test_cleanup_refuses_paths_outside_owned_temp_namespace(tmp_path): assert unrelated.exists() +@pytest.mark.asyncio +async def test_cancelled_job_retires_executor_after_worker_returns(tmp_path): + owned_repo = tmp_path / "codecrow-rag-owned" + owned_repo.mkdir() + (owned_repo / "source.py").write_text("value = 1", encoding="utf-8") + started = threading.Event() + release = threading.Event() + + manager = Mock() + + def index_until_released(**_kwargs): + started.set() + release.wait(timeout=2) + return _Stats() + + manager.index_repository.side_effect = index_until_released + consumer = RAGQueueConsumer(manager) + consumer._redis, _ = _redis_with_transactional_pipeline() + payload = json.dumps({ + "job_id": "job-cancelled", + "request": { + "repo_path": str(owned_repo), + "workspace": "ws", + "project": "project", + "branch": "main", + "commit": "abc123", + "cleanup_repo_path": True, + }, + }) + + with patch.dict("os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)}): + task = asyncio.create_task(consumer._handle_job(payload)) + while not started.is_set(): + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert owned_repo.exists() + + stop_task = asyncio.create_task(consumer.stop()) + await asyncio.sleep(0) + assert not stop_task.done() + consumer._redis.aclose.assert_not_awaited() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + + assert not any( + thread.is_alive() and thread.name.startswith("rag-index") + for thread in threading.enumerate() + ) + assert not owned_repo.exists() + + +@pytest.mark.asyncio +async def test_stop_waits_for_admitted_job_before_closing_redis(tmp_path): + owned_repo = tmp_path / "codecrow-rag-owned" + owned_repo.mkdir() + started = threading.Event() + release = threading.Event() + + manager = Mock() + + def index_until_released(**_kwargs): + started.set() + release.wait(timeout=2) + return _Stats() + + manager.index_repository.side_effect = index_until_released + consumer = RAGQueueConsumer(manager) + consumer._redis, _ = _redis_with_transactional_pipeline() + payload = json.dumps({ + "job_id": "job-shutdown", + "request": { + "repo_path": str(owned_repo), + "workspace": "ws", + "project": "project", + "branch": "main", + "commit": "abc123", + "cleanup_repo_path": False, + }, + }) + + job_task = asyncio.create_task(consumer._handle_job(payload)) + consumer._job_tasks.add(job_task) + job_task.add_done_callback(consumer._job_tasks.discard) + while not started.is_set(): + await asyncio.sleep(0) + + stop_task = asyncio.create_task(consumer.stop()) + await asyncio.sleep(0) + assert not stop_task.done() + consumer._redis.aclose.assert_not_awaited() + + release.set() + await asyncio.wait_for(stop_task, timeout=1) + + assert job_task.done() + consumer._redis.aclose.assert_awaited_once_with() + assert not any( + thread.is_alive() and thread.name.startswith("rag-index") + for thread in threading.enumerate() + ) + + +@pytest.mark.asyncio +async def test_progress_buffer_coalesces_to_latest_event(): + queue = asyncio.Queue(maxsize=1) + + RAGQueueConsumer._coalesce_progress_event(queue, {"progress": 10}) + RAGQueueConsumer._coalesce_progress_event(queue, {"progress": 20}) + + assert queue.qsize() == 1 + assert await queue.get() == {"progress": 20} + queue.task_done() + await queue.join() + + +@pytest.mark.asyncio +async def test_late_progress_does_not_replace_publisher_sentinel(): + queue = asyncio.Queue(maxsize=1) + queue.put_nowait(None) + + RAGQueueConsumer._coalesce_progress_event(queue, {"progress": 100}) + + assert await queue.get() is None + queue.task_done() + await queue.join() + + +@pytest.mark.asyncio +async def test_event_delivery_outage_logs_one_warning_until_recovery(caplog): + consumer = RAGQueueConsumer(Mock()) + consumer._redis, event_pipeline = _redis_with_transactional_pipeline() + event_pipeline.execute.side_effect = [ + RuntimeError("redis unavailable"), + RuntimeError("redis unavailable"), + [1, True], + ] + + with caplog.at_level(logging.DEBUG): + assert not await consumer._publish_event("events", {"type": "status"}) + assert not await consumer._publish_event("events", {"type": "status"}) + assert await consumer._publish_event("events", {"type": "final"}) + + outage_records = [ + record + for record in caplog.records + if "Redis RAG event delivery" in record.getMessage() + ] + assert sum(record.levelno == logging.WARNING for record in outage_records) == 1 + assert not any(record.levelno >= logging.ERROR for record in outage_records) + assert any("recovered" in record.getMessage() for record in outage_records) + + +@pytest.mark.asyncio +async def test_event_and_ttl_are_enqueued_in_one_redis_transaction(): + consumer = RAGQueueConsumer(Mock()) + consumer.event_ttl_seconds = 321 + consumer._redis, event_pipeline = _redis_with_transactional_pipeline() + + assert await consumer._publish_event( + "events", {"type": "final", "result": {"ok": True}} + ) + + consumer._redis.pipeline.assert_called_once_with(transaction=True) + event_pipeline.lpush.assert_called_once() + assert json.loads(event_pipeline.lpush.call_args.args[1]) == { + "type": "final", + "result": {"ok": True}, + } + event_pipeline.expire.assert_called_once_with("events", 321) + event_pipeline.execute.assert_awaited_once_with() + + +def test_queue_read_outage_logs_one_warning_until_recovery(caplog): + consumer = RAGQueueConsumer(Mock()) + + with caplog.at_level(logging.DEBUG): + consumer._record_queue_read_failure(RuntimeError("redis unavailable")) + consumer._record_queue_read_failure(RuntimeError("redis unavailable")) + consumer._record_queue_read_recovery() + + outage_records = [ + record + for record in caplog.records + if "Redis RAG queue read" in record.getMessage() + ] + assert sum(record.levelno == logging.WARNING for record in outage_records) == 1 + assert not any(record.levelno >= logging.ERROR for record in outage_records) + assert any("recovered" in record.getMessage() for record in outage_records) + + @pytest.mark.asyncio async def test_worker_capacity_is_reserved_before_rag_job_is_dequeued(): consumer = RAGQueueConsumer(Mock()) diff --git a/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py b/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py index 2283e1fd..8fbbc569 100644 --- a/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py +++ b/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py @@ -13,6 +13,7 @@ ) from rag_pipeline.core.generation_manifest import ( + GENERATION_MANIFEST_PATH, GENERATION_SCHEMA, GenerationManifestError, build_generation_manifest_node, @@ -20,14 +21,17 @@ collect_generation_members, compute_generation_member_digest, compute_generation_members_digest, + generation_manifest_point_id, seal_generation_members, ) from rag_pipeline.core.index_representation import ( INDEX_REPRESENTATION_PAYLOAD_KEY, ) from rag_pipeline.core.index_manager.manager import RAGIndexManager +from rag_pipeline.core.index_manager.point_operations import PointOperations from rag_pipeline.core.repository_overlay import IncrementalIndexPreconditionError from rag_pipeline.core.revision_preflight import ( + read_repository_generation_manifest_receipt, read_repository_revision_preflight, ) @@ -205,6 +209,82 @@ def test_exact_revision_preflight_returns_verified_state_identity(): ) is None +def test_alias_receipt_reads_only_deterministic_manifest_point(): + manifest = _complete_revision_payloads()[-1] + client = MagicMock() + client.retrieve.return_value = [SimpleNamespace(payload=manifest)] + digest = manifest["generation_manifest_sha256"] + + result = read_repository_generation_manifest_receipt( + client, + "physical-generation", + "workspace", + "project", + "main", + COMMIT, + digest, + ) + + assert result["generation_manifest_sha256"] == digest + client.retrieve.assert_called_once_with( + collection_name="physical-generation", + ids=[generation_manifest_point_id("workspace", "project", "main")], + with_payload=True, + with_vectors=False, + ) + client.scroll.assert_not_called() + + +def test_manifest_receipt_id_matches_point_storage_contract(): + assert generation_manifest_point_id( + "workspace", + "project", + "main", + ) == PointOperations.generate_point_id( + "workspace", + "project", + "main", + GENERATION_MANIFEST_PATH, + 0, + ) + + +def test_alias_receipt_rejects_registry_digest_mismatch(): + manifest = _complete_revision_payloads()[-1] + client = MagicMock() + client.retrieve.return_value = [SimpleNamespace(payload=manifest)] + + assert read_repository_generation_manifest_receipt( + client, + "physical-generation", + "workspace", + "project", + "main", + COMMIT, + "f" * 64, + ) is None + + +def test_alias_receipt_accepts_legacy_caller_without_registry_digest(): + manifest = _complete_revision_payloads()[-1] + client = MagicMock() + client.retrieve.return_value = [SimpleNamespace(payload=manifest)] + + receipt = read_repository_generation_manifest_receipt( + client, + "physical-generation", + "workspace", + "project", + "main", + COMMIT, + ) + + assert receipt["generation_manifest_sha256"] == manifest[ + "generation_manifest_sha256" + ] + client.scroll.assert_not_called() + + def test_exact_revision_preflight_rejects_incomplete_repository_state(): content = _facts_content() digest = hashlib.sha256(content.encode("utf-8")).hexdigest() @@ -643,3 +723,63 @@ def test_index_manager_retains_stale_representation_as_provenance(mock_read): ) assert result["index_representation_fingerprint"] == "sha256:stale" + + +@patch( + "rag_pipeline.core.index_manager.manager." + "read_repository_generation_manifest_receipt" +) +@patch( + "rag_pipeline.core.index_manager.manager." + "read_repository_revision_preflight" +) +def test_alias_publication_uses_bounded_registry_receipt( + mock_full_preflight, + mock_manifest_receipt, +): + manager = object.__new__(RAGIndexManager) + manager.config = SimpleNamespace(qdrant_collection_prefix="code") + manager.qdrant_client = MagicMock() + manager._collection_manager = MagicMock() + manager._collection_manager.resolve_collection_target.return_value = ( + "physical-generation" + ) + manager._mutation_coordinator = MagicMock() + lease = SimpleNamespace(assert_owned=MagicMock()) + manager._mutation_coordinator.acquire.return_value.__enter__.return_value = ( + lease + ) + mock_manifest_receipt.return_value = { + "workspace": "workspace", + "project": "project", + "branch": "main", + "commit": COMMIT, + "generation_manifest_sha256": "e" * 64, + } + + aliases = manager.publish_generation_aliases( + "workspace", + "project", + "main", + COMMIT, + "generation-target", + "e" * 64, + publish_branch_alias=True, + publish_legacy_project_alias=False, + ) + + assert aliases == ["code_workspace__project__main"] + mock_manifest_receipt.assert_called_once_with( + manager.qdrant_client, + "physical-generation", + "workspace", + "project", + "main", + COMMIT, + "e" * 64, + ) + mock_full_preflight.assert_not_called() + lease.assert_owned.assert_called_once() + manager._collection_manager.atomic_assign_aliases.assert_called_once_with({ + "code_workspace__project__main": "physical-generation", + }) diff --git a/python-ecosystem/rag-pipeline/tests/test_router_index.py b/python-ecosystem/rag-pipeline/tests/test_router_index.py index 6bd01f88..b65b96f4 100644 --- a/python-ecosystem/rag-pipeline/tests/test_router_index.py +++ b/python-ecosystem/rag-pipeline/tests/test_router_index.py @@ -13,6 +13,12 @@ """ import asyncio import json +import logging +import os +import shutil +import threading +from pathlib import Path +from queue import Queue import pytest from unittest.mock import patch, MagicMock from fastapi import HTTPException @@ -226,6 +232,7 @@ def index_with_progress(**kwargs): req.exclude_patterns = None req.source_tree_sha256 = None req.collection_target = "target" + req.reuse_collection_target = "prior-target" response = index_repository_stream(req) @@ -240,6 +247,285 @@ async def consume(): assert events[0]["estimatedRemainingMs"] == 1200 assert events[1]["type"] == "complete" assert events[1]["result"]["chunk_count"] == 50 + assert im.index_repository.call_args.kwargs[ + "reuse_collection_target" + ] == "prior-target" + + def test_stream_progress_is_bounded_and_keeps_latest_event(self): + from rag_pipeline.api.routers.index import _coalesce_stream_progress + + events = Queue(maxsize=1) + for batch in range(100): + _coalesce_stream_progress(events, {"completedBatches": batch}) + + assert events.qsize() == 1 + assert events.get_nowait() == {"completedBatches": 99} + + def test_orphan_cleanup_removes_only_old_owned_stream_directories( + self, + tmp_path, + ): + from rag_pipeline.api.routers.index import ( + _remove_owned_stream_repository, + _take_stream_repository_ownership, + cleanup_orphaned_index_repository_stream_workspaces, + ) + + old_owned = tmp_path / "codecrow-rag-owned-stream-old" + old_owned.mkdir() + (old_owned / "source.py").write_text("old", encoding="utf-8") + active_source = ( + tmp_path / "codecrow-rag-branch-generation-active" + ) + active_source.mkdir() + unrelated = tmp_path / "codecrow-rag-branch-generation-old" + unrelated.mkdir() + old_mtime = 1_000_000 + os.utime(old_owned, (old_mtime, old_mtime)) + os.utime(unrelated, (old_mtime, old_mtime)) + + with patch.dict( + "os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)} + ): + active_owned, active_lock = _take_stream_repository_ownership( + str(active_source) + ) + os.utime(active_owned, (old_mtime, old_mtime)) + try: + cleaned = ( + cleanup_orphaned_index_repository_stream_workspaces( + max_age_seconds=3600 + ) + ) + active_survived_cleanup = active_owned.exists() + finally: + _remove_owned_stream_repository(active_owned, active_lock) + + assert cleaned == 1 + assert not old_owned.exists() + assert active_survived_cleanup + assert not active_owned.exists() + assert unrelated.exists() + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_stream_cancellation_waits_for_admitted_index_worker( + self, + mock_get, + ): + _, im = _mock_singletons() + started = threading.Event() + release = threading.Event() + finished = threading.Event() + stats = IndexStats( + namespace="ns", document_count=10, chunk_count=50, + last_updated="2024-01-01", workspace="ws", project="proj", + branch="main", + ) + + def blocking_index(**_kwargs): + started.set() + if not release.wait(timeout=5): + raise RuntimeError("test did not release indexing worker") + finished.set() + return stats + + im.index_repository.side_effect = blocking_index + mock_get.return_value = (_, im) + from rag_pipeline.api.routers.index import ( + _index_stream_workers, + index_repository_stream, + ) + + req = MagicMock() + req.repo_path = "/tmp/repo" + req.workspace = "ws" + req.project = "proj" + req.branch = "main" + req.commit = "abc" + req.preserve_other_branches = False + req.include_patterns = None + req.exclude_patterns = None + req.source_tree_sha256 = None + req.collection_target = "target" + response = index_repository_stream(req) + + async def scenario(): + consumer_started = asyncio.Event() + + async def consume(): + consumer_started.set() + async for _item in response.body_iterator: + pass + + consumer = asyncio.create_task(consume()) + await consumer_started.wait() + for _ in range(100): + if started.is_set(): + break + await asyncio.sleep(0.01) + assert started.is_set() + assert _index_stream_workers.active_count == 1 + + consumer.cancel() + await asyncio.sleep(0.1) + assert not consumer.done() + + release.set() + with pytest.raises(asyncio.CancelledError): + await consumer + + assert finished.is_set() + assert _index_stream_workers.active_count == 0 + + try: + asyncio.run(scenario()) + finally: + release.set() + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_disconnect_cannot_delete_transferred_snapshot_under_active_worker( + self, + mock_get, + tmp_path, + ): + _, im = _mock_singletons() + source = tmp_path / "codecrow-rag-branch-generation-disconnect" + source.mkdir() + (source / "source.py").write_text("value = 1", encoding="utf-8") + started = threading.Event() + release = threading.Event() + worker_path: list[Path] = [] + stats = IndexStats( + namespace="ns", document_count=1, chunk_count=1, + last_updated="2024-01-01", workspace="ws", project="proj", + branch="main", + ) + + def blocking_index(**kwargs): + owned_path = Path(kwargs["repo_path"]) + worker_path.append(owned_path) + assert owned_path != source + assert owned_path.exists() + started.set() + if not release.wait(timeout=5): + raise RuntimeError("test did not release indexing worker") + # Java may clean its original path after the stream disconnects, + # but the atomically moved RAG-owned path remains valid. + assert owned_path.exists() + return stats + + im.index_repository.side_effect = blocking_index + mock_get.return_value = (_, im) + from rag_pipeline.api.models import IndexRequest + from rag_pipeline.api.routers.index import ( + _index_stream_workers, + index_repository_stream, + ) + + request = IndexRequest( + repo_path=str(source), + workspace="ws", + project="proj", + branch="main", + commit="abc", + collection_target="target", + transfer_repo_ownership=True, + ) + + async def scenario(): + response = index_repository_stream(request) + admitted = asyncio.Event() + + async def consume(): + async for item in response.body_iterator: + event = json.loads( + (item.decode() if isinstance(item, bytes) else item) + .removeprefix("data: ").strip() + ) + if event["type"] == "admitted": + admitted.set() + + consumer = asyncio.create_task(consume()) + await asyncio.wait_for(admitted.wait(), timeout=1) + for _ in range(100): + if started.is_set(): + break + await asyncio.sleep(0.01) + assert started.is_set() + assert not source.exists() + + consumer.cancel() + # Simulate the Java pre-admission fallback after a lost stream. + # It targets only the old path, which no longer backs the worker. + shutil.rmtree(source, ignore_errors=True) + await asyncio.sleep(0.1) + assert not consumer.done() + assert worker_path[0].exists() + + release.set() + with pytest.raises(asyncio.CancelledError): + await consumer + + assert not worker_path[0].exists() + assert _index_stream_workers.active_count == 0 + + try: + with patch.dict( + "os.environ", {"ALLOWED_REPO_ROOT": str(tmp_path)} + ): + asyncio.run(scenario()) + finally: + release.set() + for path in worker_path: + shutil.rmtree(path, ignore_errors=True) + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_stream_worker_failure_is_terminal_without_duplicate_error_log( + self, + mock_get, + caplog, + ): + _, im = _mock_singletons() + im.index_repository.side_effect = RuntimeError("qdrant unavailable") + mock_get.return_value = (_, im) + from rag_pipeline.api.routers.index import index_repository_stream + + req = MagicMock() + req.repo_path = "/tmp/repo" + req.workspace = "ws" + req.project = "proj" + req.branch = "main" + req.commit = "abc" + req.preserve_other_branches = False + req.include_patterns = None + req.exclude_patterns = None + req.source_tree_sha256 = None + req.collection_target = "target" + req.transfer_repo_ownership = False + + response = index_repository_stream(req) + + async def consume(): + return [item async for item in response.body_iterator] + + with caplog.at_level(logging.DEBUG): + events = [ + json.loads( + (item.decode() if isinstance(item, bytes) else item) + .removeprefix("data: ").strip() + ) + for item in asyncio.run(consume()) + ] + + assert events[-1] == { + "type": "error", + "message": "qdrant unavailable", + } + assert not any( + record.levelno >= logging.ERROR + and "qdrant unavailable" in record.getMessage() + for record in caplog.records + ) @patch("rag_pipeline.api.routers.index._get_singletons") def test_exact_index_forwards_readable_alias_publication(self, mock_get): @@ -258,6 +544,7 @@ def test_exact_index_forwards_readable_alias_publication(self, mock_get): repo_path="/tmp/repo", workspace="ws", project="proj", branch="develop", commit="a" * 40, collection_target="exact-develop-target", + reuse_collection_target="prior-develop-target", publish_branch_alias=True, ) @@ -267,6 +554,9 @@ def test_exact_index_forwards_readable_alias_publication(self, mock_get): "exact-develop-target" ) assert im.index_repository.call_args.kwargs["publish_branch_alias"] is True + assert im.index_repository.call_args.kwargs["reuse_collection_target"] == ( + "prior-develop-target" + ) class TestAdvanceGeneration: @@ -296,6 +586,65 @@ def test_forwards_readable_alias_publication(self, mock_get): assert im.advance_generation.call_args.kwargs["publish_branch_alias"] is True +class TestGenerationAliasPublication: + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_forwards_registry_manifest_receipt(self, mock_get): + _, im = _mock_singletons() + im.publish_generation_aliases.return_value = ["readable-main"] + mock_get.return_value = (_, im) + + from rag_pipeline.api.models import GenerationAliasPublicationRequest + from rag_pipeline.api.routers.index import publish_generation_aliases + + request = GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="exact-target", + generation_manifest_sha256="b" * 64, + ) + + result = publish_generation_aliases(request) + + assert result["status"] == "published" + assert im.publish_generation_aliases.call_args.kwargs[ + "generation_manifest_sha256" + ] == "b" * 64 + + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_retryable_failure_does_not_emit_server_error_log( + self, + mock_get, + caplog, + ): + _, im = _mock_singletons() + im.publish_generation_aliases.side_effect = RuntimeError("timed out") + mock_get.return_value = (_, im) + + from rag_pipeline.api.models import GenerationAliasPublicationRequest + from rag_pipeline.api.routers.index import publish_generation_aliases + + request = GenerationAliasPublicationRequest( + workspace="ws", + project="project", + branch="main", + commit="a" * 40, + collection_target="exact-target", + ) + + with pytest.raises(HTTPException) as exc_info: + publish_generation_aliases(request) + + assert exc_info.value.status_code == 500 + assert not any( + record.levelname == "ERROR" + and "alias" in record.getMessage().lower() + for record in caplog.records + ) + + # ───────────────────────────────────────────────────────────── # update_files / delete_files # ───────────────────────────────────────────────────────────── @@ -494,6 +843,28 @@ def test_delete_branch_not_found(self, mock_get): result = delete_branch("ws", "proj", "feat") assert result["status"] == "not_found" + @patch("rag_pipeline.api.routers.index._get_singletons") + def test_delete_exact_generation_forwards_registry_receipt(self, mock_get): + _, im = _mock_singletons() + im.delete_branch.return_value = True + mock_get.return_value = (_, im) + + from rag_pipeline.api.routers.index import delete_branch + digest = "a" * 64 + result = delete_branch( + "ws", "proj", "develop", "target", "revision", digest + ) + + assert result["status"] == "success" + im.delete_branch.assert_called_once_with( + "ws", + "proj", + "develop", + collection_target="target", + generation_revision="revision", + generation_manifest_sha256=digest, + ) + @patch("rag_pipeline.api.routers.index._get_singletons") def test_list_branches(self, mock_get): _, im = _mock_singletons() diff --git a/python-ecosystem/rag-pipeline/tests/test_router_pr.py b/python-ecosystem/rag-pipeline/tests/test_router_pr.py index 7a1276c2..30bec98b 100644 --- a/python-ecosystem/rag-pipeline/tests/test_router_pr.py +++ b/python-ecosystem/rag-pipeline/tests/test_router_pr.py @@ -38,6 +38,9 @@ def _make_index_manager(): ) im._get_project_collection_name.return_value = "rag_ws__proj" im._collection_manager.collection_exists.return_value = True + im._collection_manager.resolve_collection_target.side_effect = ( + lambda collection_name: collection_name + ) im.splitter.split_documents.return_value = [] im.splitter.split_documents_resilient.side_effect = ( lambda documents, capabilities=None: ( @@ -829,6 +832,10 @@ def test_success(self, mock_get): "pr": True, "pr_number": 42, } + assert im.qdrant_client.delete.call_args.kwargs["wait"] is True + im._collection_manager.ensure_payload_indexes.assert_called_once_with( + "rag_ws__proj" + ) @patch("rag_pipeline.api.routers.pr._get_index_manager") def test_explicit_collection_target_still_uses_tenant_filter(self, mock_get): @@ -852,7 +859,8 @@ def test_explicit_collection_target_still_uses_tenant_filter(self, mock_get): @patch("rag_pipeline.api.routers.pr._get_index_manager") def test_collection_not_found(self, mock_get): im = _make_index_manager() - im._collection_manager.collection_exists.return_value = False + im._collection_manager.resolve_collection_target.return_value = None + im._collection_manager.resolve_collection_target.side_effect = None mock_get.return_value = im from rag_pipeline.api.routers.pr import delete_pr_files diff --git a/python-ecosystem/rag-pipeline/tests/test_services.py b/python-ecosystem/rag-pipeline/tests/test_services.py index ef48a436..3bdec6b1 100644 --- a/python-ecosystem/rag-pipeline/tests/test_services.py +++ b/python-ecosystem/rag-pipeline/tests/test_services.py @@ -42,10 +42,17 @@ def test_init(self, MockQdrant, mock_info, mock_create): base = RAGQueryBase(config) assert base.config is config - MockQdrant.assert_called_once_with(url="http://localhost:6333", api_key=None) + MockQdrant.assert_called_once_with( + url="http://localhost:6333", + api_key=None, + timeout=30, + ) assert base.qdrant_client is not None assert base.embed_model is not None + base.close() + MockQdrant.return_value.close.assert_called_once_with() + @patch("rag_pipeline.services.base.create_embedding_model") @patch("rag_pipeline.services.base.get_embedding_model_info") @patch("rag_pipeline.services.base.QdrantClient") diff --git a/tools/review_quality/neutral_prompt_context_gate.py b/tools/review_quality/neutral_prompt_context_gate.py index fe805346..935fda69 100644 --- a/tools/review_quality/neutral_prompt_context_gate.py +++ b/tools/review_quality/neutral_prompt_context_gate.py @@ -139,6 +139,15 @@ def _request( currentCommitHash=_revision("head", digest), commitHash=_revision("head", digest), baseCommitHash=_revision("base", digest), + # The deterministic adapter models a PR overlay on an immutable base + # generation. Supply the same complete base binding production sends; + # the dry-run facade then returns the matching PR-generation receipts. + # Without these coordinates Stage 1 correctly treats the request as an + # unbound legacy review and must not query PR-scoped vectors. + ragCollectionTarget=f"neutral_{digest}_main_generation", + ragBaseGenerationManifestSha256=hashlib.sha256( + f"base-generation\0{digest}".encode("utf-8") + ).hexdigest(), changedFiles=list(changed), rawDiff=_raw_diff(definition), prTitle=f"Provider-free neutral prompt gate: {definition.case_id}",