From b3913753d5a89784eebc524dbf56f733fff7505a Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Wed, 15 Apr 2026 11:29:47 +0200 Subject: [PATCH 1/2] fix: re-check step outputs on re-encounter to unstick cached builds When create_step encounters an existing step (is_new=false), it now re-checks whether the step's outputs have appeared in the local nix store since the step was first created. Without this check, builds whose outputs became available between poll cycles (e.g. via concurrent builds, substitution, or external upload) would get stuck in an infinite re-load loop: the DB says finished=0, the step already exists in memory so create_build inserts the build but never routes it to handle_cached_build. The fix adds two checks before returning Valid(step): 1. If the step is already marked finished, return None immediately. 2. Query missing outputs from the local store; if all are present, mark the step finished, insert into finished_drvs, and return None so create_build routes to handle_cached_build. Adapted from zw3rk/hydra-queue-runner commit 67ef484. --- .../hydra-queue-runner/src/state/mod.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/subprojects/hydra-queue-runner/src/state/mod.rs b/subprojects/hydra-queue-runner/src/state/mod.rs index c62128fa4d..15b9527a73 100644 --- a/subprojects/hydra-queue-runner/src/state/mod.rs +++ b/subprojects/hydra-queue-runner/src/state/mod.rs @@ -1761,6 +1761,24 @@ impl State { self.steps .create(&drv_path, referring_build.as_ref(), referring_step.as_ref()); if !is_new { + // Re-check whether the step's outputs have appeared in the store + // since it was first created. This handles the case where outputs + // became available between poll cycles (e.g. built by a concurrent + // step, substituted, or uploaded externally). Without this check, + // builds whose outputs are now cached get stuck in an infinite + // re-load loop: the DB says finished=0, the step already exists + // in memory, and create_build never reaches handle_cached_build. + if step.get_finished() { + return CreateStepResult::None; + } + if let Some(output_paths) = step.get_output_paths(self.store.store_dir()) { + let missing = self.store.query_missing_outputs(output_paths).await; + if missing.is_empty() { + finished_drvs.write().insert(drv_path.clone()); + step.set_finished(true); + return CreateStepResult::None; + } + } return CreateStepResult::Valid(step); } self.metrics.queue_steps_created.inc(); From 6b374cf20cc1083b0f7adc5a4829661582b095b8 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Sun, 19 Apr 2026 12:29:04 +0200 Subject: [PATCH 2/2] ci: retrigger (flaky aarch64-linux gRPC connection reset)