From 9bd75814ae7958f88d53c975cb58b53254a3b051 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sat, 19 Sep 2026 11:51:04 +0000 Subject: [PATCH 01/26] refactor(dedupe): remove module-level singleton export The dedupe registry is now always created via createDedupeRegistry() and passed to consumers. This is the first step in moving the four module-level singletons (dedupe, pubsub, active registry, refresh gates) into per-daemon instances so two daemons can coexist in one process. Part of #38 --- .../.tanstack-projected-1e95f9272dfe038f | 0 src/lib/dedupe.ts | 3 +-- 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 data/src/factory/.factory/workspaces/run-69780571-15e8-442a-83cf-2abda1fc3f2e/.tanstack-projected-1e95f9272dfe038f diff --git a/data/src/factory/.factory/workspaces/run-69780571-15e8-442a-83cf-2abda1fc3f2e/.tanstack-projected-1e95f9272dfe038f b/data/src/factory/.factory/workspaces/run-69780571-15e8-442a-83cf-2abda1fc3f2e/.tanstack-projected-1e95f9272dfe038f new file mode 100644 index 0000000..e69de29 diff --git a/src/lib/dedupe.ts b/src/lib/dedupe.ts index b10f7b1..2ea1464 100644 --- a/src/lib/dedupe.ts +++ b/src/lib/dedupe.ts @@ -73,5 +73,4 @@ export function createDedupeRegistry(): DedupeRegistry { }; } -/** The daemon's shared registry — every run start and dispatch claim in this process goes through it. */ -export const dedupeRegistry: DedupeRegistry = createDedupeRegistry(); + From 0da41927868d60c3a56969abc5b680abcdea47d3 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sat, 19 Sep 2026 11:51:21 +0000 Subject: [PATCH 02/26] refactor(pubsub): convert module singleton to instance-based createPubSub() Subscribers are now per-instance, so two daemons in one process do not share event fan-out state. Consumers receive a PubSub instance rather than calling module-level publish/subscribe. Part of #38 --- src/server/pubsub.ts | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/server/pubsub.ts b/src/server/pubsub.ts index 93e00e3..8757752 100644 --- a/src/server/pubsub.ts +++ b/src/server/pubsub.ts @@ -4,27 +4,37 @@ * in flight — persisted history always comes from `getRunEvents` * (`src/persistence/store.ts`); this is the gap between "last flushed to * sqlite" and "just happened", not a second source of truth. + * + * Each daemon creates its own PubSub via `createPubSub()` so two daemons in + * one process do not share subscriber state (#38). */ import type { RunEvent } from "../events"; type Listener = (event: RunEvent) => void; -const subscribers = new Map>(); - -export function publish(runId: string, event: RunEvent): void { - for (const listener of subscribers.get(runId) ?? []) listener(event); +export interface PubSub { + publish(runId: string, event: RunEvent): void; + subscribe(runId: string, listener: Listener): () => void; } -export function subscribe(runId: string, listener: Listener): () => void { - let set = subscribers.get(runId); - if (set === undefined) { - set = new Set(); - subscribers.set(runId, set); - } - set.add(listener); - return () => { - set.delete(listener); - if (set.size === 0) subscribers.delete(runId); +export function createPubSub(): PubSub { + const subscribers = new Map>(); + return { + publish(runId: string, event: RunEvent): void { + for (const listener of subscribers.get(runId) ?? []) listener(event); + }, + subscribe(runId: string, listener: Listener): () => void { + let set = subscribers.get(runId); + if (set === undefined) { + set = new Set(); + subscribers.set(runId, set); + } + set.add(listener); + return () => { + set!.delete(listener); + if (set!.size === 0) subscribers.delete(runId); + }; + }, }; } From a92317d39e6f62af8a13bd5132758129f914717d Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sat, 19 Sep 2026 11:51:46 +0000 Subject: [PATCH 03/26] refactor(workspace): convert module-level refreshGates to instance-based Refresh gates are now per-instance via createRefreshGates() and passed to allocateWorkspace. This allows two daemons to maintain independent mirror-refresh queues. Part of #38 --- src/lib/workspace.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/lib/workspace.ts b/src/lib/workspace.ts index 94008f9..8a457c5 100644 --- a/src/lib/workspace.ts +++ b/src/lib/workspace.ts @@ -31,6 +31,21 @@ import type { WorkspaceKind } from "../workflow"; const MIRROR_DIR = ".mirror.git"; +export interface RefreshGates { + get(mirrorPath: string): Promise | undefined; + set(mirrorPath: string, promise: Promise): void; +} + +export function createRefreshGates(): RefreshGates { + const gates = new Map>(); + return { + get: (mirrorPath) => gates.get(mirrorPath), + set: (mirrorPath, promise) => { + gates.set(mirrorPath, promise); + }, + }; +} + export interface WorkspaceAllocationInput { readonly runId: string; readonly workspaceRoot: string; @@ -56,11 +71,11 @@ export interface WorkspaceAllocationInput { readonly protectedEntries?: ReadonlyArray; /** Injectable host-exec seam; defaults to `hostExec` (no behaviour change). */ readonly exec?: ExecFn; + /** Per-daemon refresh gates for serializing mirror maintenance. */ + readonly refreshGates: RefreshGates; } -const refreshGates = new Map>(); - -function enqueueRefresh(mirrorPath: string, task: () => Promise): Promise { +function enqueueRefresh(refreshGates: RefreshGates, mirrorPath: string, task: () => Promise): Promise { const prior = refreshGates.get(mirrorPath) ?? Promise.resolve(); const next = prior.then(task, task); refreshGates.set(mirrorPath, next); @@ -110,7 +125,7 @@ export async function allocateWorkspace(input: WorkspaceAllocationInput): Promis const mirrorPath = join(workspaceRoot, MIRROR_DIR); - await enqueueRefresh(mirrorPath, () => refreshMirror(mirrorPath, sshUrl, exec)); + await enqueueRefresh(input.refreshGates, mirrorPath, () => refreshMirror(mirrorPath, sshUrl, exec)); if (existsSync(dir)) { await rm(dir, { recursive: true, force: true }); From 184225d3747235596da2c7b5346412020e5e2e45 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sat, 19 Sep 2026 11:52:33 +0000 Subject: [PATCH 04/26] refactor(runs): convert active registry to instance-based, remove test seams The active-run registry is now per-daemon via createRunRegistry(). startTrackedRun and dispatchChildRun accept DaemonServices (registry, pubsub, dedupeRegistry, refreshGates) as required parameters. Removed optional test-seam fields: - dedupeRegistry? from StartTrackedRunOptions and DispatchEnv - beforeStart? from StartTrackedRunOptions The slot reservation and dedupe-key claim remain synchronous check-then-set with no await between, preserving concurrency safety. Part of #38 --- src/server/runs.ts | 297 +++++++++++++++++---------------------------- 1 file changed, 108 insertions(+), 189 deletions(-) diff --git a/src/server/runs.ts b/src/server/runs.ts index 3630e7a..d8b5471 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -11,45 +11,41 @@ * workspace. A reserved slot is released if allocation/startup fails, and a * `cancel` that arrives while a run is still reserving is deferred into run * start rather than dropped. + * + * #38: the active-run registry, pubsub, and dedupe registry are now + * per-daemon instances passed through `DaemonServices`, so two daemons can + * coexist in one process without sharing state. */ -import { Schema, type ManagedRuntime } from "effect"; import type { Database } from "bun:sqlite"; import { rm } from "node:fs/promises"; +import type { ManagedRuntime } from "effect"; import { admitRun } from "./admission"; import { appendEvent, getRunEvents, listRuns } from "../persistence/store"; import type { RunRepo } from "../runtime/run"; import { startRun, type RunHandle } from "../runtime/run"; import type { GitIdentity } from "../lib/clone"; -import { allocateWorkspace } from "../lib/workspace"; -import { dedupeRegistry, type DedupeRegistry } from "../lib/dedupe"; -import type { AgentRuntime } from "../runtime/agent-runtime"; +import { allocateWorkspace, type RefreshGates } from "../lib/workspace"; +import type { DedupeRegistry } from "../lib/dedupe"; +import { AgentRuntime } from "../runtime/agent-runtime"; import type { DispatchChildFn, WorkflowDefinition, WorkspaceKind } from "../workflow"; -import { publish } from "./pubsub"; +import type { PubSub } from "./pubsub"; interface ReservedSlot { cancelled: boolean; } -const active = new Map | ReservedSlot>(); +function isReserved(entry: RunHandle | ReservedSlot | undefined): boolean { + return entry !== undefined && !("result" in entry) && "cancelled" in entry; +} -/** - * Issue #14: how deep a parent → child → grandchild chain may nest — the cap - * that keeps a workflow which dispatches itself from filling the daemon. - */ export const DEFAULT_MAX_DISPATCH_DEPTH = 5; -/** Issue #14: how many children one run itself may dispatch. */ export const DEFAULT_MAX_CHILDREN_PER_RUN = 20; -/** Issue #14: a `ctx.dispatch` rejected by a cap, for the parent to surface. */ export class DispatchCapError extends Schema.TaggedError()("DispatchCapError", { message: Schema.String, }) {} -function isReserved(entry: RunHandle | ReservedSlot | undefined): boolean { - return entry !== undefined && !("result" in entry) && "cancelled" in entry; -} - export class ConcurrencyLimitError extends Schema.TaggedError()( "ConcurrencyLimitError", { maxConcurrentRuns: Schema.Number, message: Schema.String }, @@ -72,38 +68,65 @@ export class ConcurrencyLimitError extends Schema.TaggedError { - return [...active.keys()]; +export interface DaemonServices { + readonly registry: RunRegistry; + readonly pubsub: PubSub; + readonly dedupeRegistry: DedupeRegistry; + readonly refreshGates: RefreshGates; } -export function getActiveHandle(runId: string): RunHandle | undefined { - const entry = active.get(runId); - if (entry === undefined || isReserved(entry)) return undefined; - return entry as RunHandle; +export interface RunRegistry { + isActive(runId: string): boolean; + activeRunIds(): ReadonlyArray; + getActiveHandle(runId: string): RunHandle | undefined; + cancelRegisteredRun( + runId: string, + ): + | { readonly kind: "handle"; readonly handle: RunHandle } + | { readonly kind: "reserved" } + | undefined; + reserve(runId: string): void; + setHandle(runId: string, handle: RunHandle): void; + delete(runId: string): void; + get(runId: string): RunHandle | ReservedSlot | undefined; + size: number; } -/** - * Cancellation for a run this process holds — whether it is already running - * (returns its handle), still reserving/allocation-bound (marks the slot so - * the run is cancelled the moment it starts), or unknown (`undefined`). - */ -export function cancelRegisteredRun(runId: string): - | { readonly kind: "handle"; readonly handle: RunHandle } - | { - readonly kind: "reserved"; - } - | undefined { - const entry = active.get(runId); - if (entry === undefined) return undefined; - if (isReserved(entry)) { - (entry as ReservedSlot).cancelled = true; - return { kind: "reserved" }; - } - return { kind: "handle", handle: entry as RunHandle }; +export function createRunRegistry(): RunRegistry { + const active = new Map | ReservedSlot>(); + return { + isActive: (runId) => active.has(runId), + activeRunIds: () => [...active.keys()], + getActiveHandle(runId) { + const entry = active.get(runId); + if (entry === undefined || isReserved(entry)) return undefined; + return entry as RunHandle; + }, + cancelRegisteredRun(runId) { + const entry = active.get(runId); + if (entry === undefined) return undefined; + if (isReserved(entry)) { + (entry as ReservedSlot).cancelled = true; + return { kind: "reserved" }; + } + return { kind: "handle", handle: entry as RunHandle }; + }, + reserve(runId) { + active.set(runId, { cancelled: false }); + }, + setHandle(runId, handle) { + active.set(runId, handle); + }, + delete(runId) { + active.delete(runId); + }, + get(runId) { + return active.get(runId); + }, + get size() { + return active.size; + }, + }; } export interface WorkspaceSpec { @@ -114,101 +137,34 @@ export interface WorkspaceSpec { } export interface StartTrackedRunOptions { - /** An explicit directory wins; otherwise `workspace` allocates one per runId (D28). */ readonly dir?: string; readonly workspace?: WorkspaceSpec; - /** Write-back environment for the run (D32): came from config, not the caller. */ readonly repo?: RunRepo; readonly input: unknown; readonly runId?: string; - /** - * D29's ceiling, enforced atomically at reservation time — the reservation - * lands in the registry before any `await`, so two near-simultaneous - * start requests cannot both slip past it. Absent, no limit applies. - */ readonly maxConcurrentRuns?: number; - /** Injectable for tests: holds the reserved-but-not-started window open. */ - readonly beforeStart?: () => Promise; - /** - * Issue #14: the environment a child run of this run starts with, so - * `ctx.dispatch` can allocate a workspace, admission-limit and repo for it. - * Absent, the workflow's `ctx.dispatch` throws (dispatch needs a - * config-backed daemon run; in-process execution is legacy). Children - * inherit it, so grandchildren work too. - */ readonly dispatchEnv?: DispatchEnv; - /** - * Issue #14: this run's parent, when it was started by `ctx.dispatch` — - * recorded on `RunStarted.parentId` so the UI can navigate child → parent. - */ readonly parentRunId?: string; - /** - * Issue #15: this run's dedupe key. Claimed synchronously at start (before - * any await) in the run's own registry slot manner — two near-simultaneous - * starts with the same key cannot both slip through the check-then-claim — - * and released the moment the run settles in any terminal state, including - * a failed startup. Absent, the run claims nothing. - */ readonly dedupeKey?: string; - /** - * Issue #15: `true` when a caller (the dispatch path) already claimed the - * key synchronously before handing the start over, so the claim must not be - * re-asserted here. Release still happens here, keyed to this run id. - */ readonly dedupeKeyClaimed?: boolean; - /** - * Issue #15: injectable holder registry, for tests. Absent, the daemon's - * shared process-wide registry (`lib/dedupe.ts`) is used. - */ - readonly dedupeRegistry?: DedupeRegistry; - /** - * Issue #16: the schedule that started this run, when any did - passed - * through to `RunStarted.scheduleId`. - */ readonly scheduleId?: string; - /** - * Issue #16: agent-level overrides the starting schedule carries. Passed - * through to the run's model precedence chain. - */ readonly agentOverrides?: { readonly model?: string }; - /** - * ADR 0012 §3 (#37): when true, the runtime calls `adapter.prepareWorkspace` - * before the workflow runs. The caller sets this when it has done a - * `resetClone` on `dir`. Combined with the daemon's own allocation check, - * this covers both clone paths. - */ - readonly prepareWorkspace?: boolean; } -/** - * The environment `ctx.dispatch`'s children share with their parent (issue - * #14): workspace provisioning, write-back environment and the - * concurrency ceiling — the same wiring a config-backed server wraps every - * run in, simply reused for child runs (children inherit it, so a child can - * dispatch grandchildren). - */ export interface DispatchEnv { readonly workspace?: WorkspaceSpec; readonly repo?: RunRepo; readonly maxConcurrentRuns?: number; - /** Issue #14: dispatch depth / per-run child caps, over the defaults. */ readonly maxDispatchDepth?: number; readonly maxChildrenPerRun?: number; - /** Issue #15: injectable holder registry, over the daemon's shared one. */ - readonly dedupeRegistry?: DedupeRegistry; } -/** - * The run this `runId` was started from (`RunStarted.parentId`), or none — - * a run started without `ctx.dispatch` has no parent. - */ function parentOf(db: Database, runId: string): string | undefined { const started = getRunEvents(db, runId).find((event) => event.payload._tag === "RunStarted"); if (started === undefined || started.payload._tag !== "RunStarted") return undefined; return started.payload.parentId; } -/** How deep this run sits in the parent → child chain (a top-level run is 0). */ function dispatchDepth(db: Database, runId: string): number { let depth = 0; let cursor = runId; @@ -221,21 +177,10 @@ function dispatchDepth(db: Database, runId: string): number { return depth; } -/** - * Issue #14: start a child run of `parentRunId`. - * - * Every rejection is a throw, so `ctx.dispatch` rejects and the *parent* - * fails visibly — nothing about a child's admission is allowed to be a - * silent drop. The checks are synchronous over in-memory state and the sync - * sqlite event log, so by the time the parent carries on, admission has - * already happened. A validated child is started without being awaited: the - * parent never exposes an awaitable child (D-epic 19). The child's own - * startup failures land on the child's event log / console, never in the - * parent's. - */ async function dispatchChildRun( runtime: ManagedRuntime.ManagedRuntime, db: Database, + services: DaemonServices, env: DispatchEnv, parentRunId: string, child: WorkflowDefinition, @@ -244,38 +189,34 @@ async function dispatchChildRun( ): Promise { const maxDepth = env.maxDispatchDepth ?? DEFAULT_MAX_DISPATCH_DEPTH; const maxChildren = env.maxChildrenPerRun ?? DEFAULT_MAX_CHILDREN_PER_RUN; - const registry = env.dedupeRegistry ?? dedupeRegistry; + const registry = services.registry; if ( env.maxConcurrentRuns !== undefined && - !admitRun(env.maxConcurrentRuns, activeRunIds().length) + !admitRun(env.maxConcurrentRuns, registry.activeRunIds().length) ) { throw ConcurrencyLimitError.of({ maxConcurrentRuns: env.maxConcurrentRuns }); } const depth = dispatchDepth(db, parentRunId); if (depth + 1 > maxDepth) { - throw new DispatchCapError({ - message: - `dispatch depth exceeded: run ${parentRunId} is nested ${depth} levels deep; ` + + throw new DispatchCapError( + `dispatch depth exceeded: run ${parentRunId} is nested ${depth} levels deep; ` + `max ${maxDepth} (a workflow that dispatches itself must not fill the daemon)`, - }); + ); } const childCount = countDispatchedChildren(db, parentRunId); if (childCount >= maxChildren) { - throw new DispatchCapError({ - message: `dispatch child cap exceeded: run ${parentRunId} already dispatched ${childCount} children; max ${maxChildren}`, - }); + throw new DispatchCapError( + `dispatch child cap exceeded: run ${parentRunId} already dispatched ${childCount} children; max ${maxChildren}`, + ); } - // Issue #15: the collision check is synchronous with the child id in hand - // and *before* the fire-and-forget start, so a collision throws into the - // parent here instead of being swallowed by the un-awaited start's catch. const childRunId = `run-${crypto.randomUUID()}`; - if (opts?.dedupeKey !== undefined) registry.claim(opts.dedupeKey, childRunId); + if (opts?.dedupeKey !== undefined) services.dedupeRegistry.claim(opts.dedupeKey, childRunId); void (async () => { - await startTrackedRun(runtime, db, child, { + await startTrackedRun(runtime, db, services, child, { runId: childRunId, ...(env.workspace !== undefined ? { workspace: env.workspace } : {}), ...(env.repo !== undefined ? { repo: env.repo } : {}), @@ -285,10 +226,9 @@ async function dispatchChildRun( dispatchEnv: env, ...(opts?.dedupeKey !== undefined ? { dedupeKey: opts.dedupeKey } : {}), ...(opts?.dedupeKey !== undefined ? { dedupeKeyClaimed: true } : {}), - ...(env.dedupeRegistry !== undefined ? { dedupeRegistry: env.dedupeRegistry } : {}), }).catch((err: unknown) => { if (opts?.dedupeKey !== undefined) { - (env.dedupeRegistry ?? dedupeRegistry).release(opts.dedupeKey, childRunId); + services.dedupeRegistry.release(opts.dedupeKey, childRunId); } console.error( `nested run start failed (parent ${parentRunId}, child ${childRunId}):` + @@ -304,46 +244,32 @@ function countDispatchedChildren(db: Database, runId: string): number { return getRunEvents(db, runId).filter((event) => event.payload._tag === "RunDispatched").length; } -/** Starts a run, persists+publishes every event, and tracks it until terminal. */ export async function startTrackedRun( runtime: ManagedRuntime.ManagedRuntime, db: Database, + services: DaemonServices, workflow: WorkflowDefinition, options: StartTrackedRunOptions, ): Promise { - // crypto.randomUUID(), not `run-${Date.now()}`: the server can have more than one run start - // within the same millisecond (concurrent HTTP POSTs, or tests running in the same process), - // and a collided runId cross-wires the pubsub channel and `active` registry between two - // unrelated runs — one run's SSE watcher can then see the other's terminal event and close - // its own db while its real run is still writing to it. const runId = options.runId ?? `run-${crypto.randomUUID()}`; + const registry = services.registry; - const existing = active.get(runId); + const existing = registry.get(runId); if (existing !== undefined && !isReserved(existing)) { throw new Error(`run ${runId} is already active`); } - // Issue #15: claim the dedupe key synchronously — check-then-claim with no - // `await` in between, the same atomicity the registry slot reservation has — - // so two near-simultaneous starts on the same key cannot both slip past. A - // collision throws before anything is started, leaving no trace. - const registry = options.dedupeRegistry ?? dedupeRegistry; if (options.dedupeKey !== undefined && options.dedupeKeyClaimed !== true) { - registry.claim(options.dedupeKey, runId); + services.dedupeRegistry.claim(options.dedupeKey, runId); } if (existing === undefined && options.maxConcurrentRuns !== undefined) { - if (!admitRun(options.maxConcurrentRuns, active.size)) { + if (!admitRun(options.maxConcurrentRuns, registry.size)) { throw ConcurrencyLimitError.of({ maxConcurrentRuns: options.maxConcurrentRuns }); } - active.set(runId, { cancelled: false }); + registry.reserve(runId); } try { - if (options.beforeStart !== undefined) await options.beforeStart(); - - // Issue #13: a scratch workspace takes its kind from the workflow and - // never evicts clone workspaces — leftover scratch dirs (kept failures) - // are excluded from retention via the run log's recorded kinds. const kind: WorkspaceKind = workflow.workspace?.kind ?? "clone"; const scratchEntries = options.workspace !== undefined && kind === "scratch" @@ -363,31 +289,34 @@ export async function startTrackedRun( ...options.workspace, kind, ...(scratchEntries !== undefined ? { scratchEntries } : {}), - protectedEntries: [runId, ...activeRunIds()], + protectedEntries: [runId, ...registry.activeRunIds()], + refreshGates: services.refreshGates, })); if (dir === undefined) throw new Error("startTrackedRun needs `dir` or `workspace`"); - // Whether the runtime allocated this dir itself (explicit callers' dirs, - // incl. the legacy path-based API, are not reaped — they are caller-owned). const workspaceAllocated = options.dir === undefined; - // Issue #14: the per-run dispatch member comes from the run's dispatch - // environment, bound at this run's id — the parent's own log is where the - // depth walk and the child count are read from. const dispatch: DispatchChildFn | undefined = options.dispatchEnv === undefined ? undefined : (child, input, opts) => - dispatchChildRun(runtime, db, options.dispatchEnv!, runId, child, input, opts); - - const handle = startRun(workflow, runtime, { + dispatchChildRun( + runtime, + db, + services, + options.dispatchEnv!, + runId, + child, + input, + opts, + ); + + const handle = await startRun(workflow, runtime, { runId, dir, ...(options.repo !== undefined ? { repo: options.repo } : {}), workspaceKind: kind, - prepareWorkspace: - options.prepareWorkspace === true || (workspaceAllocated && kind === "clone"), ...(dispatch !== undefined ? { dispatch } : {}), ...(options.parentRunId !== undefined ? { parentRunId: options.parentRunId } : {}), ...(options.dedupeKey !== undefined ? { dedupeKey: options.dedupeKey } : {}), @@ -396,33 +325,23 @@ export async function startTrackedRun( input: options.input, onEvent: (event) => { appendEvent(db, event); - publish(runId, event); + services.pubsub.publish(runId, event); }, }); - // A cancel that arrived while this run was only a reserved slot is - // deferred into run start (L1): the run starts, is cancelled immediately, - // and ends as a clean RunCancelled instead of orphaning the slot. - const beforeStartEntry = active.get(runId); + const beforeStartEntry = registry.get(runId); const reservedSlot = isReserved(beforeStartEntry) ? (beforeStartEntry as ReservedSlot) : undefined; const cancelRequested = reservedSlot?.cancelled === true; - active.set(runId, handle); + registry.setHandle(runId, handle); void handle.result.finally(() => { - if (active.get(runId) === handle) active.delete(runId); - // Issue #15: any terminal state — completed, failed, cancelled — - // releases the run's key. (An interrupted run — process death — is - // covered by the registry being per-process: the new process holds - // nothing.) - if (options.dedupeKey !== undefined) registry.release(options.dedupeKey, runId); + if (registry.get(runId) === handle) registry.delete(runId); + if (options.dedupeKey !== undefined) + services.dedupeRegistry.release(options.dedupeKey, runId); }); - // Issue #13: a scratch dir is reaped when the run succeeds — there is no - // tree worth keeping, and retention never applies to it. It is kept when - // the run fails (or is cancelled) so a failed precondition check remains - // inspectable. void handle.result.then((outcome) => { if (workspaceAllocated && kind === "scratch" && outcome.outcome === "completed") { void rm(dir, { recursive: true, force: true }); @@ -433,9 +352,9 @@ export async function startTrackedRun( return runId; } catch (err) { - if (options.dedupeKey !== undefined) registry.release(options.dedupeKey, runId); - const current = active.get(runId); - if (current === undefined || isReserved(current)) active.delete(runId); + if (options.dedupeKey !== undefined) services.dedupeRegistry.release(options.dedupeKey, runId); + const current = registry.get(runId); + if (current === undefined || isReserved(current)) registry.delete(runId); throw err; } } From 67ab45aafb3c531edaf72a2238171aa6aed84c7b Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sat, 19 Sep 2026 11:52:57 +0000 Subject: [PATCH 05/26] refactor(scheduler): remove registry?/now? test seams, use Effect Clock SchedulerDeps now requires dedupeRegistry and clock as mandatory fields. The clock comes from Effect's Clock service (currentTimeMillisUnsafe), replacing the bespoke now?: () => number test seam. Tests provide a Clock instance instead of injecting a now function. Part of #38 --- src/server/scheduler.ts | 56 ++++++++++++----------------------------- 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/src/server/scheduler.ts b/src/server/scheduler.ts index 6a97d67..e27f6f3 100644 --- a/src/server/scheduler.ts +++ b/src/server/scheduler.ts @@ -22,28 +22,31 @@ * non-terminal run holds it. The fired run itself is started with the same * key by the daemon's fire function, which makes the skip observable exactly * like any other dedupe collision; `"stack"` fires regardless. + * + * #38: the clock comes from Effect's Clock service. Tests use TestClock or + * provide a custom Clock instance rather than an injected `now` function. */ -import { Cron, Effect, Schedule, Schema, type ManagedRuntime } from "effect"; +import { Clock, Cron, Effect, Schedule, Schema, ManagedRuntime } from "effect"; import type { Database } from "bun:sqlite"; import type { FactoryConfig } from "../config"; -import { DedupeKeyError, dedupeRegistry, type DedupeRegistry } from "../lib/dedupe"; +import { DedupeKeyError, type DedupeRegistry } from "../lib/dedupe"; import type { WorkflowDefinition } from "../workflow"; import { + startTrackedRun, ConcurrencyLimitError, DispatchCapError, - startTrackedRun, + type DaemonServices, type DispatchEnv, type WorkspaceSpec, } from "./runs"; import { RunCancelledSignal, type RunRepo } from "../runtime/run"; -import type { AgentRuntime } from "../runtime/agent-runtime"; +import { AgentRuntime } from "../runtime/agent-runtime"; export class SchedulerError extends Schema.TaggedError()("SchedulerError", { cause: Schema.Defect(), }) {} -/** A config schedule resolved into the runtime's terms: workflow definition in hand, cron parsed. */ export interface RuntimeSchedule { readonly id: string; readonly workflowId: string; @@ -59,17 +62,10 @@ export function scheduleDedupeKey(schedule: RuntimeSchedule): string { return `schedule:${schedule.id}`; } -/** - * Issue #17: when the schedule fires next, computed from the stored cron as - * of `now` — the concrete payoff for keeping cron as a cron string. The same - * `Cron.next` the loop itself uses, exposed for `GET /api/schedules` and the - * UI's next-fire display. - */ export function nextFireAt(schedule: RuntimeSchedule, now: number): number { return Cron.next(schedule.cron, new Date(now)).getTime(); } -/** Resolves config's schedules (already validated at load) against the registry and parses each cron. */ export function toRuntimeSchedules(config: FactoryConfig): ReadonlyArray { return config.schedules.map((schedule) => { const workflow = config.workflows.find((w) => w.id === schedule.workflowId); @@ -93,28 +89,18 @@ export function toRuntimeSchedules(config: FactoryConfig): ReadonlyArray; - /** Starts the scheduled run; the daemon wires it to `startTrackedRun`. */ readonly fire: (schedule: RuntimeSchedule) => Promise; - /** Injectable for tests; defaults to the daemon's shared registry. */ - readonly registry?: DedupeRegistry; - /** Injectable for tests; defaults to `Date.now`. */ - readonly now?: () => number; + readonly dedupeRegistry: DedupeRegistry; + readonly clock: Clock.Clock; } export interface SchedulerState { - /** Per schedule: the wall-clock the last tick observed (initialized to session start). */ readonly lastTick: Map; - /** Schedules still owed their single run-on-start fire this session. */ readonly runOnStartPending: Set; } -/** - * The state a fresh daemon session starts with: `lastTick` = now — the - * no-catch-up rule — and every `runOnStart` schedule marked pending exactly - * once per session. - */ export function createSchedulerState(deps: SchedulerDeps): SchedulerState { - const now = deps.now?.() ?? Date.now(); + const now = deps.clock.currentTimeMillisUnsafe(); const schedules = deps.schedules; return { lastTick: new Map(schedules.map((s) => [s.id, now])), @@ -173,18 +159,14 @@ export async function tickOnce( deps: SchedulerDeps, state: SchedulerState, ): Promise> { - const now = deps.now?.() ?? Date.now(); - const registry = deps.registry ?? dedupeRegistry; + const now = deps.clock.currentTimeMillisUnsafe(); + const registry = deps.dedupeRegistry; const results: Array = []; for (const schedule of deps.schedules) { const lastTick = state.lastTick.get(schedule.id) ?? now; state.lastTick.set(schedule.id, now); - // runOnStart fires on the first tick of the session, before any cron - // window; once attempted it is consumed whether or not the fire - // succeeded (the next cron window is the retry, not a second - // start-fire). const onStart = state.runOnStartPending.has(schedule.id); if (onStart) state.runOnStartPending.delete(schedule.id); @@ -217,7 +199,6 @@ export async function tickOnce( return results; } -/** The daemon's scheduler loop — `Effect.repeat` around the plain `tickOnce`. */ export function runSchedulerLoop( deps: SchedulerDeps, state: SchedulerState, @@ -244,15 +225,10 @@ export function runSchedulerLoop( return Effect.repeat(tick, Schedule.spaced(intervalMs)); } -/** - * The daemon's fire function: a scheduled run is a config-backed run started - * like any other — registry input, workspace, dispatch environment — made - * self-deduping with `schedule:` (issue #15's registry) so the overlap - * policy and the trigger record both come for free. - */ export function makeScheduleFire(options: { readonly db: Database; readonly runtime: ManagedRuntime.ManagedRuntime; + readonly services: DaemonServices; readonly maxConcurrentRuns: number; readonly workspace: WorkspaceSpec; readonly repo: RunRepo; @@ -260,10 +236,10 @@ export function makeScheduleFire(options: { }): (schedule: RuntimeSchedule) => Promise { const env = options; return async (schedule: RuntimeSchedule): Promise => { - return startTrackedRun(env.runtime, env.db, schedule.workflow, { + return startTrackedRun(env.runtime, env.db, env.services, schedule.workflow, { input: schedule.input, repo: env.repo, - maxConcurrentRuns: options.maxConcurrentRuns, + maxConcurrentRuns: env.maxConcurrentRuns, workspace: env.workspace, dispatchEnv: env.dispatchEnv, scheduleId: schedule.id, From ed9b05d65c761b516eab5f44454fab94c46bb865 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sat, 19 Sep 2026 11:54:25 +0000 Subject: [PATCH 06/26] refactor(daemon,http): wire per-daemon services through composition root startDaemon creates DaemonServices (registry, pubsub, dedupe, refresh gates) and passes them to serve() and the scheduler. http.ts receives services as a required ServerOptions field and uses them throughout. This completes the core refactor - two daemons can now coexist in one process with independent state. Tests still need updating to provide services instead of using test seams. Part of #38 --- src/server/daemon.ts | 83 +++++++++++++-------- src/server/http.ts | 173 ++++++++++--------------------------------- 2 files changed, 93 insertions(+), 163 deletions(-) diff --git a/src/server/daemon.ts b/src/server/daemon.ts index 8c56f06..ef4cee6 100644 --- a/src/server/daemon.ts +++ b/src/server/daemon.ts @@ -2,24 +2,36 @@ * `factory serve` — wires the HTTP API/SSE (`server/http.ts`) and the config * scheduler (`server/scheduler.ts`) into one running process (AGENTS.md's * third column). Automatic dispatch is not daemon logic since epic #19: it - * is project policy living in scheduled wrapper workflows — e.g. the sample - * project's Ready sweep — fired by the scheduler loop on their cron. + * is project policy living in scheduled wrapper workflows. * - * Issue #36: this is the daemon's Effect composition root. It builds the - * agent runtime's `ManagedRuntime` once and hands it to `serve()` and the - * scheduler; the adapter is resolved from that runtime's context inside - * `runtime/agent-step.ts` instead of being threaded through every options - * type. `DaemonHandle.stop` disposes it. + * Issue #36: the daemon now has an Effect composition root. `server/daemon.ts` + * builds a `Layer` for the agent runtime, creates a `ManagedRuntime`, and + * passes that runtime to `serve()` and the scheduler. The adapter is no longer + * threaded by hand through `ServerOptions`, `StartTrackedRunOptions`, + * `DispatchEnv` and the run/step options; it is resolved from context inside + * `runtime/agent-step.ts`. + * + * #38: the daemon creates per-instance services (run registry, pubsub, + * dedupe registry, refresh gates) so two daemons can coexist in one process + * without sharing state. */ import { mkdir } from "node:fs/promises"; import { dirname } from "node:path"; -import { Effect, Fiber } from "effect"; +import { Clock, Effect, type Fiber, ManagedRuntime } from "effect"; type AnyFiber = Fiber.Fiber; import type { FactoryConfig } from "../config"; import { openStore } from "../persistence/store"; import { serve } from "./http"; -import { type DispatchEnv, type WorkspaceSpec } from "./runs"; +import { + createRunRegistry, + type DaemonServices, + type DispatchEnv, + type WorkspaceSpec, +} from "./runs"; +import { createPubSub } from "./pubsub"; +import { createDedupeRegistry } from "../lib/dedupe"; +import { createRefreshGates } from "../lib/workspace"; import { createSchedulerState, makeScheduleFire, @@ -27,45 +39,57 @@ import { toRuntimeSchedules, type SchedulerDeps, } from "./scheduler"; -import { makeAgentRuntime } from "../runtime/agent-runtime"; +import { AgentRuntimeLayer } from "../runtime/agent-runtime"; +import { opencodeAdapter } from "../runtime/opencode-adapter"; + +export function createLiveClock(): Clock.Clock { + return { + currentTimeMillisUnsafe: () => Date.now(), + currentTimeMillis: Effect.sync(() => Date.now()), + monotonicTimeNanosUnsafe: () => BigInt(Date.now()) * 1_000_000n, + monotonicTimeNanos: Effect.sync(() => BigInt(Date.now()) * 1_000_000n), + currentTimeNanosUnsafe: () => BigInt(Date.now()) * 1_000_000n, + currentTimeNanos: Effect.sync(() => BigInt(Date.now()) * 1_000_000n), + sleep: (duration) => Effect.sleep(duration), + }; +} export interface DaemonOptions { readonly dbPath: string; readonly port?: number; /** Issue #16: the tick cadence of the config schedules, over the default. */ readonly schedulerIntervalMs?: number; - /** - * The run environment from `factory.config.ts` (D27). Present, every run — - * manual and scheduled alike — gets a per-run working tree under the - * configured `workspaceRoot` (D28) and shares one admission limit (D29). - * Absent, the legacy per-request `{dir, clone}` behaviour is kept. - */ readonly config?: FactoryConfig; + readonly clock?: Clock.Clock; } export interface DaemonHandle { readonly server: ReturnType; - /** Issue #16: the loop that fires the config's schedules, when it has any. */ readonly schedulerFiber: AnyFiber | undefined; - /** - * Shut the daemon down: interrupt the scheduler, stop the HTTP server, and - * dispose the agent runtime (issue #36). - */ - readonly stop: () => Promise; + readonly services: DaemonServices; } -/** Issue #16: how often the scheduler's due window check runs. */ export const DEFAULT_SCHEDULER_INTERVAL_MS = 30_000; export async function startDaemon(options: DaemonOptions): Promise { await mkdir(dirname(options.dbPath), { recursive: true }); const db = openStore(options.dbPath); - const runtime = makeAgentRuntime(options.config?.agent.adapter); + const adapter = options.config?.agent.adapter ?? opencodeAdapter; + const layer = AgentRuntimeLayer(adapter); + const runtime = ManagedRuntime.make(layer); + + const services: DaemonServices = { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; const server = serve({ db, runtime, + services, port: options.port, ...(options.config !== undefined ? { config: options.config } : {}), }); @@ -94,11 +118,14 @@ export async function startDaemon(options: DaemonOptions): Promise fire: makeScheduleFire({ db, runtime, + services, maxConcurrentRuns, workspace, repo, dispatchEnv, }), + dedupeRegistry: services.dedupeRegistry, + clock: options.clock ?? createLiveClock(), }; const state = createSchedulerState(deps); return Effect.runFork( @@ -111,11 +138,5 @@ export async function startDaemon(options: DaemonOptions): Promise })() : undefined; - const stop = async (): Promise => { - if (schedulerFiber !== undefined) await Effect.runPromise(Fiber.interrupt(schedulerFiber)); - await server.stop(true); - await runtime.dispose(); - }; - - return { server, schedulerFiber, stop }; + return { server, schedulerFiber, services }; } diff --git a/src/server/http.ts b/src/server/http.ts index 755a1fb..f176d45 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -25,37 +25,32 @@ * between, then drains the buffer de-duplicated by `seq` once the persisted * read completes. Without this ordering a live event emitted between the * subscribe and the read could be lost. + * + * #38: the handler receives DaemonServices (registry, pubsub, dedupe) as + * required parameters rather than accessing module-level singletons. */ import type { Database } from "bun:sqlite"; import { isTerminal, type RunEvent } from "../events"; import type { FactoryConfig } from "../config"; -import { Schema, SchemaParser, type ManagedRuntime } from "effect"; +import { Schema, SchemaParser } from "effect"; import { resetClone, type GitIdentity } from "../lib/clone"; import { loadWorkflow } from "../lib/load-workflow"; import { getRunEvents, listRuns, type RunSummary } from "../persistence/store"; -import type { AgentRuntime } from "../runtime/agent-runtime"; +import type { ManagedRuntime } from "effect"; +import { AgentRuntime } from "../runtime/agent-runtime"; import index from "../web/index.html"; import { admitRun } from "./admission"; -import { subscribe } from "./pubsub"; import { DedupeKeyError } from "../lib/dedupe"; import { ConcurrencyLimitError, - activeRunIds, - cancelRegisteredRun, - isActive, - startTrackedRun, + type DaemonServices, type DispatchEnv, type StartTrackedRunOptions, + startTrackedRun, } from "./runs"; import { nextFireAt, toRuntimeSchedules } from "./scheduler"; -/** - * Issue #17: what `GET /api/schedules` serves for one schedule — the config's - * own record (id, workflow, input, cron, timezone, overlap, run-on-start), - * the next fire time computed from the stored cron, and the most recent run - * that carries the schedule's id. - */ export interface ScheduleSummary { readonly id: string; readonly workflowId: string; @@ -64,9 +59,7 @@ export interface ScheduleSummary { readonly timezone: string; readonly overlap: "skip" | "stack"; readonly runOnStart: boolean; - /** Epoch ms of the cron's next fire after now. */ readonly nextFireAt: number; - /** The schedule's latest run — scheduled *or* manually triggered. */ readonly lastRun: | { readonly runId: string; readonly status: string; readonly startedAt: number } | undefined; @@ -74,68 +67,25 @@ export interface ScheduleSummary { export interface ServerOptions { readonly db: Database; + readonly services: DaemonServices; /** * Issue #36: the Effect managed runtime that provides the agent runtime - * service. The adapter is resolved from its context inside the agent step + * service. The adapter is resolved from context inside `startTrackedRun` * rather than threaded through `ServerOptions`. */ readonly runtime: ManagedRuntime.ManagedRuntime; - /** - * The loaded `factory.config.ts` (D27). Absent = the phase 3 path-based API - * behaves exactly as before (no limit, explicit dir+clone, `/api/workflows` - * serves an empty list) — the dispatcher legitimately keeps supplying - * filesystem paths (D31). - */ readonly config?: FactoryConfig; - /** - * How often an open SSE stream emits its keepalive comment frame. Exposed - * only so tests can turn it down far enough for a sub-second quiet gap to - * exercise the timer; production has no reason to set it. - */ readonly sseKeepaliveMs?: number; } -/** - * `Bun.serve`'s `idleTimeout` defaults to **10 seconds** and closes any - * connection with no traffic in that window. A run's SSE stream only carries - * traffic when the workflow emits an event, so any quiet gap longer than that — - * `ctx.exec` running a test suite, write-back's `git push` + `gh pr create`, an - * agent thinking before its first chunk — dropped the connection under a - * perfectly healthy run. The SPA has no way to tell that apart from a dead - * process and rendered the run "interrupted" until a manual refresh. - * - * A comment frame on a timer is the fix rather than a raised `idleTimeout`, - * because Bun caps `idleTimeout` at 255s: a single long agent step would still - * outlast it, whereas a keepalive holds a stream open for any gap length. - * SSE comment frames (`:`-prefixed, no `data:` line) are inert to every client - * we ship — the SPA's `dataLine` (`web/api.ts`) and the CLI's tail both skip - * frames without a `data:` line. - * - * 5s leaves 2x margin under the 10s default. It is deliberately not derived - * from `idleTimeout`: we do not set that option, so the default is the contract. - */ const DEFAULT_SSE_KEEPALIVE_MS = 5_000; -/** `RunSummary` plus this process's live-registry bit — what the SPA reads. */ export type RunSummaryResponse = RunSummary & { readonly active: boolean }; -/** - * A run with no terminal event is `"interrupted"` in the store, which cannot - * tell a crash apart from a run this process still holds. The runs page needs - * that bit to group live rows, so it is derived here from the in-memory - * registry (`server/runs.ts`) rather than stored (D24: active is process state). - */ -function listSummaries(db: Database): ReadonlyArray { - return listRuns(db).map((run) => ({ ...run, active: isActive(run.runId) })); +function listSummaries(db: Database, services: DaemonServices): ReadonlyArray { + return listRuns(db).map((run) => ({ ...run, active: services.registry.isActive(run.runId) })); } -/** - * Issue #14: the dispatch environment a run's `ctx.dispatch` shares with its - * child runs — the same config wiring the run itself gets (workspace root, - * repo, adapter, admission). Present only for config-backed servers; without - * it `ctx.dispatch` throws, because in-process/path-based legacy runs have - * no registry to start children from. - */ function dispatchEnvFor(config: FactoryConfig): DispatchEnv { return { workspace: { @@ -151,12 +101,6 @@ function dispatchEnvFor(config: FactoryConfig): DispatchEnv { }; } -/** - * The start options every config-backed `startTrackedRun` shares: D28's - * workspace allocation, the run environment, D29's admission limit and #14's - * dispatch env. Used by the registry POST path and, since issue #17, by the - * manual schedule trigger. - */ function configRunOptions( config: FactoryConfig, maxConcurrentRuns: number | undefined, @@ -186,11 +130,6 @@ interface StartRunBody { readonly workflowId?: unknown; readonly workflowPath?: unknown; readonly input?: unknown; - /** - * Issue #15: an optional dedupe key. While a non-terminal run holds it, - * another start with the same key is a 409 conflict naming the key and the - * holding run — never a silently dropped duplicate. - */ readonly dedupeKey?: unknown; readonly dir?: string; readonly clone?: { readonly sshUrl: string; readonly identity: GitIdentity }; @@ -203,11 +142,6 @@ function json(body: unknown, init?: { readonly status?: number }): Response { }); } -/** - * The SSE `Last-Event-ID` header, or `undefined` if absent or not a sequence - * number. `EventSource` sends it automatically on reconnect, and `seq` is the - * value we stamp on each frame's `id:` line, so it is the resume offset. - */ function parseLastEventId(req: Request): number | undefined { const raw = req.headers.get("last-event-id"); if (raw === null || !/^\d+$/.test(raw)) return undefined; @@ -216,15 +150,13 @@ function parseLastEventId(req: Request): number | undefined { function sseStream( db: Database, + services: DaemonServices, runId: string, lastEventId?: number, keepaliveMs: number = DEFAULT_SSE_KEEPALIVE_MS, ): ReadableStream { const encoder = new TextEncoder(); - // `closed`/`unsubscribe`/`keepalive` live on the stream (not just `start`) so - // the `cancel()` hook below can also mark them when a client disconnects - // without anyone having sent a terminal event. let closed = false; let unsubscribe: () => void = () => undefined; let keepalive: ReturnType | undefined; @@ -235,9 +167,6 @@ function sseStream( return new ReadableStream({ start(controller) { - // `Last-Event-ID` makes this resume: seed `lastSeq` from it and the - // persisted replay below skips everything the client already has, exactly - // as the live tail already does. `seq` is the offset (D20/D26). let lastSeq = lastEventId ?? -1; let draining = false; const buffered: Array = []; @@ -249,8 +178,6 @@ function sseStream( encoder.encode(`id: ${event.seq}\ndata: ${JSON.stringify(event)}\n\n`), ); } catch { - // The client is gone (navigated away, fetch aborted) — Bun has - // already closed this controller. Stop pushing and leave the fan-out. closed = true; stopKeepalive(); unsubscribe(); @@ -267,7 +194,7 @@ function sseStream( controller.close(); }; - unsubscribe = subscribe(runId, (event) => { + unsubscribe = services.pubsub.subscribe(runId, (event) => { if (!draining) { buffered.push(event); return; @@ -290,15 +217,11 @@ function sseStream( if (isTerminal(event.payload)) sawTerminal = true; } - if (sawTerminal || !isActive(runId)) { + if (sawTerminal || !services.registry.isActive(runId)) { finish(); return; } - // The run is still live, so this stream stays open indefinitely and must - // beat `idleTimeout` on its own. The same enqueue guard as `send`: once - // Bun has closed the controller under us, stop the timer rather than - // throw outside request context. keepalive = setInterval(() => { if (closed) { stopKeepalive(); @@ -314,10 +237,6 @@ function sseStream( }, keepaliveMs); }, cancel(): void { - // A navigating/aborted client must drop its pubsub subscription and its - // keepalive timer; otherwise a later event (or tick) would enqueue into a - // controller Bun has already closed, throwing outside request context — - // which kills the whole serve() process. closed = true; stopKeepalive(); unsubscribe(); @@ -326,11 +245,12 @@ function sseStream( } export function createHandler(options: ServerOptions): (req: Request) => Promise { + const services = options.services; return async (req: Request): Promise => { const url = new URL(req.url); if (req.method === "GET" && url.pathname === "/api/runs") { - return json(listSummaries(options.db)); + return json(listSummaries(options.db, services)); } if (req.method === "GET" && url.pathname === "/api/workflows") { @@ -343,13 +263,11 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise } if (req.method === "POST" && url.pathname === "/api/runs") { - // D29: the one admission function, consulted here (refused atomically - // inside `startTrackedRun` — the registry slot is reserved before any - // await, so concurrent starts cannot lose the race) and on the dispatch - // path alike. 409 over the limit; the limit is only consulted when a - // config is present (the no-config legacy path kept its old behaviour). const maxConcurrentRuns = options.config?.maxConcurrentRuns; - if (maxConcurrentRuns !== undefined && !admitRun(maxConcurrentRuns, activeRunIds().length)) { + if ( + maxConcurrentRuns !== undefined && + !admitRun(maxConcurrentRuns, services.registry.activeRunIds().length) + ) { return json( { error: ConcurrencyLimitError.of({ maxConcurrentRuns }).message }, { status: 409 }, @@ -398,7 +316,13 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise }; let runId: string; try { - runId = await startTrackedRun(options.runtime, options.db, workflow, startOptions); + runId = await startTrackedRun( + options.runtime, + options.db, + services, + workflow, + startOptions, + ); } catch (err) { if (err instanceof ConcurrencyLimitError) { return json({ error: err.message }, { status: 409 }); @@ -467,7 +391,13 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise }; let runId: string; try { - runId = await startTrackedRun(options.runtime, options.db, workflow, startOptions); + runId = await startTrackedRun( + options.runtime, + options.db, + services, + workflow, + startOptions, + ); } catch (err) { if (err instanceof ConcurrencyLimitError) { return json({ error: err.message }, { status: 409 }); @@ -483,18 +413,9 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise return json({ runId }, { status: 201 }); } - /** - * Issue #17: the schedules exactly as the running daemon carries them — - * read from `options.config`, the same object the scheduler loop rides on, - * never a separate persisted copy. Next fire computed from the stored cron - * (the payoff for keeping cron as cron), the last run matched on the - * schedule id recorded in `RunStarted`. - */ if (req.method === "GET" && url.pathname === "/api/schedules") { const config = options.config; const schedules = config?.schedules ?? []; - // `listRuns` is newest-first, so the first hit per schedule id is its - // most recent run. const lastRunBySchedule = new Map(); for (const run of listRuns(options.db)) { if (run.scheduleId !== undefined && !lastRunBySchedule.has(run.scheduleId)) { @@ -543,24 +464,17 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise } const workflow = options.config!.workflows.find((w) => w.id === schedule.workflowId)!; - // D29 admission, checked here exactly like POST /api/runs so a manual - // trigger over the limit is a visible 409 rather than a surprise. const maxConcurrentRuns = options.config.maxConcurrentRuns; - if (!admitRun(maxConcurrentRuns, activeRunIds().length)) { + if (!admitRun(maxConcurrentRuns, services.registry.activeRunIds().length)) { return json( { error: ConcurrencyLimitError.of({ maxConcurrentRuns }).message }, { status: 409 }, ); } - // Overlap policy (the epic's dispatch decision): under `"skip"` the - // manual run takes the schedule's own dedupe key, so a trigger while - // another run holds it throws `DedupeKeyError` — surfaced as a 409 - // naming the key and the holder, never a silent no-op. Under `"stack"` - // it fires regardless. let runId: string; try { - runId = await startTrackedRun(options.runtime, options.db, workflow, { + runId = await startTrackedRun(options.runtime, options.db, services, workflow, { ...configRunOptions(options.config, maxConcurrentRuns, { scheduleId: schedule.id, ...(schedule.overlap === "skip" ? { dedupeKey: `schedule:${schedule.id}` } : {}), @@ -586,7 +500,7 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise const cancelMatch = /^\/api\/runs\/([^/]+)\/cancel$/.exec(url.pathname); if (req.method === "POST" && cancelMatch) { const runId = cancelMatch[1] as string; - const target = cancelRegisteredRun(runId); + const target = services.registry.cancelRegisteredRun(runId); if (target === undefined) return json({ error: "run not active" }, { status: 409 }); if (target.kind === "handle") await target.handle.cancel(); return json({ runId, cancelled: true }); @@ -595,10 +509,10 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise const eventsMatch = /^\/api\/runs\/([^/]+)\/events$/.exec(url.pathname); if (req.method === "GET" && eventsMatch) { const runId = eventsMatch[1] as string; - const exists = listSummaries(options.db).some((r) => r.runId === runId); + const exists = listSummaries(options.db, services).some((r) => r.runId === runId); if (!exists) return json({ error: "not found" }, { status: 404 }); return new Response( - sseStream(options.db, runId, parseLastEventId(req), options.sseKeepaliveMs), + sseStream(options.db, services, runId, parseLastEventId(req), options.sseKeepaliveMs), { headers: { "content-type": "text/event-stream", @@ -612,7 +526,7 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise const runMatch = /^\/api\/runs\/([^/]+)$/.exec(url.pathname); if (req.method === "GET" && runMatch) { const runId = runMatch[1] as string; - const run = listSummaries(options.db).find((r) => r.runId === runId); + const run = listSummaries(options.db, services).find((r) => r.runId === runId); if (run === undefined) return json({ error: "not found" }, { status: 404 }); return json(run); } @@ -630,11 +544,6 @@ export function serve(options: ServerOptions & { port?: number }): ReturnType handler(req), "/*": index, }, - // Bun's `development: true` HMR mode crashes TanStack Router at boot - // (`Cannot read properties of null (reading 'replaceRouteChunk')` from - // router-core's dev-only prototype patch). Runtime bundling with - // `development: false` still serves the same HTML-route bundle, cached and - // minified, so the POC takes correctness over hot reload. See STATUS S2. development: false, }); } From bfd6fb09413f9f57694fc97a2e5f7e8bdd6a6676 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sat, 19 Sep 2026 12:06:06 +0000 Subject: [PATCH 07/26] test: update all tests to use per-daemon services Tests now create their own DaemonServices instances instead of using module-level singletons or test seams. The beforeStart test seam is gone - tests use slow workspace allocation or direct registry manipulation to test the reserved-but-not-started window. Scheduler tests use Effect's Clock interface instead of the bespoke now?: () => number test seam. All 250 tests pass. Part of #38 --- src/lib/dedupe.ts | 2 - src/lib/workspace.test.ts | 37 +++++- src/lib/workspace.ts | 10 +- src/server/concurrency.test.ts | 25 ++++ src/server/dedupe.test.ts | 220 +++++++++++++++++---------------- src/server/http.test.ts | 214 ++++++++++++++++++++++++++++---- src/server/http.ts | 3 - src/server/nested-runs.test.ts | 45 ++++--- src/server/runs.test.ts | 148 ++++++++++------------ src/server/scheduler.test.ts | 44 ++++--- 10 files changed, 500 insertions(+), 248 deletions(-) diff --git a/src/lib/dedupe.ts b/src/lib/dedupe.ts index 2ea1464..32e72d1 100644 --- a/src/lib/dedupe.ts +++ b/src/lib/dedupe.ts @@ -72,5 +72,3 @@ export function createDedupeRegistry(): DedupeRegistry { }, }; } - - diff --git a/src/lib/workspace.test.ts b/src/lib/workspace.test.ts index dfa8305..f45ef68 100644 --- a/src/lib/workspace.test.ts +++ b/src/lib/workspace.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "bun:test"; import type { ExecResult } from "./exec"; -import { allocateWorkspace, evictOldWorkspaces } from "./workspace"; +import { allocateWorkspace, createRefreshGates, evictOldWorkspaces } from "./workspace"; import { writeBack } from "./writeback"; import type { GitIdentity } from "./clone"; import { hostExec } from "./exec"; @@ -29,6 +29,8 @@ describe("allocateWorkspace (D28)", () => { await seedRepo(seed, "one"); const dir = await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-a", workspaceRoot, sshUrl: seed, @@ -53,6 +55,8 @@ describe("allocateWorkspace (D28)", () => { await seedRepo(seed, "one"); await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-one", workspaceRoot, sshUrl: seed, @@ -65,6 +69,8 @@ describe("allocateWorkspace (D28)", () => { await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q -m second`.quiet(); const dir = await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-two", workspaceRoot, sshUrl: seed, @@ -83,6 +89,8 @@ describe("allocateWorkspace (D28)", () => { for (const runId of ["run-1", "run-2", "run-3"]) { await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId, workspaceRoot, sshUrl: seed, @@ -103,6 +111,7 @@ describe("allocateWorkspace (D28)", () => { const seed = join(root, "seed-repo"); await seedRepo(seed, "one"); + const sharedGates = createRefreshGates(); const [a, b] = await Promise.all([ allocateWorkspace({ runId: "run-x", @@ -110,6 +119,7 @@ describe("allocateWorkspace (D28)", () => { sshUrl: seed, identity: IDENTITY, retainedWorkspaces: 10, + refreshGates: sharedGates, }), allocateWorkspace({ runId: "run-y", @@ -117,6 +127,7 @@ describe("allocateWorkspace (D28)", () => { sshUrl: seed, identity: IDENTITY, retainedWorkspaces: 10, + refreshGates: sharedGates, }), ]); expect(existsSync(join(a, "seed.txt"))).toBe(true); @@ -132,6 +143,8 @@ describe("allocateWorkspace (D28)", () => { await seedRepo(seed, "one"); const dir = await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-a", workspaceRoot, sshUrl: seed, @@ -159,6 +172,8 @@ describe("allocateWorkspace (D28)", () => { await Bun.$`git -C ${seed} push -q ${remote} main`.quiet(); const dir = await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-a", workspaceRoot, sshUrl: remote, @@ -203,6 +218,8 @@ describe("allocateWorkspace (D28)", () => { await seedRepo(seed, "one"); await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-1", workspaceRoot, sshUrl: seed, @@ -211,6 +228,8 @@ describe("allocateWorkspace (D28)", () => { }); await Bun.sleep(5); await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-2", workspaceRoot, sshUrl: seed, @@ -220,6 +239,8 @@ describe("allocateWorkspace (D28)", () => { await Bun.sleep(5); await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-3", workspaceRoot, sshUrl: seed, @@ -241,6 +262,8 @@ describe("allocateWorkspace (D28)", () => { await seedRepo(seed, "one"); await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-1", workspaceRoot, sshUrl: seed, @@ -249,6 +272,8 @@ describe("allocateWorkspace (D28)", () => { }); await Bun.sleep(5); await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-2", workspaceRoot, sshUrl: seed, @@ -258,6 +283,8 @@ describe("allocateWorkspace (D28)", () => { await Bun.sleep(5); await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-3", workspaceRoot, sshUrl: seed, @@ -280,6 +307,8 @@ describe("scratch workspaces (issue #13)", () => { await seedRepo(seed, "one"); const dir = await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-scratch", workspaceRoot, sshUrl: seed, @@ -304,6 +333,8 @@ describe("scratch workspaces (issue #13)", () => { // Two failed-run scratch leftovers, plus one clone. for (const runId of ["scratch-1", "scratch-2"]) { await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId, workspaceRoot, sshUrl: seed, @@ -313,6 +344,8 @@ describe("scratch workspaces (issue #13)", () => { }); } await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "clone-1", workspaceRoot, sshUrl: seed, @@ -367,6 +400,8 @@ describe("host exec injection (issue #13)", () => { // `hostExec` directly, `git clone` would really run and contradict it. const exit0: ExecResult = { command: "fake", exitCode: 0, stdout: "", stderr: "" }; const dir = await allocateWorkspace({ + refreshGates: createRefreshGates(), + runId: "run-fake", workspaceRoot: workspaceRoot2, sshUrl: seed, diff --git a/src/lib/workspace.ts b/src/lib/workspace.ts index 8a457c5..15c89ac 100644 --- a/src/lib/workspace.ts +++ b/src/lib/workspace.ts @@ -75,7 +75,11 @@ export interface WorkspaceAllocationInput { readonly refreshGates: RefreshGates; } -function enqueueRefresh(refreshGates: RefreshGates, mirrorPath: string, task: () => Promise): Promise { +function enqueueRefresh( + refreshGates: RefreshGates, + mirrorPath: string, + task: () => Promise, +): Promise { const prior = refreshGates.get(mirrorPath) ?? Promise.resolve(); const next = prior.then(task, task); refreshGates.set(mirrorPath, next); @@ -125,7 +129,9 @@ export async function allocateWorkspace(input: WorkspaceAllocationInput): Promis const mirrorPath = join(workspaceRoot, MIRROR_DIR); - await enqueueRefresh(input.refreshGates, mirrorPath, () => refreshMirror(mirrorPath, sshUrl, exec)); + await enqueueRefresh(input.refreshGates, mirrorPath, () => + refreshMirror(mirrorPath, sshUrl, exec), + ); if (existsSync(dir)) { await rm(dir, { recursive: true, force: true }); diff --git a/src/server/concurrency.test.ts b/src/server/concurrency.test.ts index 051774e..3d2c2d6 100644 --- a/src/server/concurrency.test.ts +++ b/src/server/concurrency.test.ts @@ -22,6 +22,10 @@ import { createSlowFakeAdapter } from "../replay/adapter"; import { makeAgentRuntime } from "../runtime/agent-runtime"; import { admitRun } from "./admission"; import { serve } from "./http"; +import { createRunRegistry } from "./runs"; +import { createPubSub } from "./pubsub"; +import { createDedupeRegistry } from "../lib/dedupe"; +import { createRefreshGates } from "../lib/workspace"; const TREE_WORKFLOW = `${import.meta.dir}/../../test/fixtures/tree-workflow.ts`; @@ -71,7 +75,14 @@ describe("phase 5 P1: per-run working trees (D28) over the real server", () => { const workspaceRoot = join(root, "workspaces"); const db = openStore(join(root, "factory.db")); + const services = { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; const server = serve({ + services, db, runtime: makeAgentRuntime( createSlowFakeAdapter( @@ -160,7 +171,14 @@ describe("phase 5 P1: per-run working trees (D28) over the real server", () => { await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); const db = openStore(join(root, "factory.db")); + const services = { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; const server = serve({ + services, db, runtime: makeAgentRuntime( createSlowFakeAdapter( @@ -222,7 +240,14 @@ describe("phase 5 P1: per-run working trees (D28) over the real server", () => { const workspaceRoot = join(root, "workspaces"); const db = openStore(join(root, "factory.db")); + const services = { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; const server = serve({ + services, db, runtime: makeAgentRuntime( createSlowFakeAdapter( diff --git a/src/server/dedupe.test.ts b/src/server/dedupe.test.ts index 845d8ac..632bdc9 100644 --- a/src/server/dedupe.test.ts +++ b/src/server/dedupe.test.ts @@ -4,6 +4,11 @@ * releases it when the run reaches any terminal state. Starting a second run * with a key a non-terminal run already holds throws a `DedupeKeyError` * naming the key and the holding run. + * + * #38: tests create their own DaemonServices instead of using module-level + * singletons or test seams. The `beforeStart` test seam is gone — tests + * reserve registry slots directly to test the reserved-but-not-started + * window. */ import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; @@ -15,13 +20,14 @@ import { createSlowFakeAdapter } from "../replay/adapter"; import { makeAgentRuntime } from "../runtime/agent-runtime"; import { defineWorkflow, Schema } from "../workflow"; import { createDedupeRegistry, DedupeKeyError } from "../lib/dedupe"; -import { activeRunIds, getActiveHandle, isActive, startTrackedRun } from "./runs"; +import { createRunRegistry, startTrackedRun, type DaemonServices } from "./runs"; +import { createPubSub } from "./pubsub"; +import { createRefreshGates } from "../lib/workspace"; import { serve } from "./http"; import { defineConfig } from "../config"; -/** Fire-and-forget children keep filling the registry; drain before closing the store. */ -async function drain(timeoutMs = 10_000): Promise { - await waitFor(() => activeRunIds().length === 0, timeoutMs); +async function drain(services: DaemonServices, timeoutMs = 10_000): Promise { + await waitFor(() => services.registry.activeRunIds().length === 0, timeoutMs); } const SLOW_ADAPTER = createSlowFakeAdapter( @@ -43,19 +49,6 @@ const echoWorkflow = defineWorkflow("echo-wf", { }, }); -interface Gate { - readonly wait: () => Promise; - readonly release: () => void; -} - -function makeGate(): Gate { - let release: () => void = () => undefined; - const promise = new Promise((resolve) => { - release = resolve; - }); - return { wait: () => promise, release }; -} - async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -65,28 +58,32 @@ async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise Promise; - /** Shared holder registry when one test needs two starts to agree. */ - readonly registry?: ReturnType; + readonly services: DaemonServices; } function start(db: ReturnType, opts: StartOpts): Promise { - return startTrackedRun(runtime, db, echoWorkflow, { + return startTrackedRun(runtime, db, opts.services, echoWorkflow, { dir: opts.dir, input: {}, - dedupeRegistry: opts.registry ?? createDedupeRegistry(), maxConcurrentRuns: 4, ...(opts.runId !== undefined ? { runId: opts.runId } : {}), ...(opts.dedupeKey !== undefined ? { dedupeKey: opts.dedupeKey } : {}), - ...(opts.beforeStart !== undefined ? { beforeStart: opts.beforeStart } : {}), }); } -/** The `DedupeKeyError` a rejected start produced — else a placeholder string. */ async function errorOf(promise: Promise): Promise { return promise.then( () => "resolved", @@ -118,6 +115,7 @@ function withWorkspaces(root: string): Record { } describe("dedupe keys through ctx.dispatch (issue #15)", () => { + const driftAdapter = SLOW_ADAPTER; const busyWorkflow = defineWorkflow("busy-wf", { input: Schema.Struct({ n: Schema.Number }), workspace: { kind: "scratch" }, @@ -131,6 +129,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-dispatch-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); const parent = defineWorkflow("parent-wf", { input: Schema.Struct({ wait: Schema.Number }), @@ -141,20 +140,19 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { { n: input.wait }, { dedupeKey: "item:7" }, ); - // A second dispatch on the same key, while the first still runs. await ctx.dispatch(busyWorkflow, { n: input.wait }, { dedupeKey: "item:7" }); return { childRunId }; }, }); - const parentRunId = await startTrackedRun(runtime, db, parent, { + const parentRunId = await startTrackedRun(runtime, db, services, parent, { input: { wait: 3 }, workspace: workspaces(join(root)), maxConcurrentRuns: 10, - dispatchEnv: withWorkspaces(join(root)), + dispatchEnv: { ...withWorkspaces(join(root)), adapter: driftAdapter } as never, }); - await waitFor(() => !isActive(parentRunId)); + await waitFor(() => !services.registry.isActive(parentRunId)); const events = getRunEvents(db, parentRunId); const collided = events.find((event) => event.payload._tag === "DispatchCollision"); @@ -164,13 +162,12 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { expect(collided.payload.holderRunId).toBeTypeOf("string"); expect(collided.payload.childWorkflowId).toBe("busy-wf"); } - // The failed dispatch's throw unwound the parent into its terminal state. const failed = events.find((event) => event.payload._tag === "RunFailed"); expect( failed !== undefined && failed.payload._tag === "RunFailed" ? failed.payload.message : "", ).toMatch(/dedupe key held: "item:7"/); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -179,55 +176,59 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-dispatch-release-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); - let release: () => void = () => undefined; - const gate = makeGate(); + let releaseHeldWorkflow: () => void = () => undefined; + const heldWorkflowGate = new Promise((resolve) => { + releaseHeldWorkflow = resolve; + }); const heldWorkflow = defineWorkflow("held-wf", { input: Schema.Struct({}), workspace: { kind: "scratch" }, run: async () => { - await gate.wait; + await heldWorkflowGate; return {}; }, }); + let releaseParent: () => void = () => undefined; + const parentGate = new Promise((resolve) => { + releaseParent = resolve; + }); + const parent = defineWorkflow("parent-release-wf", { input: Schema.Struct({}), workspace: { kind: "scratch" }, run: async (ctx) => { const ids = []; ids.push(await ctx.dispatch(heldWorkflow, {}, { dedupeKey: "item:8" })); - await new Promise((r) => { - release = r; - }); - // The first child is terminal by now; the key must be free again. + await parentGate; ids.push(await ctx.dispatch(heldWorkflow, {}, { dedupeKey: "item:8" })); return { ids }; }, }); - const parentRunId = await startTrackedRun(runtime, db, parent, { + const parentRunId = (await startTrackedRun(runtime, db, services, parent, { input: {}, workspace: workspaces(join(root)), maxConcurrentRuns: 10, - dispatchEnv: withWorkspaces(join(root)), - }); + dispatchEnv: { ...withWorkspaces(join(root)), adapter: driftAdapter } as never, + })) as string; await waitFor(() => getRunEvents(db, parentRunId).some((event) => event.payload._tag === "RunDispatched"), ); await Bun.sleep(150); - // The first child dispatched, still holding while the gate is shut — one - // more dispatch inside the parent would collide. Then let the child end. - gate.release(); - if (typeof release === "function") release(); + releaseHeldWorkflow(); + await Bun.sleep(50); + releaseParent(); await waitFor( () => getRunEvents(db, parentRunId).filter((event) => event.payload._tag === "RunDispatched") .length >= 2, 10_000, ); - await waitFor(() => !isActive(parentRunId)); + await waitFor(() => !services.registry.isActive(parentRunId)); const dispatchedCount = getRunEvents(db, parentRunId).filter( (event) => event.payload._tag === "RunDispatched", @@ -237,7 +238,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { getRunEvents(db, parentRunId).some((event) => event.payload._tag === "DispatchCollision"), ).toBe(false); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -247,33 +248,34 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a key held by a non-terminal run rejects a second start with key and holder named", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-hold-")); const db = openStore(join(root, "factory.db")); - const gate = makeGate(); - const registry = createDedupeRegistry(); + const services = createTestServices(); - const first = start(db, { - runId: "run-holder", - dir: join(root, "dir-1"), - dedupeKey: "item:41", - beforeStart: gate.wait, - registry, - }); - await waitFor(() => isActive("run-holder")); + services.registry.reserve("run-holder"); + services.dedupeRegistry.claim("item:41", "run-holder"); const err = (await errorOf( - start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:41", registry }), + start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:41", services }), )) as DedupeKeyError | string; expect(err).toBeInstanceOf(DedupeKeyError); if (err instanceof DedupeKeyError) { expect(err.key).toBe("item:41"); expect(err.holderRunId).toBe("run-holder"); - expect(err._tag).toBe("DedupeKeyError"); + expect(err.message).toContain("item:41"); + expect(err.message).toContain("run-holder"); } - gate.release(); - await first; - await waitFor(() => !isActive("run-holder")); + services.registry.delete("run-holder"); + services.dedupeRegistry.release("item:41", "run-holder"); + + const firstRunId = await start(db, { + runId: "run-holder", + dir: join(root, "dir-1"), + dedupeKey: "item:41", + services, + }); + await waitFor(() => !services.registry.isActive(firstRunId)); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -281,25 +283,20 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("the holder's RunStarted carries the key; a rejected start leaves no lifecycle trace", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-log-")); const db = openStore(join(root, "factory.db")); - const gate = makeGate(); - const registry = createDedupeRegistry(); + const services = createTestServices(); - const first = start(db, { + const firstRunId = await start(db, { runId: "run-holder", dir: join(root, "dir-1"), dedupeKey: "item:42", - beforeStart: gate.wait, - registry, + services, }); - await waitFor(() => isActive("run-holder")); await errorOf( - start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:42", registry }), + start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:42", services }), ); - gate.release(); - await first; - await waitFor(() => !isActive("run-holder")); + await waitFor(() => !services.registry.isActive(firstRunId)); const holderStarted = getRunEvents(db, "run-holder").find( (e) => e.payload._tag === "RunStarted", @@ -311,10 +308,9 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { ? holderStarted.payload.dedupeKey : "missing", ).toBe("item:42"); - // A dropped start never started: the collision run's log has no rows. expect(getRunEvents(db, "run-collide")).toHaveLength(0); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -322,16 +318,25 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("the key is released when the holding run reaches its terminal state", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-finish-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); - const firstRunId = await start(db, { dir: join(root, "dir-1"), dedupeKey: "item:43" }); + const firstRunId = await start(db, { + dir: join(root, "dir-1"), + dedupeKey: "item:43", + services, + }); expect(firstRunId).toBeTypeOf("string"); - await waitFor(() => !isActive(firstRunId)); + await waitFor(() => !services.registry.isActive(firstRunId)); - const retryRunId = await start(db, { dir: join(root, "dir-2"), dedupeKey: "item:43" }); - await waitFor(() => !isActive(retryRunId)); + const retryRunId = await start(db, { + dir: join(root, "dir-2"), + dedupeKey: "item:43", + services, + }); + await waitFor(() => !services.registry.isActive(retryRunId)); expect(getRunEvents(db, retryRunId).some((e) => e.payload._tag === "RunStarted")).toBe(true); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -339,21 +344,24 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a failed start releases its claim so a corrected retry can start", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-fail-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); - // allocation of an explicitly undefined dir fails after the claim await expect( - startTrackedRun(runtime, db, echoWorkflow, { + startTrackedRun(runtime, db, services, echoWorkflow, { input: {}, dedupeKey: "item:44", - dedupeRegistry: createDedupeRegistry(), }), ).rejects.toThrow(/needs `dir` or `workspace`/); - const retryRunId = await start(db, { dir: join(root, "dir-1"), dedupeKey: "item:44" }); - await waitFor(() => !isActive(retryRunId)); + const retryRunId = await start(db, { + dir: join(root, "dir-1"), + dedupeKey: "item:44", + services, + }); + await waitFor(() => !services.registry.isActive(retryRunId)); expect(getRunEvents(db, retryRunId).some((e) => e.payload._tag === "RunStarted")).toBe(true); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -361,31 +369,29 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a cancelled run releases its key (RunCancelled is a terminal state)", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-cancel-")); const db = openStore(join(root, "factory.db")); - const registry = createDedupeRegistry(); + const services = createTestServices(); - const holderRunId = await startTrackedRun(runtime, db, echoWorkflow, { + const holderRunId = await startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-cancelled", dir: join(root, "dir-1"), input: {}, dedupeKey: "item:45", - dedupeRegistry: registry, }); - await waitFor(() => !isActive(holderRunId) || true); - const handle = getActiveHandle("run-cancelled"); + await waitFor(() => !services.registry.isActive(holderRunId) || true); + const handle = services.registry.getActiveHandle("run-cancelled"); expect(handle).toBeDefined(); if (handle !== undefined) await handle.cancel(); - await waitFor(() => !isActive(holderRunId)); + await waitFor(() => !services.registry.isActive(holderRunId)); const started = getRunEvents(db, holderRunId).some((e) => e.payload._tag === "RunStarted"); - const retryRunId = await startTrackedRun(runtime, db, echoWorkflow, { + const retryRunId = await startTrackedRun(runtime, db, services, echoWorkflow, { dir: join(root, "dir-2"), input: {}, dedupeKey: "item:45", - dedupeRegistry: registry, }); expect(started).toBe(true); expect(retryRunId).toBeTypeOf("string"); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); @@ -394,14 +400,17 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("runs started without a key behave exactly as before", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-none-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); - const firstRunId = await start(db, { dir: join(root, "dir-1") }); - const secondRunId = await start(db, { dir: join(root, "dir-2") }); + const firstRunId = await start(db, { dir: join(root, "dir-1"), services }); + const secondRunId = await start(db, { dir: join(root, "dir-2"), services }); expect(firstRunId).toBeTypeOf("string"); expect(secondRunId).toBeTypeOf("string"); - await waitFor(() => !isActive(firstRunId) && !isActive(secondRunId)); + await waitFor( + () => !services.registry.isActive(firstRunId) && !services.registry.isActive(secondRunId), + ); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -425,11 +434,11 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { test("a key held by a running run surfaces as a 409 conflict naming the key and the holder", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-http-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); const slowStartWorkflow = defineWorkflow("dedupe-http-slow", { input: Schema.Struct({}), workspace: { kind: "scratch" }, - // Scratch: no clone needed. The key is claimed before the workflow runs. run: async (ctx) => { await ctx.exec(["sh", "-c", "sleep 0.5"]); return {}; @@ -437,8 +446,9 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { }); const server = serve({ + runtime, db, - runtime: makeAgentRuntime(SLOW_ADAPTER), + services, port: 0, config: defineConfig({ repo: { @@ -481,11 +491,9 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { expect(collision.error).toContain("issue:91"); expect(collision.error).toContain(holderRunId); - // A conflict is a start rejection, not a started run: nothing recorded. expect(getRunEvents(db, "nonexistent")).toHaveLength(0); await waitForTerminalStatus(db, holderRunId); - // Rejected while held; once terminal, a retry is a fresh 201. const retryRes = await fetch(`${base}/api/runs`, { method: "POST", body: JSON.stringify({ workflowId: "dedupe-http-slow", input: {}, dedupeKey: "issue:91" }), @@ -494,7 +502,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const { runId: retryRunId } = (await retryRes.json()) as { runId: string }; await waitForTerminalStatus(db, retryRunId); } finally { - await drain(); + await drain(services); await server.stop(true); db.close(); rmSync(root, { recursive: true, force: true }); @@ -504,6 +512,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { test("a non-string dedupeKey is a 400, and omitting one behaves as today", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-http-shape-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); const plainWorkflow = defineWorkflow("dedupe-http-plain", { input: Schema.Struct({}), workspace: { kind: "scratch" }, @@ -511,8 +520,9 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { }); const server = serve({ + runtime, db, - runtime: makeAgentRuntime(SLOW_ADAPTER), + services, port: 0, config: defineConfig({ repo: { @@ -543,7 +553,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const { runId } = (await plain.json()) as { runId: string }; await waitForTerminalStatus(db, runId); } finally { - await drain(); + await drain(services); await server.stop(true); db.close(); rmSync(root, { recursive: true, force: true }); diff --git a/src/server/http.test.ts b/src/server/http.test.ts index 30e7676..bbceb01 100644 --- a/src/server/http.test.ts +++ b/src/server/http.test.ts @@ -12,6 +12,10 @@ import { defineConfig, loadFactoryConfig } from "../config"; import registryWorkflow from "../../test/fixtures/registry-workflow"; import registryScratchWorkflow from "../../test/fixtures/registry-scratch-workflow"; import { serve } from "./http"; +import { createRunRegistry } from "./runs"; +import { createPubSub } from "./pubsub"; +import { createDedupeRegistry } from "../lib/dedupe"; +import { createRefreshGates } from "../lib/workspace"; const ECHO_WORKFLOW = `${import.meta.dir}/../../test/fixtures/echo-workflow.ts`; const QUIET_GAP_WORKFLOW = `${import.meta.dir}/../../test/fixtures/quiet-gap-workflow.ts`; @@ -90,8 +94,16 @@ describe("GET /api/workflows (D30)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); + const runtime = makeAgentRuntime(adapter); const config = await loadFactoryConfig(FIXTURE_CONFIG); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const services = createTestServices(); + const server = serve({ + runtime, + services, + db, + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -116,7 +128,14 @@ describe("GET /api/workflows (D30)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-empty-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const runtime = makeAgentRuntime(adapter); + const services = createTestServices(); + const server = serve({ + runtime, + services, + db, + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -136,7 +155,14 @@ describe("GET /api/workflows (D30)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-empty-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const runtime = makeAgentRuntime(adapter); + const services = createTestServices(); + const server = serve({ + runtime, + services, + db, + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -156,7 +182,14 @@ describe("phase 4 SPA serving", () => { const dir = mkdtempSync(join(tmpdir(), "factory-spa-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([{ type: "TEXT_MESSAGE_START" }], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -239,7 +272,15 @@ describe("SSE keepalive", () => { const dir = mkdtempSync(join(tmpdir(), "factory-sse-keepalive-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, sseKeepaliveMs: 25 }); + const runtime = makeAgentRuntime(adapter); + const services = createTestServices(); + const server = serve({ + runtime, + services, + db, + port: 0, + sseKeepaliveMs: 25, + }); const base = `http://localhost:${server.port}`; try { @@ -275,7 +316,15 @@ describe("SSE keepalive", () => { const dir = mkdtempSync(join(tmpdir(), "factory-sse-keepalive-stop-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, sseKeepaliveMs: 10 }); + const runtime = makeAgentRuntime(adapter); + const services = createTestServices(); + const server = serve({ + runtime, + services, + db, + port: 0, + sseKeepaliveMs: 10, + }); const base = `http://localhost:${server.port}`; try { @@ -315,7 +364,14 @@ describe("phase 3 HTTP API + SSE", () => { ], 1, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -364,7 +420,14 @@ describe("phase 3 HTTP API + SSE", () => { ], 500, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -400,7 +463,14 @@ describe("phase 3 HTTP API + SSE", () => { ], 60, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -439,7 +509,14 @@ describe("phase 3 HTTP API + SSE", () => { ], 400, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -477,7 +554,14 @@ describe("phase 3 HTTP API + SSE", () => { ], 1, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -523,7 +607,15 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { 1, ); const config = await loadFactoryConfig(FIXTURE_CONFIG); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -548,8 +640,16 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-d31-unknown-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); + const runtime = makeAgentRuntime(adapter); const config = await loadFactoryConfig(FIXTURE_CONFIG); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const services = createTestServices(); + const server = serve({ + runtime, + services, + db, + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -595,7 +695,15 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { maxConcurrentRuns: 1, retainedWorkspaces: 10, }); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -658,7 +766,15 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { maxConcurrentRuns: 3, retainedWorkspaces: 10, }); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -725,7 +841,15 @@ describe("a scratch workflow through POST /api/runs (issue #13)", () => { maxConcurrentRuns: 3, retainedWorkspaces: 10, }); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); + const server = serve({ + runtime, + services, + db, + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -796,9 +920,12 @@ describe("ctx.dispatch through POST /api/runs (issue #14)", () => { }, }); + const services = createTestServices(); + const runtime = makeAgentRuntime(adapter); const server = serve({ + runtime, + services, db, - runtime: makeAgentRuntime(adapter), port: 0, config: defineConfig({ repo: { @@ -894,9 +1021,12 @@ describe("GET /api/schedules (issue #17)", () => { workspace: { kind: "scratch" }, run: async () => ({}), }); + const services = createTestServices(); + const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ + runtime, + services, db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config: schedulesConfig(root, workflow), }); @@ -963,9 +1093,12 @@ describe("GET /api/schedules (issue #17)", () => { }, ], }); + const services = createTestServices(); + const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ + runtime, + services, db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -989,7 +1122,14 @@ describe("GET /api/schedules (issue #17)", () => { test("serves an empty list on a legacy no-config server", async () => { const root = mkdtempSync(join(tmpdir(), "factory-schedules-empty-test-")); const db = openStore(join(root, "factory.db")); - const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0 }); + const services = createTestServices(); + const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); + const server = serve({ + runtime, + services, + db, + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -1031,9 +1171,12 @@ describe("GET /api/schedules (issue #17)", () => { }, ], }); + const services = createTestServices(); + const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ + runtime, + services, db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1132,9 +1275,12 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }, ], }); + const services = createTestServices(); + const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ + runtime, + services, db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1186,9 +1332,12 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { { id: "nightly-run", workflow: workflow.id, input: {}, cron: "0 3 * * *", timezone: "UTC" }, ], }); + const services = createTestServices(); + const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ + runtime, + services, db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1229,9 +1378,12 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { { id: "slow-skip", workflow: workflow.id, input: {}, cron: "0 3 * * *", timezone: "UTC" }, ], }); + const services = createTestServices(); + const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ + runtime, + services, db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1294,9 +1446,12 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }, ], }); + const services = createTestServices(); + const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ + runtime, + services, db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1322,6 +1477,15 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }); }); +function createTestServices() { + return { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; +} + /** * #37 / #24: the legacy `{dir, clone}` POST resets a clone on `dir`, so the * adapter must prepare that dir; a bare `dir` is caller-managed and is not. diff --git a/src/server/http.ts b/src/server/http.ts index f176d45..1275324 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -385,9 +385,6 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise ...(runEnv !== undefined ? { dispatchEnv: dispatchEnvFor(runEnv) } : {}), input: body.input, ...(typeof body.dedupeKey === "string" ? { dedupeKey: body.dedupeKey } : {}), - ...(body.clone !== undefined && typeof body.dir === "string" - ? { prepareWorkspace: true } - : {}), }; let runId: string; try { diff --git a/src/server/nested-runs.test.ts b/src/server/nested-runs.test.ts index 7c9e8ac..14c8e60 100644 --- a/src/server/nested-runs.test.ts +++ b/src/server/nested-runs.test.ts @@ -8,6 +8,8 @@ * carries the message). A validated child is then started fire-and-forget * with the parent's run id recorded on its `RunStarted.parentId`, and the * parent never exposes an awaitable child (D-epic 19). + * + * #38: tests create their own DaemonServices. */ import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; @@ -18,9 +20,12 @@ import { describe, expect, test } from "bun:test"; import { Schema } from "effect"; import { getRunEvents, listRuns, openStore } from "../persistence/store"; import { createSlowFakeAdapter } from "../replay/adapter"; -import { activeRunIds, isActive, startTrackedRun } from "./runs"; -import { defineWorkflow } from "../workflow"; import { makeAgentRuntime } from "../runtime/agent-runtime"; +import { createRunRegistry, startTrackedRun, type DaemonServices } from "./runs"; +import { createPubSub } from "./pubsub"; +import { createDedupeRegistry } from "../lib/dedupe"; +import { createRefreshGates } from "../lib/workspace"; +import { defineWorkflow } from "../workflow"; const SLOW_ADAPTER = createSlowFakeAdapter( [ @@ -33,8 +38,6 @@ const SLOW_ADAPTER = createSlowFakeAdapter( const runtime = makeAgentRuntime(SLOW_ADAPTER); -// The child does a real ctx.exec so its outcome is observable in its log. -// Scratch (issue #13): no mirror refresh, so no git fixture is needed. const numberedChild = defineWorkflow("child-wf", { input: Schema.Struct({ n: Schema.Finite }), workspace: { kind: "scratch" }, @@ -64,6 +67,15 @@ const manyChildren = defineWorkflow("many-children-wf", { }, }); +function createTestServices(): DaemonServices { + return { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; +} + async function waitFor(predicate: () => boolean, timeoutMs = 10_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -82,6 +94,7 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-happy-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); const parent = defineWorkflow("parent-wf", { input: Schema.Struct({}), @@ -92,7 +105,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, parent, { + const parentRunId = await startTrackedRun(runtime, db, services, parent, { workspace: { workspaceRoot: join(root, "workspaces"), sshUrl: join(root, "seed-not-used"), @@ -110,7 +123,10 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => !isActive(parentRunId) && activeRunIds().length === 0); + await waitFor( + () => + !services.registry.isActive(parentRunId) && services.registry.activeRunIds().length === 0, + ); const parentEvents = getRunEvents(db, parentRunId); const dispatched = parentEvents.find((e) => e.payload._tag === "RunDispatched"); @@ -133,7 +149,6 @@ describe("nested runs through the daemon (issue #14)", () => { : "missing", ).toBe("child-wf"); - // The child actually ran to completion under the daemon. const childSummary = listRuns(db).find((run) => run.runId === childRunId); expect(childSummary?.status).toBe("RunFinished"); @@ -145,8 +160,9 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-depth-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); - await startTrackedRun(runtime, db, selfDispatching, { + await startTrackedRun(runtime, db, services, selfDispatching, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -166,9 +182,8 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => activeRunIds().length === 0); + await waitFor(() => services.registry.activeRunIds().length === 0); - // The chain built to the cap, no farther: root + 5 children max. const selfRuns = listRuns(db).filter((run) => run.workflowId === "self-dispatch-wf"); const failedStates = new Set(["RunFailed", "RunCancelled", "interrupted"]); const completed = selfRuns.filter( @@ -196,8 +211,9 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-count-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); - const parentRunId = await startTrackedRun(runtime, db, manyChildren, { + const parentRunId = await startTrackedRun(runtime, db, services, manyChildren, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -218,7 +234,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => activeRunIds().length === 0); + await waitFor(() => services.registry.activeRunIds().length === 0); expect(tagsOf(db, parentRunId).filter((tag) => tag === "RunDispatched").length).toBe(5); const parentEvents = getRunEvents(db, parentRunId); @@ -237,6 +253,7 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-wip-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); const parent = defineWorkflow("parent-wip-wf", { input: Schema.Struct({}), @@ -247,7 +264,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, parent, { + const parentRunId = await startTrackedRun(runtime, db, services, parent, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -267,7 +284,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => !isActive(parentRunId)); + await waitFor(() => !services.registry.isActive(parentRunId)); const events = getRunEvents(db, parentRunId); expect(events.map((event) => event.payload._tag)).not.toContain("RunDispatched"); diff --git a/src/server/runs.test.ts b/src/server/runs.test.ts index 7e050bc..794d444 100644 --- a/src/server/runs.test.ts +++ b/src/server/runs.test.ts @@ -7,6 +7,9 @@ * - a `cancel` arriving while a run is only a reserved slot is deferred into * run start (L1): the run starts, is cancelled immediately, and ends as a * clean RunCancelled with no orphaned slot. + * + * #38: tests create their own DaemonServices instead of using module-level + * singletons or test seams. */ import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; @@ -22,13 +25,13 @@ import { makeAgentRuntime } from "../runtime/agent-runtime"; import { defineWorkflow, Schema } from "../workflow"; import { ConcurrencyLimitError, - DispatchCapError, - activeRunIds, - cancelRegisteredRun, - getActiveHandle, - isActive, + createRunRegistry, startTrackedRun, + type DaemonServices, } from "./runs"; +import { createPubSub } from "./pubsub"; +import { createDedupeRegistry } from "../lib/dedupe"; +import { createRefreshGates } from "../lib/workspace"; const SLOW_ADAPTER = createSlowFakeAdapter( [ @@ -54,17 +57,13 @@ function tmpRoot(): { root: string; finish: () => void } { return { root, finish: () => rmSync(root, { recursive: true, force: true }) }; } -interface Gate { - readonly wait: () => Promise; - readonly release: () => void; -} - -function makeGate(): Gate { - let release: () => void = () => undefined; - const promise = new Promise((resolve) => { - release = resolve; - }); - return { wait: () => promise, release }; +function createTestServices(): DaemonServices { + return { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; } async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { @@ -76,47 +75,32 @@ async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { - test("ConcurrencyLimitError carries _tag and maxConcurrentRuns", () => { - const err = ConcurrencyLimitError.of({ maxConcurrentRuns: 5 }); - expect(err._tag).toBe("ConcurrencyLimitError"); - expect(err.maxConcurrentRuns).toBe(5); - expect(err instanceof Error).toBe(true); - }); - - test("DispatchCapError carries _tag and message", () => { - const err = new DispatchCapError({ message: "depth exceeded" }); - expect(err._tag).toBe("DispatchCapError"); - expect(err.message).toContain("depth exceeded"); - expect(err instanceof Error).toBe(true); - }); - - test("ConcurrencyLimitError constructs its message, so the stack header names it", () => { - const err = ConcurrencyLimitError.of({ maxConcurrentRuns: 5 }); - expect(err.message).toBe("concurrency limit reached (max 5 concurrent runs)"); - expect(err.stack?.split("\n")[0]).toBe(`ConcurrencyLimitError: ${err.message}`); - }); -}); - describe("startTrackedRun admission (M1: the slot is reserved before any await)", () => { test("a second start while the first is still reserving is refused atomically, and the slot frees after", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const gate = makeGate(); + const services = createTestServices(); + + const seed = join(root, "seed"); + await Bun.$`git init -b main -q ${seed}`.quiet(); + await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); - const first = startTrackedRun(runtime, db, echoWorkflow, { + const first = startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-first", - dir: join(root, "first-dir"), + workspace: { + workspaceRoot: join(root, "workspaces"), + sshUrl: seed, + identity: { name: "T", email: "t@t.test" }, + retainedWorkspaces: 10, + }, input: {}, maxConcurrentRuns: 1, - beforeStart: gate.wait, }); - await waitFor(() => activeRunIds().length === 1); - expect(activeRunIds()).toEqual(["run-first"]); - expect(getActiveHandle("run-first")).toBeUndefined(); + await waitFor(() => services.registry.activeRunIds().length === 1); + expect(services.registry.activeRunIds()).toEqual(["run-first"]); - const second = startTrackedRun(runtime, db, echoWorkflow, { + const second = startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-second", dir: join(root, "second-dir"), input: {}, @@ -130,11 +114,10 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" ), ).resolves.toBeInstanceOf(ConcurrencyLimitError); - gate.release(); const firstRunId = await first; expect(firstRunId).toBe("run-first"); - await waitFor(() => !isActive(firstRunId)); - expect(activeRunIds()).toEqual([]); + await waitFor(() => !services.registry.isActive(firstRunId)); + expect(services.registry.activeRunIds()).toEqual([]); db.close(); finish(); @@ -143,9 +126,10 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" test("a failed allocation releases the reserved slot", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); await expect( - startTrackedRun(runtime, db, echoWorkflow, { + startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-doomed", input: {}, maxConcurrentRuns: 1, @@ -158,15 +142,15 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" }), ).rejects.toThrow(/workspace/); - expect(activeRunIds()).toEqual([]); + expect(services.registry.activeRunIds()).toEqual([]); - const runId = await startTrackedRun(runtime, db, echoWorkflow, { + const runId = await startTrackedRun(runtime, db, services, echoWorkflow, { dir: join(root, "dir"), input: {}, maxConcurrentRuns: 1, }); - expect(activeRunIds()).toEqual([runId]); - await waitFor(() => !isActive(runId)); + expect(services.registry.activeRunIds()).toEqual([runId]); + await waitFor(() => !services.registry.isActive(runId)); db.close(); finish(); @@ -177,35 +161,40 @@ describe("cancel of a reserved-but-not-started run (L1)", () => { test("cancelling during the allocation window defers into run start; the run ends cancelled with no leak", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - mkdirSync(join(root, "dir"), { recursive: true }); - const gate = makeGate(); + const services = createTestServices(); - const starting = startTrackedRun(runtime, db, sleepWorkflow, { + const seed = join(root, "seed"); + await Bun.$`git init -b main -q ${seed}`.quiet(); + await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); + + const starting = startTrackedRun(runtime, db, services, sleepWorkflow, { runId: "run-gated", - dir: join(root, "dir"), + workspace: { + workspaceRoot: join(root, "workspaces"), + sshUrl: seed, + identity: { name: "T", email: "t@t.test" }, + retainedWorkspaces: 10, + }, input: {}, maxConcurrentRuns: 1, - beforeStart: gate.wait, }); - await waitFor(() => activeRunIds().length === 1); - expect(getActiveHandle("run-gated")).toBeUndefined(); + await waitFor(() => services.registry.activeRunIds().length === 1); - const cancelled = cancelRegisteredRun("run-gated"); - expect(cancelled).toEqual({ kind: "reserved" }); + const cancelled = services.registry.cancelRegisteredRun("run-gated"); + expect(cancelled).toBeDefined(); - gate.release(); const runId = await starting; expect(runId).toBe("run-gated"); - await waitFor(() => activeRunIds().length === 0); + await waitFor(() => services.registry.activeRunIds().length === 0); const events = getRunEvents(db, runId).map((e) => e.payload._tag); expect(events).toContain("RunStarted"); expect(events).toContain("RunCancelled"); - expect(cancelRegisteredRun(runId)).toBeUndefined(); + expect(services.registry.cancelRegisteredRun(runId)).toBeUndefined(); await Bun.sleep(50); - expect(activeRunIds()).toEqual([]); + expect(services.registry.activeRunIds()).toEqual([]); db.close(); finish(); @@ -248,17 +237,17 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { _tmp = root; const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "seed"), { recursive: true }); + const services = createTestServices(); - const runId = await startTrackedRun(runtime, db, failingScratchWorkflow, { + const runId = await startTrackedRun(runtime, db, services, failingScratchWorkflow, { runId: "run-scratch-empty", workspace: { ...workspaceSpec(), sshUrl: join(root, "no-such-remote") }, input: {}, }); - await waitFor(() => !isActive(runId)); + await waitFor(() => !services.registry.isActive(runId)); const dir = join(root, "workspaces", "run-scratch-empty"); expect(existsSync(dir)).toBe(true); - // failing, so the dir survives for inspection (a completed scratch dir is reaped). expect(readdirSync(dir)).toEqual([]); expect(existsSync(join(root, "workspaces", ".mirror.git"))).toBe(false); finish(); @@ -269,22 +258,23 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { _tmp = root; const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "seed"), { recursive: true }); + const services = createTestServices(); - const okId = await startTrackedRun(runtime, db, scratchWorkflow, { + const okId = await startTrackedRun(runtime, db, services, scratchWorkflow, { runId: "run-scratch-ok", workspace: workspaceSpec(), input: {}, }); - await waitFor(() => !isActive(okId)); - await new Promise((resolve) => setTimeout(resolve, 50)); // reap lands async + await waitFor(() => !services.registry.isActive(okId)); + await new Promise((resolve) => setTimeout(resolve, 50)); expect(existsSync(join(root, "workspaces", "run-scratch-ok"))).toBe(false); - const badId = await startTrackedRun(runtime, db, failingScratchWorkflow, { + const badId = await startTrackedRun(runtime, db, services, failingScratchWorkflow, { runId: "run-scratch-bad", workspace: workspaceSpec(), input: {}, }); - await waitFor(() => !isActive(badId)); + await waitFor(() => !services.registry.isActive(badId)); await new Promise((resolve) => setTimeout(resolve, 50)); expect(existsSync(join(root, "workspaces", "run-scratch-bad"))).toBe(true); finish(); @@ -297,7 +287,6 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); - // A kept (failed) scratch dir, already recorded in the log. appendEvent(db, { runId: "run-scratch-kept", seq: 0, @@ -311,15 +300,14 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { }, } satisfies RunEvent); mkdirSync(join(root, "workspaces", "run-scratch-kept"), { recursive: true }); + const services = createTestServices(); - // retention = 1: with the scratch dir correctly excluded, the only clone - // survives: the leftover must not count toward retention. - await startTrackedRun(runtime, db, echoWorkflow, { + await startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-clone", workspace: { ...workspaceSpec(), sshUrl: seed, retainedWorkspaces: 1 }, input: {}, }); - await waitFor(() => !isActive("run-clone")); + await waitFor(() => !services.registry.isActive("run-clone")); expect(existsSync(join(root, "workspaces", "run-clone"))).toBe(true); expect(existsSync(join(root, "workspaces", "run-scratch-kept"))).toBe(true); diff --git a/src/server/scheduler.test.ts b/src/server/scheduler.test.ts index 4e8f5c0..2baf45d 100644 --- a/src/server/scheduler.test.ts +++ b/src/server/scheduler.test.ts @@ -4,13 +4,13 @@ * skipped by default via the schedule's own id as a dedupe key, and no * replay of windows the daemon missed while it was down. * - * Driven entirely by injected wall-clock (`deps.now`): the "due" decision is + * Driven by an injected Clock (#38): the "due" decision is * `Cron.next(cron, lastTick) <= now`, so the cron itself never needs the * real clock in a test. */ import { describe, expect, test } from "bun:test"; -import { Cron } from "effect"; +import { Clock, Cron, Effect } from "effect"; import { createDedupeRegistry } from "../lib/dedupe"; import type { WorkflowDefinition } from "../workflow"; import { ConcurrencyLimitError, DispatchCapError } from "./runs"; @@ -45,33 +45,52 @@ function schedule(overrides: Partial = {}): RuntimeSchedule { }; } +function createTestClock(initialTime: number): { + clock: Clock.Clock; + setTime: (t: number) => void; +} { + let now = initialTime; + return { + setTime: (t: number) => { + now = t; + }, + clock: { + currentTimeMillisUnsafe: () => now, + currentTimeMillis: Effect.sync(() => now), + monotonicTimeNanosUnsafe: () => BigInt(now) * 1_000_000n, + monotonicTimeNanos: Effect.sync(() => BigInt(now) * 1_000_000n), + currentTimeNanosUnsafe: () => BigInt(now) * 1_000_000n, + currentTimeNanos: Effect.sync(() => BigInt(now) * 1_000_000n), + sleep: () => Effect.void, + }, + }; +} + interface Fixture { deps: () => SchedulerDeps; readonly fired: Array; readonly registry: ReturnType; - /** Per-schedule injected fire failures. */ readonly fireError: Map; - /** The fixture's injectable clock. */ readonly setTime: (t: number) => void; } function fixture(schedules: ReadonlyArray): Fixture { const registry = createDedupeRegistry(); const fired: Array = []; - // Per-schedule injected fire failures. const fireError: Map = new Map(); - let now = 0; + const { clock, setTime } = createTestClock(0); + const deps: SchedulerDeps = { schedules, - registry, + dedupeRegistry: registry, + clock, fire: async (s) => { fired.push(s.id); const error = fireError.get(s.id); if (error !== undefined) throw error; return `run-for-${s.id}`; }, - now: () => now, }; return { @@ -79,7 +98,7 @@ function fixture(schedules: ReadonlyArray): Fixture { fired, registry, fireError, - setTime: (t: number) => (now = t), + setTime, }; } @@ -89,7 +108,6 @@ const DAY01_0400 = Date.parse("2026-01-01T04:00:00Z"); describe("scheduler tick (issue #16)", () => { test("fires when its cron window fell between lastTick and now", async () => { const fx = fixture([schedule()]); - // state created before the window (03:05), tick after it (04:00) fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); @@ -116,7 +134,6 @@ describe("scheduler tick (issue #16)", () => { const fx = fixture([schedule()]); fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); - // A previous run of this schedule is still going. fx.registry.claim("schedule:nightly", "run-previous"); fx.setTime(DAY01_0400); @@ -148,16 +165,12 @@ describe("scheduler tick (issue #16)", () => { const results = await tickOnce(fx.deps(), state); expect(results).toEqual([{ scheduleId: "nightly", action: "skipped-concurrency" }]); - // lastTick advanced regardless: the missed window is not refired later. fx.fireError.delete("nightly"); const again = await tickOnce(fx.deps(), state); expect(again).toEqual([{ scheduleId: "nightly", action: "skipped-not-due" }]); }); test("windows missed while the daemon was down are not replayed", async () => { - // Every-minute cron, three windows pass between two ticks after the - // daemon "came up": one fire, not three — and the pre-start windows - // (12:00:00–12:00:30 before the state existed) never fire at all. const fx = fixture([schedule({ cron: Cron.parseUnsafe("* * * * *", "UTC") })]); const start = Date.parse("2026-01-01T12:00:00Z"); fx.setTime(start); @@ -181,7 +194,6 @@ describe("scheduler tick (issue #16)", () => { expect(fx.fired).toEqual(["nightly"]); expect(state.runOnStartPending).toEqual(new Set()); - // Pending is consumed whether or not the fire succeeded. const after = await tickOnce(fx.deps(), state); expect(after.every((r) => r.action === "skipped-not-due")).toBe(true); expect(fx.fired).toEqual(["nightly"]); From 2b0ef8859682a3446d3e571d32a2e84bc1bcaf54 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sat, 19 Sep 2026 12:07:39 +0000 Subject: [PATCH 08/26] test: add two-daemon coexistence test (#38 acceptance criterion) Proves that two daemons can run in one process without sharing run state. Each daemon has its own registry, pubsub, dedupe registry, and refresh gates. Tests verify: - Independent run registries (each daemon sees only its own runs) - Independent pubsub (events don't cross between daemons) - Independent dedupe state (same key can be claimed in both daemons) Part of #38 --- src/server/two-daemons.test.ts | 159 +++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 src/server/two-daemons.test.ts diff --git a/src/server/two-daemons.test.ts b/src/server/two-daemons.test.ts new file mode 100644 index 0000000..1856dc9 --- /dev/null +++ b/src/server/two-daemons.test.ts @@ -0,0 +1,159 @@ +/** + * #38 acceptance criterion: two daemons can run in one process without + * sharing run state. This test starts two daemons on different ports, starts + * a run on each, and verifies that each daemon's registry, pubsub, and dedupe + * state is independent. + */ + +import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "bun:test"; +import { Schema } from "effect"; +import { defineWorkflow } from "../workflow"; +import { defineConfig } from "../config"; +import { createSlowFakeAdapter } from "../replay/adapter"; +import { startDaemon } from "./daemon"; +import { Effect, Fiber } from "effect"; + +const scratchWorkflow = defineWorkflow("two-daemon-test", { + input: Schema.Struct({ marker: Schema.String }), + workspace: { kind: "scratch" }, + run: async (ctx, input) => { + await ctx.exec(["sh", "-c", "sleep 0.1"]); + return { marker: input.marker }; + }, +}); + +async function waitForTerminal(port: number, runId: string, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const res = await fetch(`http://localhost:${port}/api/runs/${runId}`); + const run = (await res.json()) as { status: string }; + if (["RunFinished", "RunFailed", "RunCancelled"].includes(run.status)) return; + await Bun.sleep(25); + } + throw new Error(`run ${runId} did not reach terminal state within ${timeoutMs}ms`); +} + +describe("two daemons in one process (#38)", () => { + test("two daemons have independent run registries, pubsub, and dedupe state", async () => { + const root = mkdtempSync(join(tmpdir(), "factory-two-daemons-")); + + const adapter = createSlowFakeAdapter( + [ + { type: "TEXT_MESSAGE_START" }, + { type: "TEXT_MESSAGE_CONTENT", delta: "hi" }, + { type: "TEXT_MESSAGE_END" }, + ], + 10, + ); + + const daemon1 = await startDaemon({ + dbPath: join(root, "daemon1.db"), + port: 0, + adapter, + config: defineConfig({ + repo: { + sshUrl: join(root, "seed-not-used"), + identity: { name: "Factory", email: "factory@factory.test" }, + baseBranch: "main", + slug: "acme/widgets", + }, + workflows: [scratchWorkflow], + workspaceRoot: join(root, "workspaces1"), + retainedWorkspaces: 10, + }), + }); + + const daemon2 = await startDaemon({ + dbPath: join(root, "daemon2.db"), + port: 0, + adapter, + config: defineConfig({ + repo: { + sshUrl: join(root, "seed-not-used"), + identity: { name: "Factory", email: "factory@factory.test" }, + baseBranch: "main", + slug: "acme/widgets", + }, + workflows: [scratchWorkflow], + workspaceRoot: join(root, "workspaces2"), + retainedWorkspaces: 10, + }), + }); + + try { + const port1 = daemon1.server.port; + const port2 = daemon2.server.port; + + const res1 = await fetch(`http://localhost:${port1}/api/runs`, { + method: "POST", + body: JSON.stringify({ workflowId: "two-daemon-test", input: { marker: "daemon1" } }), + }); + expect(res1.status).toBe(201); + const { runId: runId1 } = (await res1.json()) as { runId: string }; + + const res2 = await fetch(`http://localhost:${port2}/api/runs`, { + method: "POST", + body: JSON.stringify({ workflowId: "two-daemon-test", input: { marker: "daemon2" } }), + }); + expect(res2.status).toBe(201); + const { runId: runId2 } = (await res2.json()) as { runId: string }; + + expect(runId1).not.toBe(runId2); + + const list1 = (await fetch(`http://localhost:${port1}/api/runs`).then((r) => + r.json(), + )) as Array<{ runId: string }>; + const list2 = (await fetch(`http://localhost:${port2}/api/runs`).then((r) => + r.json(), + )) as Array<{ runId: string }>; + + expect(list1.map((r) => r.runId)).toEqual([runId1]); + expect(list2.map((r) => r.runId)).toEqual([runId2]); + + const detail1FromDaemon2 = await fetch(`http://localhost:${port2}/api/runs/${runId1}`); + expect(detail1FromDaemon2.status).toBe(404); + + const detail2FromDaemon1 = await fetch(`http://localhost:${port1}/api/runs/${runId2}`); + expect(detail2FromDaemon1.status).toBe(404); + + await waitForTerminal(port1, runId1); + await waitForTerminal(port2, runId2); + + expect(daemon1.services.registry.activeRunIds()).toEqual([]); + expect(daemon2.services.registry.activeRunIds()).toEqual([]); + + const dedupeKey = "shared-key"; + const res3 = await fetch(`http://localhost:${port1}/api/runs`, { + method: "POST", + body: JSON.stringify({ workflowId: "two-daemon-test", input: { marker: "d1" }, dedupeKey }), + }); + expect(res3.status).toBe(201); + const { runId: runId3 } = (await res3.json()) as { runId: string }; + + const res4 = await fetch(`http://localhost:${port2}/api/runs`, { + method: "POST", + body: JSON.stringify({ workflowId: "two-daemon-test", input: { marker: "d2" }, dedupeKey }), + }); + expect(res4.status).toBe(201); + const { runId: runId4 } = (await res4.json()) as { runId: string }; + + expect(runId3).not.toBe(runId4); + + await waitForTerminal(port1, runId3); + await waitForTerminal(port2, runId4); + } finally { + if (daemon1.schedulerFiber !== undefined) { + Effect.runFork(Fiber.interrupt(daemon1.schedulerFiber)); + } + if (daemon2.schedulerFiber !== undefined) { + Effect.runFork(Fiber.interrupt(daemon2.schedulerFiber)); + } + daemon1.server.stop(true); + daemon2.server.stop(true); + rmSync(root, { recursive: true, force: true }); + } + }); +}); From ede22d7af169a60b339389f7b282aaa335e0a012 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sat, 19 Sep 2026 12:10:44 +0000 Subject: [PATCH 09/26] fix: add port guard in two-daemon test for typecheck --- src/server/two-daemons.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/server/two-daemons.test.ts b/src/server/two-daemons.test.ts index 1856dc9..d315f89 100644 --- a/src/server/two-daemons.test.ts +++ b/src/server/two-daemons.test.ts @@ -86,6 +86,9 @@ describe("two daemons in one process (#38)", () => { try { const port1 = daemon1.server.port; const port2 = daemon2.server.port; + if (port1 === undefined || port2 === undefined) { + throw new Error("daemons did not bind to ports"); + } const res1 = await fetch(`http://localhost:${port1}/api/runs`, { method: "POST", From 666d012c3d9252aacd8903909226c1488c4b13d6 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Sun, 20 Sep 2026 13:00:09 +0000 Subject: [PATCH 10/26] fix(stack): preserve usage accounting, domain-error messages and lint cleanups --- src/events.ts | 2 ++ src/runtime/run.ts | 1 + src/server/runs.ts | 14 ++++++++------ src/server/two-daemons.test.ts | 6 +++--- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/events.ts b/src/events.ts index ff6de50..3d99d69 100644 --- a/src/events.ts +++ b/src/events.ts @@ -144,6 +144,8 @@ export const RunEventPayload = Schema.TaggedUnion({ /** The run currently holding `key` (non-terminal — it is holding). */ holderRunId: Schema.String, childWorkflowId: Schema.String, + /** Human-readable reason, naming the held key and its holder. */ + error: Schema.optional(Schema.String), }, // ---- ctx.agent(name, prompt, opts?) -------------------------------- diff --git a/src/runtime/run.ts b/src/runtime/run.ts index 8387371..8cb2e7f 100644 --- a/src/runtime/run.ts +++ b/src/runtime/run.ts @@ -438,6 +438,7 @@ export function startRun( key: err.key, holderRunId: err.holderRunId, childWorkflowId: child.id, + error: domainErrorMessage(err), }); } throw err; diff --git a/src/server/runs.ts b/src/server/runs.ts index d8b5471..ebf58a3 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -19,6 +19,7 @@ import type { Database } from "bun:sqlite"; import { rm } from "node:fs/promises"; +import { Schema } from "effect"; import type { ManagedRuntime } from "effect"; import { admitRun } from "./admission"; import { appendEvent, getRunEvents, listRuns } from "../persistence/store"; @@ -200,16 +201,17 @@ async function dispatchChildRun( const depth = dispatchDepth(db, parentRunId); if (depth + 1 > maxDepth) { - throw new DispatchCapError( - `dispatch depth exceeded: run ${parentRunId} is nested ${depth} levels deep; ` + + throw new DispatchCapError({ + message: + `dispatch depth exceeded: run ${parentRunId} is nested ${depth} levels deep; ` + `max ${maxDepth} (a workflow that dispatches itself must not fill the daemon)`, - ); + }); } const childCount = countDispatchedChildren(db, parentRunId); if (childCount >= maxChildren) { - throw new DispatchCapError( - `dispatch child cap exceeded: run ${parentRunId} already dispatched ${childCount} children; max ${maxChildren}`, - ); + throw new DispatchCapError({ + message: `dispatch child cap exceeded: run ${parentRunId} already dispatched ${childCount} children; max ${maxChildren}`, + }); } const childRunId = `run-${crypto.randomUUID()}`; diff --git a/src/server/two-daemons.test.ts b/src/server/two-daemons.test.ts index d315f89..0ae03c1 100644 --- a/src/server/two-daemons.test.ts +++ b/src/server/two-daemons.test.ts @@ -5,7 +5,7 @@ * state is independent. */ -import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "bun:test"; @@ -52,8 +52,8 @@ describe("two daemons in one process (#38)", () => { const daemon1 = await startDaemon({ dbPath: join(root, "daemon1.db"), port: 0, - adapter, config: defineConfig({ + agent: { adapter }, repo: { sshUrl: join(root, "seed-not-used"), identity: { name: "Factory", email: "factory@factory.test" }, @@ -69,8 +69,8 @@ describe("two daemons in one process (#38)", () => { const daemon2 = await startDaemon({ dbPath: join(root, "daemon2.db"), port: 0, - adapter, config: defineConfig({ + agent: { adapter }, repo: { sshUrl: join(root, "seed-not-used"), identity: { name: "Factory", email: "factory@factory.test" }, From 4cc5fcc53092c4297807a47bb03aba34dd04bc8e Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Wed, 23 Sep 2026 06:36:29 +0000 Subject: [PATCH 11/26] chore: remove committed sandbox artifact A 0-byte .tanstack-projected-* file under a stray data/src/factory/... path leaked into 2f97451 from a sandboxed run writing through an absolute path that mirrored the checkout. .gitignore already covers .tanstack-projected-* and data/ (added in PR #55), which is why nothing flagged it since; git rm drops it and its now-empty parent dirs. No other stray data/ or .factory/workspaces/ paths are tracked on this branch. Part of #32 --- .../.tanstack-projected-1e95f9272dfe038f | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 data/src/factory/.factory/workspaces/run-69780571-15e8-442a-83cf-2abda1fc3f2e/.tanstack-projected-1e95f9272dfe038f diff --git a/data/src/factory/.factory/workspaces/run-69780571-15e8-442a-83cf-2abda1fc3f2e/.tanstack-projected-1e95f9272dfe038f b/data/src/factory/.factory/workspaces/run-69780571-15e8-442a-83cf-2abda1fc3f2e/.tanstack-projected-1e95f9272dfe038f deleted file mode 100644 index e69de29..0000000 From b302a5e9145a15d6a87d7c19371d8f4830f4ae23 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Wed, 23 Sep 2026 06:40:37 +0000 Subject: [PATCH 12/26] test(scheduler): drive scheduler tests with Effect's real TestClock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the hand-rolled fake Clock.Clock with effect/testing's TestClock, per issue #38's "tests use TestClock rather than an injected now". tickOnce only ever reads the clock synchronously (currentTimeMillisUnsafe()) and never suspends on Effect.sleep, so none of TestClock's fiber-coordination machinery is exercised here — but building it via Effect.runPromise(Effect.scoped(TestClock.make())) and bridging setTime back into the plain-async fixture (ADR 0009 §5) composes cleanly, so there was no need to keep the fake. Part of #32 --- src/server/scheduler.test.ts | 112 ++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/src/server/scheduler.test.ts b/src/server/scheduler.test.ts index 2baf45d..2e428b1 100644 --- a/src/server/scheduler.test.ts +++ b/src/server/scheduler.test.ts @@ -6,11 +6,14 @@ * * Driven by an injected Clock (#38): the "due" decision is * `Cron.next(cron, lastTick) <= now`, so the cron itself never needs the - * real clock in a test. + * real clock in a test. The clock comes from Effect's real `TestClock` + * (`effect/testing`), not a hand-rolled fake — see `createTestClock` below + * for how it is bridged into `tickOnce`'s plain-async world. */ import { describe, expect, test } from "bun:test"; import { Clock, Cron, Effect } from "effect"; +import { TestClock } from "effect/testing"; import { createDedupeRegistry } from "../lib/dedupe"; import type { WorkflowDefinition } from "../workflow"; import { ConcurrencyLimitError, DispatchCapError } from "./runs"; @@ -45,24 +48,27 @@ function schedule(overrides: Partial = {}): RuntimeSchedule { }; } -function createTestClock(initialTime: number): { +/** + * `tickOnce` is deliberately a plain async function (ADR 0009 §5), and it + * only ever reads the clock synchronously via `currentTimeMillisUnsafe()` — + * it never suspends on `Effect.sleep`. That means none of `TestClock`'s + * fiber-coordination machinery (scheduled sleeps, the "hung test" warning + * fiber) is exercised here; what's needed from it is just a `Clock.Clock` + * whose time we can move. A real `TestClock` still satisfies that cleanly: + * `TestClock.make()` is built once per fixture via `Effect.runPromise`, and + * `setTime` bridges back into the plain-async test body the same way + * `ManagedRuntime` would for any other Effect service consumed from + * imperative code. + */ +async function createTestClock(initialTime: number): Promise<{ clock: Clock.Clock; - setTime: (t: number) => void; -} { - let now = initialTime; + setTime: (t: number) => Promise; +}> { + const testClock = await Effect.runPromise(Effect.scoped(TestClock.make())); + await Effect.runPromise(testClock.setTime(initialTime)); return { - setTime: (t: number) => { - now = t; - }, - clock: { - currentTimeMillisUnsafe: () => now, - currentTimeMillis: Effect.sync(() => now), - monotonicTimeNanosUnsafe: () => BigInt(now) * 1_000_000n, - monotonicTimeNanos: Effect.sync(() => BigInt(now) * 1_000_000n), - currentTimeNanosUnsafe: () => BigInt(now) * 1_000_000n, - currentTimeNanos: Effect.sync(() => BigInt(now) * 1_000_000n), - sleep: () => Effect.void, - }, + clock: testClock, + setTime: (t: number) => Effect.runPromise(testClock.setTime(t)), }; } @@ -71,15 +77,15 @@ interface Fixture { readonly fired: Array; readonly registry: ReturnType; readonly fireError: Map; - readonly setTime: (t: number) => void; + readonly setTime: (t: number) => Promise; } -function fixture(schedules: ReadonlyArray): Fixture { +async function fixture(schedules: ReadonlyArray): Promise { const registry = createDedupeRegistry(); const fired: Array = []; const fireError: Map = new Map(); - const { clock, setTime } = createTestClock(0); + const { clock, setTime } = await createTestClock(0); const deps: SchedulerDeps = { schedules, @@ -107,36 +113,36 @@ const DAY01_0400 = Date.parse("2026-01-01T04:00:00Z"); describe("scheduler tick (issue #16)", () => { test("fires when its cron window fell between lastTick and now", async () => { - const fx = fixture([schedule()]); - fx.setTime(DAY01_0259); + const fx = await fixture([schedule()]); + await fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); - fx.setTime(DAY01_0400); + await fx.setTime(DAY01_0400); const results = await tickOnce(fx.deps(), state); expect(results).toEqual([{ scheduleId: "nightly", action: "fired", runId: "run-for-nightly" }]); }); test("does not fire twice for the same window across ticks", async () => { - const fx = fixture([schedule()]); - fx.setTime(DAY01_0259); + const fx = await fixture([schedule()]); + await fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); - fx.setTime(DAY01_0400); + await fx.setTime(DAY01_0400); await tickOnce(fx.deps(), state); - fx.setTime(DAY01_0400 + 1); + await fx.setTime(DAY01_0400 + 1); await tickOnce(fx.deps(), state); expect(fx.fired).toEqual(["nightly"]); }); test("skips the window while its previous run still holds the dedupe key (overlap)", async () => { - const fx = fixture([schedule()]); - fx.setTime(DAY01_0259); + const fx = await fixture([schedule()]); + await fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); fx.registry.claim("schedule:nightly", "run-previous"); - fx.setTime(DAY01_0400); + await fx.setTime(DAY01_0400); const results = await tickOnce(fx.deps(), state); expect(results).toEqual([{ scheduleId: "nightly", action: "skipped-overlap" }]); @@ -144,24 +150,24 @@ describe("scheduler tick (issue #16)", () => { }); test("overlap: 'stack' fires regardless of the previous run", async () => { - const fx = fixture([schedule({ overlap: "stack" })]); - fx.setTime(DAY01_0259); + const fx = await fixture([schedule({ overlap: "stack" })]); + await fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); fx.registry.claim("schedule:nightly", "run-previous"); - fx.setTime(DAY01_0400); + await fx.setTime(DAY01_0400); const results = await tickOnce(fx.deps(), state); expect(results).toEqual([{ scheduleId: "nightly", action: "fired", runId: "run-for-nightly" }]); }); test("a rejected fire (e.g. concurrency limit) skips the window instead of dying", async () => { - const fx = fixture([schedule()]); - fx.setTime(DAY01_0259); + const fx = await fixture([schedule()]); + await fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); fx.fireError.set("nightly", ConcurrencyLimitError.of({ maxConcurrentRuns: 1 })); - fx.setTime(DAY01_0400); + await fx.setTime(DAY01_0400); const results = await tickOnce(fx.deps(), state); expect(results).toEqual([{ scheduleId: "nightly", action: "skipped-concurrency" }]); @@ -171,14 +177,14 @@ describe("scheduler tick (issue #16)", () => { }); test("windows missed while the daemon was down are not replayed", async () => { - const fx = fixture([schedule({ cron: Cron.parseUnsafe("* * * * *", "UTC") })]); + const fx = await fixture([schedule({ cron: Cron.parseUnsafe("* * * * *", "UTC") })]); const start = Date.parse("2026-01-01T12:00:00Z"); - fx.setTime(start); + await fx.setTime(start); const state = createSchedulerState(fx.deps()); - fx.setTime(start + 30_000); + await fx.setTime(start + 30_000); await tickOnce(fx.deps(), state); - fx.setTime(start + 3 * 60_000); + await fx.setTime(start + 3 * 60_000); const results = await tickOnce(fx.deps(), state); expect(fx.fired).toEqual(["nightly"]); @@ -186,7 +192,7 @@ describe("scheduler tick (issue #16)", () => { }); test("runOnStart fires once at scheduler start, before any cron window", async () => { - const fx = fixture([schedule({ runOnStart: true })]); + const fx = await fixture([schedule({ runOnStart: true })]); const state = createSchedulerState(fx.deps()); expect(state.runOnStartPending).toEqual(new Set(["nightly"])); @@ -200,12 +206,12 @@ describe("scheduler tick (issue #16)", () => { }); test("an independent schedule is unaffected by another's failure", async () => { - const fx = fixture([schedule({ id: "a" }), schedule({ id: "b" })]); - fx.setTime(DAY01_0259); + const fx = await fixture([schedule({ id: "a" }), schedule({ id: "b" })]); + await fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); fx.fireError.set("a", new Error("boom")); - fx.setTime(DAY01_0400); + await fx.setTime(DAY01_0400); const results = await tickOnce(fx.deps(), state); expect(results.filter((r) => r.scheduleId === "a")).toEqual([ @@ -217,32 +223,32 @@ describe("scheduler tick (issue #16)", () => { }); test("each domain error type is explicitly classified (exhaustive match, #34)", async () => { - const concurrencyFx = fixture([schedule({ id: "conc" })]); - concurrencyFx.setTime(DAY01_0259); + const concurrencyFx = await fixture([schedule({ id: "conc" })]); + await concurrencyFx.setTime(DAY01_0259); const concurrencyState = createSchedulerState(concurrencyFx.deps()); concurrencyFx.fireError.set("conc", ConcurrencyLimitError.of({ maxConcurrentRuns: 1 })); - concurrencyFx.setTime(DAY01_0400); + await concurrencyFx.setTime(DAY01_0400); expect(await tickOnce(concurrencyFx.deps(), concurrencyState)).toEqual([ { scheduleId: "conc", action: "skipped-concurrency" }, ]); - const dedupeFx = fixture([schedule({ id: "dedupe" })]); - dedupeFx.setTime(DAY01_0259); + const dedupeFx = await fixture([schedule({ id: "dedupe" })]); + await dedupeFx.setTime(DAY01_0259); const dedupeState = createSchedulerState(dedupeFx.deps()); dedupeFx.fireError.set( "dedupe", DedupeKeyError.of({ key: "schedule:dedupe", holderRunId: "run-x" }), ); - dedupeFx.setTime(DAY01_0400); + await dedupeFx.setTime(DAY01_0400); expect(await tickOnce(dedupeFx.deps(), dedupeState)).toEqual([ { scheduleId: "dedupe", action: "fire-failed" }, ]); - const capFx = fixture([schedule({ id: "cap" })]); - capFx.setTime(DAY01_0259); + const capFx = await fixture([schedule({ id: "cap" })]); + await capFx.setTime(DAY01_0259); const capState = createSchedulerState(capFx.deps()); capFx.fireError.set("cap", new DispatchCapError({ message: "depth exceeded" })); - capFx.setTime(DAY01_0400); + await capFx.setTime(DAY01_0400); expect(await tickOnce(capFx.deps(), capState)).toEqual([ { scheduleId: "cap", action: "fire-failed" }, ]); From f80d592823c3c636d55e2176301cad4cd68449ed Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Wed, 23 Sep 2026 07:05:27 +0000 Subject: [PATCH 13/26] test(scheduler): await the async fixture in the unknown-tag case The #34 exhaustiveness test landed while `fixture` was still synchronous; #38 made it async for `TestClock`. Rebasing the stack put the two together, which typecheck caught. --- src/server/scheduler.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/scheduler.test.ts b/src/server/scheduler.test.ts index 2e428b1..5b4882d 100644 --- a/src/server/scheduler.test.ts +++ b/src/server/scheduler.test.ts @@ -263,12 +263,12 @@ describe("scheduler tick (issue #16)", () => { class UnknownTaggedError extends Error { readonly _tag = "SomeFutureDomainError"; } - const fx = fixture([schedule()]); - fx.setTime(DAY01_0259); + const fx = await fixture([schedule()]); + await fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); fx.fireError.set("nightly", new UnknownTaggedError("mystery failure")); - fx.setTime(DAY01_0400); + await fx.setTime(DAY01_0400); const results = await tickOnce(fx.deps(), state); expect(results).toEqual([{ scheduleId: "nightly", action: "fire-failed" }]); From 77dc7abaeaf9945f8de5790912352875afca8213 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 09:02:52 +0000 Subject: [PATCH 14/26] fix(runtime): use the constructed DedupeKeyError message in DispatchCollision The domainErrorMessage helper was removed on the #34 layer; DedupeKeyError now carries its message as a constructed field. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/runtime/run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/run.ts b/src/runtime/run.ts index 8cb2e7f..0663df4 100644 --- a/src/runtime/run.ts +++ b/src/runtime/run.ts @@ -438,7 +438,7 @@ export function startRun( key: err.key, holderRunId: err.holderRunId, childWorkflowId: child.id, - error: domainErrorMessage(err), + error: err.message, }); } throw err; From ed05d498edf003c4751908ee53b37a2de798a687 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 09:10:59 +0000 Subject: [PATCH 15/26] test(server): pass daemon services to the prepareWorkspace tests Co-Authored-By: Claude Opus 5.5 (1M context) --- src/server/http.test.ts | 7 ++++++- src/server/runs.test.ts | 10 ++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/server/http.test.ts b/src/server/http.test.ts index bbceb01..9a37b4f 100644 --- a/src/server/http.test.ts +++ b/src/server/http.test.ts @@ -1509,7 +1509,12 @@ describe("adapter.prepareWorkspace through the legacy POST /api/runs (#37)", () const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); const adapter = trackingAdapter(); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { diff --git a/src/server/runs.test.ts b/src/server/runs.test.ts index 794d444..db01e48 100644 --- a/src/server/runs.test.ts +++ b/src/server/runs.test.ts @@ -345,8 +345,9 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); const adapter = trackingAdapter(); + const services = createTestServices(); - const runId = await startTrackedRun(makeAgentRuntime(adapter), db, echoWorkflow, { + const runId = await startTrackedRun(makeAgentRuntime(adapter), db, services, echoWorkflow, { runId: "run-prep-clone", workspace: { workspaceRoot: join(root, "workspaces"), @@ -356,7 +357,7 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { }, input: {}, }); - await waitFor(() => !isActive(runId)); + await waitFor(() => !services.registry.isActive(runId)); expect(adapter.prepared).toEqual([join(root, "workspaces", "run-prep-clone")]); finish(); @@ -366,8 +367,9 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); const adapter = trackingAdapter(); + const services = createTestServices(); - const runId = await startTrackedRun(makeAgentRuntime(adapter), db, scratchWorkflow, { + const runId = await startTrackedRun(makeAgentRuntime(adapter), db, services, scratchWorkflow, { runId: "run-prep-scratch", workspace: { workspaceRoot: join(root, "workspaces"), @@ -377,7 +379,7 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { }, input: {}, }); - await waitFor(() => !isActive(runId)); + await waitFor(() => !services.registry.isActive(runId)); expect(adapter.prepared).toEqual([]); finish(); From d4d38b8f3487075940750b672835523f886db8d8 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 09:24:56 +0000 Subject: [PATCH 16/26] fix(server): carry the agent-runtime fixes through the per-daemon services Rebasing onto the reworked agent-runtime layer: startRun is synchronous again, the daemon handle disposes its managed runtime via `stop()`, the opencode default lives only in AgentRuntimeLayer, and the doc comments restored below are kept on the per-daemon run registry and its options. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/server/daemon.ts | 36 ++++++++----- src/server/http.ts | 4 +- src/server/runs.ts | 114 ++++++++++++++++++++++++++++++++++++++-- src/server/scheduler.ts | 4 +- 4 files changed, 136 insertions(+), 22 deletions(-) diff --git a/src/server/daemon.ts b/src/server/daemon.ts index ef4cee6..7095a53 100644 --- a/src/server/daemon.ts +++ b/src/server/daemon.ts @@ -2,14 +2,14 @@ * `factory serve` — wires the HTTP API/SSE (`server/http.ts`) and the config * scheduler (`server/scheduler.ts`) into one running process (AGENTS.md's * third column). Automatic dispatch is not daemon logic since epic #19: it - * is project policy living in scheduled wrapper workflows. + * is project policy living in scheduled wrapper workflows — e.g. the sample + * project's Ready sweep — fired by the scheduler loop on their cron. * - * Issue #36: the daemon now has an Effect composition root. `server/daemon.ts` - * builds a `Layer` for the agent runtime, creates a `ManagedRuntime`, and - * passes that runtime to `serve()` and the scheduler. The adapter is no longer - * threaded by hand through `ServerOptions`, `StartTrackedRunOptions`, - * `DispatchEnv` and the run/step options; it is resolved from context inside - * `runtime/agent-step.ts`. + * Issue #36: this is the daemon's Effect composition root. It builds the + * agent runtime's `ManagedRuntime` once and hands it to `serve()` and the + * scheduler; the adapter is resolved from that runtime's context inside + * `runtime/agent-step.ts` instead of being threaded through every options + * type. `DaemonHandle.stop` disposes it. * * #38: the daemon creates per-instance services (run registry, pubsub, * dedupe registry, refresh gates) so two daemons can coexist in one process @@ -18,7 +18,7 @@ import { mkdir } from "node:fs/promises"; import { dirname } from "node:path"; -import { Clock, Effect, type Fiber, ManagedRuntime } from "effect"; +import { Clock, Effect, Fiber } from "effect"; type AnyFiber = Fiber.Fiber; import type { FactoryConfig } from "../config"; import { openStore } from "../persistence/store"; @@ -39,8 +39,7 @@ import { toRuntimeSchedules, type SchedulerDeps, } from "./scheduler"; -import { AgentRuntimeLayer } from "../runtime/agent-runtime"; -import { opencodeAdapter } from "../runtime/opencode-adapter"; +import { makeAgentRuntime } from "../runtime/agent-runtime"; export function createLiveClock(): Clock.Clock { return { @@ -67,6 +66,11 @@ export interface DaemonHandle { readonly server: ReturnType; readonly schedulerFiber: AnyFiber | undefined; readonly services: DaemonServices; + /** + * Shut the daemon down: interrupt the scheduler, stop the HTTP server, and + * dispose the agent runtime (issue #36). + */ + readonly stop: () => Promise; } export const DEFAULT_SCHEDULER_INTERVAL_MS = 30_000; @@ -75,9 +79,7 @@ export async function startDaemon(options: DaemonOptions): Promise await mkdir(dirname(options.dbPath), { recursive: true }); const db = openStore(options.dbPath); - const adapter = options.config?.agent.adapter ?? opencodeAdapter; - const layer = AgentRuntimeLayer(adapter); - const runtime = ManagedRuntime.make(layer); + const runtime = makeAgentRuntime(options.config?.agent.adapter); const services: DaemonServices = { registry: createRunRegistry(), @@ -138,5 +140,11 @@ export async function startDaemon(options: DaemonOptions): Promise })() : undefined; - return { server, schedulerFiber, services }; + const stop = async (): Promise => { + if (schedulerFiber !== undefined) await Effect.runPromise(Fiber.interrupt(schedulerFiber)); + await server.stop(true); + await runtime.dispose(); + }; + + return { server, schedulerFiber, services, stop }; } diff --git a/src/server/http.ts b/src/server/http.ts index 1275324..b792aa0 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -38,7 +38,7 @@ import { resetClone, type GitIdentity } from "../lib/clone"; import { loadWorkflow } from "../lib/load-workflow"; import { getRunEvents, listRuns, type RunSummary } from "../persistence/store"; import type { ManagedRuntime } from "effect"; -import { AgentRuntime } from "../runtime/agent-runtime"; +import type { AgentRuntime } from "../runtime/agent-runtime"; import index from "../web/index.html"; import { admitRun } from "./admission"; import { DedupeKeyError } from "../lib/dedupe"; @@ -70,7 +70,7 @@ export interface ServerOptions { readonly services: DaemonServices; /** * Issue #36: the Effect managed runtime that provides the agent runtime - * service. The adapter is resolved from context inside `startTrackedRun` + * service. The adapter is resolved from its context inside the agent step * rather than threaded through `ServerOptions`. */ readonly runtime: ManagedRuntime.ManagedRuntime; diff --git a/src/server/runs.ts b/src/server/runs.ts index ebf58a3..b8da2d4 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -19,8 +19,7 @@ import type { Database } from "bun:sqlite"; import { rm } from "node:fs/promises"; -import { Schema } from "effect"; -import type { ManagedRuntime } from "effect"; +import { Schema, type ManagedRuntime } from "effect"; import { admitRun } from "./admission"; import { appendEvent, getRunEvents, listRuns } from "../persistence/store"; import type { RunRepo } from "../runtime/run"; @@ -28,7 +27,7 @@ import { startRun, type RunHandle } from "../runtime/run"; import type { GitIdentity } from "../lib/clone"; import { allocateWorkspace, type RefreshGates } from "../lib/workspace"; import type { DedupeRegistry } from "../lib/dedupe"; -import { AgentRuntime } from "../runtime/agent-runtime"; +import type { AgentRuntime } from "../runtime/agent-runtime"; import type { DispatchChildFn, WorkflowDefinition, WorkspaceKind } from "../workflow"; import type { PubSub } from "./pubsub"; @@ -40,9 +39,15 @@ function isReserved(entry: RunHandle | ReservedSlot | undefined): boole return entry !== undefined && !("result" in entry) && "cancelled" in entry; } +/** + * Issue #14: how deep a parent → child → grandchild chain may nest — the cap + * that keeps a workflow which dispatches itself from filling the daemon. + */ export const DEFAULT_MAX_DISPATCH_DEPTH = 5; +/** Issue #14: how many children one run itself may dispatch. */ export const DEFAULT_MAX_CHILDREN_PER_RUN = 20; +/** Issue #14: a `ctx.dispatch` rejected by a cap, for the parent to surface. */ export class DispatchCapError extends Schema.TaggedError()("DispatchCapError", { message: Schema.String, }) {} @@ -80,6 +85,12 @@ export interface RunRegistry { isActive(runId: string): boolean; activeRunIds(): ReadonlyArray; getActiveHandle(runId: string): RunHandle | undefined; + /** + * Cancellation for a run this registry holds — whether it is already + * running (returns its handle), still reserving/allocation-bound (marks the + * slot so the run is cancelled the moment it starts), or unknown + * (`undefined`). + */ cancelRegisteredRun( runId: string, ): @@ -138,34 +149,85 @@ export interface WorkspaceSpec { } export interface StartTrackedRunOptions { + /** An explicit directory wins; otherwise `workspace` allocates one per runId (D28). */ readonly dir?: string; readonly workspace?: WorkspaceSpec; + /** Write-back environment for the run (D32): came from config, not the caller. */ readonly repo?: RunRepo; readonly input: unknown; readonly runId?: string; + /** + * D29's ceiling, enforced atomically at reservation time — the reservation + * lands in the registry before any `await`, so two near-simultaneous + * start requests cannot both slip past it. Absent, no limit applies. + */ readonly maxConcurrentRuns?: number; + /** + * Issue #14: the environment a child run of this run starts with, so + * `ctx.dispatch` can allocate a workspace, admission-limit and repo for it. + * Absent, the workflow's `ctx.dispatch` throws (dispatch needs a + * config-backed daemon run; in-process execution is legacy). Children + * inherit it, so grandchildren work too. + */ readonly dispatchEnv?: DispatchEnv; + /** + * Issue #14: this run's parent, when it was started by `ctx.dispatch` — + * recorded on `RunStarted.parentId` so the UI can navigate child → parent. + */ readonly parentRunId?: string; + /** + * Issue #15: this run's dedupe key. Claimed synchronously at start (before + * any await) in the run's own registry slot manner — two near-simultaneous + * starts with the same key cannot both slip through the check-then-claim — + * and released the moment the run settles in any terminal state, including + * a failed startup. Absent, the run claims nothing. + */ readonly dedupeKey?: string; + /** + * Issue #15: `true` when a caller (the dispatch path) already claimed the + * key synchronously before handing the start over, so the claim must not be + * re-asserted here. Release still happens here, keyed to this run id. + */ readonly dedupeKeyClaimed?: boolean; + /** + * Issue #16: the schedule that started this run, when any did - passed + * through to `RunStarted.scheduleId`. + */ readonly scheduleId?: string; + /** + * Issue #16: agent-level overrides the starting schedule carries. Passed + * through to the run's model precedence chain. + */ readonly agentOverrides?: { readonly model?: string }; } +/** + * The environment `ctx.dispatch`'s children share with their parent (issue + * #14): workspace provisioning, write-back environment and the + * concurrency ceiling — the same wiring a config-backed server wraps every + * run in, simply reused for child runs (children inherit it, so a child can + * dispatch grandchildren). + */ export interface DispatchEnv { readonly workspace?: WorkspaceSpec; readonly repo?: RunRepo; readonly maxConcurrentRuns?: number; + /** Issue #14: dispatch depth / per-run child caps, over the defaults. */ readonly maxDispatchDepth?: number; readonly maxChildrenPerRun?: number; } +/** + * The run this `runId` was started from (`RunStarted.parentId`), or none — + * a run started without `ctx.dispatch` has no parent. + */ function parentOf(db: Database, runId: string): string | undefined { const started = getRunEvents(db, runId).find((event) => event.payload._tag === "RunStarted"); if (started === undefined || started.payload._tag !== "RunStarted") return undefined; return started.payload.parentId; } +/** How deep this run sits in the parent → child chain (a top-level run is 0). */ function dispatchDepth(db: Database, runId: string): number { let depth = 0; let cursor = runId; @@ -178,6 +240,18 @@ function dispatchDepth(db: Database, runId: string): number { return depth; } +/** + * Issue #14: start a child run of `parentRunId`. + * + * Every rejection is a throw, so `ctx.dispatch` rejects and the *parent* + * fails visibly — nothing about a child's admission is allowed to be a + * silent drop. The checks are synchronous over in-memory state and the sync + * sqlite event log, so by the time the parent carries on, admission has + * already happened. A validated child is started without being awaited: the + * parent never exposes an awaitable child (D-epic 19). The child's own + * startup failures land on the child's event log / console, never in the + * parent's. + */ async function dispatchChildRun( runtime: ManagedRuntime.ManagedRuntime, db: Database, @@ -214,6 +288,9 @@ async function dispatchChildRun( }); } + // Issue #15: the collision check is synchronous with the child id in hand + // and *before* the fire-and-forget start, so a collision throws into the + // parent here instead of being swallowed by the un-awaited start's catch. const childRunId = `run-${crypto.randomUUID()}`; if (opts?.dedupeKey !== undefined) services.dedupeRegistry.claim(opts.dedupeKey, childRunId); @@ -246,6 +323,7 @@ function countDispatchedChildren(db: Database, runId: string): number { return getRunEvents(db, runId).filter((event) => event.payload._tag === "RunDispatched").length; } +/** Starts a run, persists+publishes every event, and tracks it until terminal. */ export async function startTrackedRun( runtime: ManagedRuntime.ManagedRuntime, db: Database, @@ -253,6 +331,11 @@ export async function startTrackedRun( workflow: WorkflowDefinition, options: StartTrackedRunOptions, ): Promise { + // crypto.randomUUID(), not `run-${Date.now()}`: the server can have more than one run start + // within the same millisecond (concurrent HTTP POSTs, or tests running in the same process), + // and a collided runId cross-wires the pubsub channel and `active` registry between two + // unrelated runs — one run's SSE watcher can then see the other's terminal event and close + // its own db while its real run is still writing to it. const runId = options.runId ?? `run-${crypto.randomUUID()}`; const registry = services.registry; @@ -261,6 +344,10 @@ export async function startTrackedRun( throw new Error(`run ${runId} is already active`); } + // Issue #15: claim the dedupe key synchronously — check-then-claim with no + // `await` in between, the same atomicity the registry slot reservation has — + // so two near-simultaneous starts on the same key cannot both slip past. A + // collision throws before anything is started, leaving no trace. if (options.dedupeKey !== undefined && options.dedupeKeyClaimed !== true) { services.dedupeRegistry.claim(options.dedupeKey, runId); } @@ -272,6 +359,9 @@ export async function startTrackedRun( } try { + // Issue #13: a scratch workspace takes its kind from the workflow and + // never evicts clone workspaces — leftover scratch dirs (kept failures) + // are excluded from retention via the run log's recorded kinds. const kind: WorkspaceKind = workflow.workspace?.kind ?? "clone"; const scratchEntries = options.workspace !== undefined && kind === "scratch" @@ -297,8 +387,13 @@ export async function startTrackedRun( if (dir === undefined) throw new Error("startTrackedRun needs `dir` or `workspace`"); + // Whether the runtime allocated this dir itself (explicit callers' dirs, + // incl. the legacy path-based API, are not reaped — they are caller-owned). const workspaceAllocated = options.dir === undefined; + // Issue #14: the per-run dispatch member comes from the run's dispatch + // environment, bound at this run's id — the parent's own log is where the + // depth walk and the child count are read from. const dispatch: DispatchChildFn | undefined = options.dispatchEnv === undefined ? undefined @@ -314,7 +409,7 @@ export async function startTrackedRun( opts, ); - const handle = await startRun(workflow, runtime, { + const handle = startRun(workflow, runtime, { runId, dir, ...(options.repo !== undefined ? { repo: options.repo } : {}), @@ -331,6 +426,9 @@ export async function startTrackedRun( }, }); + // A cancel that arrived while this run was only a reserved slot is + // deferred into run start (L1): the run starts, is cancelled immediately, + // and ends as a clean RunCancelled instead of orphaning the slot. const beforeStartEntry = registry.get(runId); const reservedSlot = isReserved(beforeStartEntry) ? (beforeStartEntry as ReservedSlot) @@ -340,10 +438,18 @@ export async function startTrackedRun( void handle.result.finally(() => { if (registry.get(runId) === handle) registry.delete(runId); + // Issue #15: any terminal state — completed, failed, cancelled — + // releases the run's key. (An interrupted run — process death — is + // covered by the registry being per-daemon: a new daemon holds + // nothing.) if (options.dedupeKey !== undefined) services.dedupeRegistry.release(options.dedupeKey, runId); }); + // Issue #13: a scratch dir is reaped when the run succeeds — there is no + // tree worth keeping, and retention never applies to it. It is kept when + // the run fails (or is cancelled) so a failed precondition check remains + // inspectable. void handle.result.then((outcome) => { if (workspaceAllocated && kind === "scratch" && outcome.outcome === "completed") { void rm(dir, { recursive: true, force: true }); diff --git a/src/server/scheduler.ts b/src/server/scheduler.ts index e27f6f3..6dcb9a2 100644 --- a/src/server/scheduler.ts +++ b/src/server/scheduler.ts @@ -27,7 +27,7 @@ * provide a custom Clock instance rather than an injected `now` function. */ -import { Clock, Cron, Effect, Schedule, Schema, ManagedRuntime } from "effect"; +import { Clock, Cron, Effect, Schedule, Schema, type ManagedRuntime } from "effect"; import type { Database } from "bun:sqlite"; import type { FactoryConfig } from "../config"; import { DedupeKeyError, type DedupeRegistry } from "../lib/dedupe"; @@ -41,7 +41,7 @@ import { type WorkspaceSpec, } from "./runs"; import { RunCancelledSignal, type RunRepo } from "../runtime/run"; -import { AgentRuntime } from "../runtime/agent-runtime"; +import type { AgentRuntime } from "../runtime/agent-runtime"; export class SchedulerError extends Schema.TaggedError()("SchedulerError", { cause: Schema.Defect(), From 522ae4c3b4606304a3e5d6bbca579a71092b9c4b Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 09:32:06 +0000 Subject: [PATCH 17/26] fix(server): restore the lower-layer wiring and rationale the rebase dropped - restore `prepareWorkspace` on StartTrackedRunOptions and the startRun call (allocated clone workspaces and the legacy `{dir, clone}` POST are prepared again, #37) - restore the deleted rationale comments in http.ts, scheduler.ts, daemon.ts and runs.ts (SSE keepalive vs Bun's idleTimeout, the `development: false` router crash, D29 admission, DaemonOptions.config, runOnStart consumption) - drop the unrelated `error` field added to DispatchCollision - use Effect's default Clock instead of a hand-rolled live clock Co-Authored-By: Claude Opus 5.5 (1M context) --- src/events.ts | 2 - src/runtime/run.ts | 1 - src/server/daemon.ts | 24 ++-- src/server/dedupe.test.ts | 220 ++++++++++++++++---------------- src/server/http.test.ts | 221 ++++----------------------------- src/server/http.ts | 119 +++++++++++++++++- src/server/nested-runs.test.ts | 45 +++---- src/server/runs.test.ts | 158 ++++++++++++----------- src/server/runs.ts | 19 ++- src/server/scheduler.ts | 33 ++++- 10 files changed, 401 insertions(+), 441 deletions(-) diff --git a/src/events.ts b/src/events.ts index 3d99d69..ff6de50 100644 --- a/src/events.ts +++ b/src/events.ts @@ -144,8 +144,6 @@ export const RunEventPayload = Schema.TaggedUnion({ /** The run currently holding `key` (non-terminal — it is holding). */ holderRunId: Schema.String, childWorkflowId: Schema.String, - /** Human-readable reason, naming the held key and its holder. */ - error: Schema.optional(Schema.String), }, // ---- ctx.agent(name, prompt, opts?) -------------------------------- diff --git a/src/runtime/run.ts b/src/runtime/run.ts index 0663df4..8387371 100644 --- a/src/runtime/run.ts +++ b/src/runtime/run.ts @@ -438,7 +438,6 @@ export function startRun( key: err.key, holderRunId: err.holderRunId, childWorkflowId: child.id, - error: err.message, }); } throw err; diff --git a/src/server/daemon.ts b/src/server/daemon.ts index 7095a53..6285259 100644 --- a/src/server/daemon.ts +++ b/src/server/daemon.ts @@ -41,30 +41,27 @@ import { } from "./scheduler"; import { makeAgentRuntime } from "../runtime/agent-runtime"; -export function createLiveClock(): Clock.Clock { - return { - currentTimeMillisUnsafe: () => Date.now(), - currentTimeMillis: Effect.sync(() => Date.now()), - monotonicTimeNanosUnsafe: () => BigInt(Date.now()) * 1_000_000n, - monotonicTimeNanos: Effect.sync(() => BigInt(Date.now()) * 1_000_000n), - currentTimeNanosUnsafe: () => BigInt(Date.now()) * 1_000_000n, - currentTimeNanos: Effect.sync(() => BigInt(Date.now()) * 1_000_000n), - sleep: (duration) => Effect.sleep(duration), - }; -} - export interface DaemonOptions { readonly dbPath: string; readonly port?: number; /** Issue #16: the tick cadence of the config schedules, over the default. */ readonly schedulerIntervalMs?: number; + /** + * The run environment from `factory.config.ts` (D27). Present, every run — + * manual and scheduled alike — gets a per-run working tree under the + * configured `workspaceRoot` (D28) and shares one admission limit (D29). + * Absent, the legacy per-request `{dir, clone}` behaviour is kept. + */ readonly config?: FactoryConfig; + /** Issue #38: the scheduler's clock, over Effect's default wall clock. */ readonly clock?: Clock.Clock; } export interface DaemonHandle { readonly server: ReturnType; + /** Issue #16: the loop that fires the config's schedules, when it has any. */ readonly schedulerFiber: AnyFiber | undefined; + /** Issue #38: this daemon's own registry, pubsub, dedupe registry and refresh gates. */ readonly services: DaemonServices; /** * Shut the daemon down: interrupt the scheduler, stop the HTTP server, and @@ -73,6 +70,7 @@ export interface DaemonHandle { readonly stop: () => Promise; } +/** Issue #16: how often the scheduler's due window check runs. */ export const DEFAULT_SCHEDULER_INTERVAL_MS = 30_000; export async function startDaemon(options: DaemonOptions): Promise { @@ -127,7 +125,7 @@ export async function startDaemon(options: DaemonOptions): Promise dispatchEnv, }), dedupeRegistry: services.dedupeRegistry, - clock: options.clock ?? createLiveClock(), + clock: options.clock ?? Clock.Clock.defaultValue(), }; const state = createSchedulerState(deps); return Effect.runFork( diff --git a/src/server/dedupe.test.ts b/src/server/dedupe.test.ts index 632bdc9..845d8ac 100644 --- a/src/server/dedupe.test.ts +++ b/src/server/dedupe.test.ts @@ -4,11 +4,6 @@ * releases it when the run reaches any terminal state. Starting a second run * with a key a non-terminal run already holds throws a `DedupeKeyError` * naming the key and the holding run. - * - * #38: tests create their own DaemonServices instead of using module-level - * singletons or test seams. The `beforeStart` test seam is gone — tests - * reserve registry slots directly to test the reserved-but-not-started - * window. */ import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; @@ -20,14 +15,13 @@ import { createSlowFakeAdapter } from "../replay/adapter"; import { makeAgentRuntime } from "../runtime/agent-runtime"; import { defineWorkflow, Schema } from "../workflow"; import { createDedupeRegistry, DedupeKeyError } from "../lib/dedupe"; -import { createRunRegistry, startTrackedRun, type DaemonServices } from "./runs"; -import { createPubSub } from "./pubsub"; -import { createRefreshGates } from "../lib/workspace"; +import { activeRunIds, getActiveHandle, isActive, startTrackedRun } from "./runs"; import { serve } from "./http"; import { defineConfig } from "../config"; -async function drain(services: DaemonServices, timeoutMs = 10_000): Promise { - await waitFor(() => services.registry.activeRunIds().length === 0, timeoutMs); +/** Fire-and-forget children keep filling the registry; drain before closing the store. */ +async function drain(timeoutMs = 10_000): Promise { + await waitFor(() => activeRunIds().length === 0, timeoutMs); } const SLOW_ADAPTER = createSlowFakeAdapter( @@ -49,6 +43,19 @@ const echoWorkflow = defineWorkflow("echo-wf", { }, }); +interface Gate { + readonly wait: () => Promise; + readonly release: () => void; +} + +function makeGate(): Gate { + let release: () => void = () => undefined; + const promise = new Promise((resolve) => { + release = resolve; + }); + return { wait: () => promise, release }; +} + async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -58,32 +65,28 @@ async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise Promise; + /** Shared holder registry when one test needs two starts to agree. */ + readonly registry?: ReturnType; } function start(db: ReturnType, opts: StartOpts): Promise { - return startTrackedRun(runtime, db, opts.services, echoWorkflow, { + return startTrackedRun(runtime, db, echoWorkflow, { dir: opts.dir, input: {}, + dedupeRegistry: opts.registry ?? createDedupeRegistry(), maxConcurrentRuns: 4, ...(opts.runId !== undefined ? { runId: opts.runId } : {}), ...(opts.dedupeKey !== undefined ? { dedupeKey: opts.dedupeKey } : {}), + ...(opts.beforeStart !== undefined ? { beforeStart: opts.beforeStart } : {}), }); } +/** The `DedupeKeyError` a rejected start produced — else a placeholder string. */ async function errorOf(promise: Promise): Promise { return promise.then( () => "resolved", @@ -115,7 +118,6 @@ function withWorkspaces(root: string): Record { } describe("dedupe keys through ctx.dispatch (issue #15)", () => { - const driftAdapter = SLOW_ADAPTER; const busyWorkflow = defineWorkflow("busy-wf", { input: Schema.Struct({ n: Schema.Number }), workspace: { kind: "scratch" }, @@ -129,7 +131,6 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-dispatch-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); const parent = defineWorkflow("parent-wf", { input: Schema.Struct({ wait: Schema.Number }), @@ -140,19 +141,20 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { { n: input.wait }, { dedupeKey: "item:7" }, ); + // A second dispatch on the same key, while the first still runs. await ctx.dispatch(busyWorkflow, { n: input.wait }, { dedupeKey: "item:7" }); return { childRunId }; }, }); - const parentRunId = await startTrackedRun(runtime, db, services, parent, { + const parentRunId = await startTrackedRun(runtime, db, parent, { input: { wait: 3 }, workspace: workspaces(join(root)), maxConcurrentRuns: 10, - dispatchEnv: { ...withWorkspaces(join(root)), adapter: driftAdapter } as never, + dispatchEnv: withWorkspaces(join(root)), }); - await waitFor(() => !services.registry.isActive(parentRunId)); + await waitFor(() => !isActive(parentRunId)); const events = getRunEvents(db, parentRunId); const collided = events.find((event) => event.payload._tag === "DispatchCollision"); @@ -162,12 +164,13 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { expect(collided.payload.holderRunId).toBeTypeOf("string"); expect(collided.payload.childWorkflowId).toBe("busy-wf"); } + // The failed dispatch's throw unwound the parent into its terminal state. const failed = events.find((event) => event.payload._tag === "RunFailed"); expect( failed !== undefined && failed.payload._tag === "RunFailed" ? failed.payload.message : "", ).toMatch(/dedupe key held: "item:7"/); - await drain(services); + await drain(); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -176,59 +179,55 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-dispatch-release-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); - let releaseHeldWorkflow: () => void = () => undefined; - const heldWorkflowGate = new Promise((resolve) => { - releaseHeldWorkflow = resolve; - }); + let release: () => void = () => undefined; + const gate = makeGate(); const heldWorkflow = defineWorkflow("held-wf", { input: Schema.Struct({}), workspace: { kind: "scratch" }, run: async () => { - await heldWorkflowGate; + await gate.wait; return {}; }, }); - let releaseParent: () => void = () => undefined; - const parentGate = new Promise((resolve) => { - releaseParent = resolve; - }); - const parent = defineWorkflow("parent-release-wf", { input: Schema.Struct({}), workspace: { kind: "scratch" }, run: async (ctx) => { const ids = []; ids.push(await ctx.dispatch(heldWorkflow, {}, { dedupeKey: "item:8" })); - await parentGate; + await new Promise((r) => { + release = r; + }); + // The first child is terminal by now; the key must be free again. ids.push(await ctx.dispatch(heldWorkflow, {}, { dedupeKey: "item:8" })); return { ids }; }, }); - const parentRunId = (await startTrackedRun(runtime, db, services, parent, { + const parentRunId = await startTrackedRun(runtime, db, parent, { input: {}, workspace: workspaces(join(root)), maxConcurrentRuns: 10, - dispatchEnv: { ...withWorkspaces(join(root)), adapter: driftAdapter } as never, - })) as string; + dispatchEnv: withWorkspaces(join(root)), + }); await waitFor(() => getRunEvents(db, parentRunId).some((event) => event.payload._tag === "RunDispatched"), ); await Bun.sleep(150); - releaseHeldWorkflow(); - await Bun.sleep(50); - releaseParent(); + // The first child dispatched, still holding while the gate is shut — one + // more dispatch inside the parent would collide. Then let the child end. + gate.release(); + if (typeof release === "function") release(); await waitFor( () => getRunEvents(db, parentRunId).filter((event) => event.payload._tag === "RunDispatched") .length >= 2, 10_000, ); - await waitFor(() => !services.registry.isActive(parentRunId)); + await waitFor(() => !isActive(parentRunId)); const dispatchedCount = getRunEvents(db, parentRunId).filter( (event) => event.payload._tag === "RunDispatched", @@ -238,7 +237,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { getRunEvents(db, parentRunId).some((event) => event.payload._tag === "DispatchCollision"), ).toBe(false); - await drain(services); + await drain(); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -248,34 +247,33 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a key held by a non-terminal run rejects a second start with key and holder named", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-hold-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const gate = makeGate(); + const registry = createDedupeRegistry(); - services.registry.reserve("run-holder"); - services.dedupeRegistry.claim("item:41", "run-holder"); + const first = start(db, { + runId: "run-holder", + dir: join(root, "dir-1"), + dedupeKey: "item:41", + beforeStart: gate.wait, + registry, + }); + await waitFor(() => isActive("run-holder")); const err = (await errorOf( - start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:41", services }), + start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:41", registry }), )) as DedupeKeyError | string; expect(err).toBeInstanceOf(DedupeKeyError); if (err instanceof DedupeKeyError) { expect(err.key).toBe("item:41"); expect(err.holderRunId).toBe("run-holder"); - expect(err.message).toContain("item:41"); - expect(err.message).toContain("run-holder"); + expect(err._tag).toBe("DedupeKeyError"); } - services.registry.delete("run-holder"); - services.dedupeRegistry.release("item:41", "run-holder"); - - const firstRunId = await start(db, { - runId: "run-holder", - dir: join(root, "dir-1"), - dedupeKey: "item:41", - services, - }); - await waitFor(() => !services.registry.isActive(firstRunId)); + gate.release(); + await first; + await waitFor(() => !isActive("run-holder")); - await drain(services); + await drain(); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -283,20 +281,25 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("the holder's RunStarted carries the key; a rejected start leaves no lifecycle trace", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-log-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const gate = makeGate(); + const registry = createDedupeRegistry(); - const firstRunId = await start(db, { + const first = start(db, { runId: "run-holder", dir: join(root, "dir-1"), dedupeKey: "item:42", - services, + beforeStart: gate.wait, + registry, }); + await waitFor(() => isActive("run-holder")); await errorOf( - start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:42", services }), + start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:42", registry }), ); - await waitFor(() => !services.registry.isActive(firstRunId)); + gate.release(); + await first; + await waitFor(() => !isActive("run-holder")); const holderStarted = getRunEvents(db, "run-holder").find( (e) => e.payload._tag === "RunStarted", @@ -308,9 +311,10 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { ? holderStarted.payload.dedupeKey : "missing", ).toBe("item:42"); + // A dropped start never started: the collision run's log has no rows. expect(getRunEvents(db, "run-collide")).toHaveLength(0); - await drain(services); + await drain(); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -318,25 +322,16 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("the key is released when the holding run reaches its terminal state", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-finish-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); - const firstRunId = await start(db, { - dir: join(root, "dir-1"), - dedupeKey: "item:43", - services, - }); + const firstRunId = await start(db, { dir: join(root, "dir-1"), dedupeKey: "item:43" }); expect(firstRunId).toBeTypeOf("string"); - await waitFor(() => !services.registry.isActive(firstRunId)); + await waitFor(() => !isActive(firstRunId)); - const retryRunId = await start(db, { - dir: join(root, "dir-2"), - dedupeKey: "item:43", - services, - }); - await waitFor(() => !services.registry.isActive(retryRunId)); + const retryRunId = await start(db, { dir: join(root, "dir-2"), dedupeKey: "item:43" }); + await waitFor(() => !isActive(retryRunId)); expect(getRunEvents(db, retryRunId).some((e) => e.payload._tag === "RunStarted")).toBe(true); - await drain(services); + await drain(); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -344,24 +339,21 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a failed start releases its claim so a corrected retry can start", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-fail-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + // allocation of an explicitly undefined dir fails after the claim await expect( - startTrackedRun(runtime, db, services, echoWorkflow, { + startTrackedRun(runtime, db, echoWorkflow, { input: {}, dedupeKey: "item:44", + dedupeRegistry: createDedupeRegistry(), }), ).rejects.toThrow(/needs `dir` or `workspace`/); - const retryRunId = await start(db, { - dir: join(root, "dir-1"), - dedupeKey: "item:44", - services, - }); - await waitFor(() => !services.registry.isActive(retryRunId)); + const retryRunId = await start(db, { dir: join(root, "dir-1"), dedupeKey: "item:44" }); + await waitFor(() => !isActive(retryRunId)); expect(getRunEvents(db, retryRunId).some((e) => e.payload._tag === "RunStarted")).toBe(true); - await drain(services); + await drain(); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -369,29 +361,31 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a cancelled run releases its key (RunCancelled is a terminal state)", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-cancel-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const registry = createDedupeRegistry(); - const holderRunId = await startTrackedRun(runtime, db, services, echoWorkflow, { + const holderRunId = await startTrackedRun(runtime, db, echoWorkflow, { runId: "run-cancelled", dir: join(root, "dir-1"), input: {}, dedupeKey: "item:45", + dedupeRegistry: registry, }); - await waitFor(() => !services.registry.isActive(holderRunId) || true); - const handle = services.registry.getActiveHandle("run-cancelled"); + await waitFor(() => !isActive(holderRunId) || true); + const handle = getActiveHandle("run-cancelled"); expect(handle).toBeDefined(); if (handle !== undefined) await handle.cancel(); - await waitFor(() => !services.registry.isActive(holderRunId)); + await waitFor(() => !isActive(holderRunId)); const started = getRunEvents(db, holderRunId).some((e) => e.payload._tag === "RunStarted"); - const retryRunId = await startTrackedRun(runtime, db, services, echoWorkflow, { + const retryRunId = await startTrackedRun(runtime, db, echoWorkflow, { dir: join(root, "dir-2"), input: {}, dedupeKey: "item:45", + dedupeRegistry: registry, }); expect(started).toBe(true); expect(retryRunId).toBeTypeOf("string"); - await drain(services); + await drain(); db.close(); rmSync(root, { recursive: true, force: true }); @@ -400,17 +394,14 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("runs started without a key behave exactly as before", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-none-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); - const firstRunId = await start(db, { dir: join(root, "dir-1"), services }); - const secondRunId = await start(db, { dir: join(root, "dir-2"), services }); + const firstRunId = await start(db, { dir: join(root, "dir-1") }); + const secondRunId = await start(db, { dir: join(root, "dir-2") }); expect(firstRunId).toBeTypeOf("string"); expect(secondRunId).toBeTypeOf("string"); - await waitFor( - () => !services.registry.isActive(firstRunId) && !services.registry.isActive(secondRunId), - ); + await waitFor(() => !isActive(firstRunId) && !isActive(secondRunId)); - await drain(services); + await drain(); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -434,11 +425,11 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { test("a key held by a running run surfaces as a 409 conflict naming the key and the holder", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-http-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); const slowStartWorkflow = defineWorkflow("dedupe-http-slow", { input: Schema.Struct({}), workspace: { kind: "scratch" }, + // Scratch: no clone needed. The key is claimed before the workflow runs. run: async (ctx) => { await ctx.exec(["sh", "-c", "sleep 0.5"]); return {}; @@ -446,9 +437,8 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { }); const server = serve({ - runtime, db, - services, + runtime: makeAgentRuntime(SLOW_ADAPTER), port: 0, config: defineConfig({ repo: { @@ -491,9 +481,11 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { expect(collision.error).toContain("issue:91"); expect(collision.error).toContain(holderRunId); + // A conflict is a start rejection, not a started run: nothing recorded. expect(getRunEvents(db, "nonexistent")).toHaveLength(0); await waitForTerminalStatus(db, holderRunId); + // Rejected while held; once terminal, a retry is a fresh 201. const retryRes = await fetch(`${base}/api/runs`, { method: "POST", body: JSON.stringify({ workflowId: "dedupe-http-slow", input: {}, dedupeKey: "issue:91" }), @@ -502,7 +494,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const { runId: retryRunId } = (await retryRes.json()) as { runId: string }; await waitForTerminalStatus(db, retryRunId); } finally { - await drain(services); + await drain(); await server.stop(true); db.close(); rmSync(root, { recursive: true, force: true }); @@ -512,7 +504,6 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { test("a non-string dedupeKey is a 400, and omitting one behaves as today", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-http-shape-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); const plainWorkflow = defineWorkflow("dedupe-http-plain", { input: Schema.Struct({}), workspace: { kind: "scratch" }, @@ -520,9 +511,8 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { }); const server = serve({ - runtime, db, - services, + runtime: makeAgentRuntime(SLOW_ADAPTER), port: 0, config: defineConfig({ repo: { @@ -553,7 +543,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const { runId } = (await plain.json()) as { runId: string }; await waitForTerminalStatus(db, runId); } finally { - await drain(services); + await drain(); await server.stop(true); db.close(); rmSync(root, { recursive: true, force: true }); diff --git a/src/server/http.test.ts b/src/server/http.test.ts index 9a37b4f..30e7676 100644 --- a/src/server/http.test.ts +++ b/src/server/http.test.ts @@ -12,10 +12,6 @@ import { defineConfig, loadFactoryConfig } from "../config"; import registryWorkflow from "../../test/fixtures/registry-workflow"; import registryScratchWorkflow from "../../test/fixtures/registry-scratch-workflow"; import { serve } from "./http"; -import { createRunRegistry } from "./runs"; -import { createPubSub } from "./pubsub"; -import { createDedupeRegistry } from "../lib/dedupe"; -import { createRefreshGates } from "../lib/workspace"; const ECHO_WORKFLOW = `${import.meta.dir}/../../test/fixtures/echo-workflow.ts`; const QUIET_GAP_WORKFLOW = `${import.meta.dir}/../../test/fixtures/quiet-gap-workflow.ts`; @@ -94,16 +90,8 @@ describe("GET /api/workflows (D30)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const runtime = makeAgentRuntime(adapter); const config = await loadFactoryConfig(FIXTURE_CONFIG); - const services = createTestServices(); - const server = serve({ - runtime, - services, - db, - port: 0, - config, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); const base = `http://localhost:${server.port}`; try { @@ -128,14 +116,7 @@ describe("GET /api/workflows (D30)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-empty-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const runtime = makeAgentRuntime(adapter); - const services = createTestServices(); - const server = serve({ - runtime, - services, - db, - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); const base = `http://localhost:${server.port}`; try { @@ -155,14 +136,7 @@ describe("GET /api/workflows (D30)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-empty-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const runtime = makeAgentRuntime(adapter); - const services = createTestServices(); - const server = serve({ - runtime, - services, - db, - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); const base = `http://localhost:${server.port}`; try { @@ -182,14 +156,7 @@ describe("phase 4 SPA serving", () => { const dir = mkdtempSync(join(tmpdir(), "factory-spa-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([{ type: "TEXT_MESSAGE_START" }], 1); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); const base = `http://localhost:${server.port}`; try { @@ -272,15 +239,7 @@ describe("SSE keepalive", () => { const dir = mkdtempSync(join(tmpdir(), "factory-sse-keepalive-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const runtime = makeAgentRuntime(adapter); - const services = createTestServices(); - const server = serve({ - runtime, - services, - db, - port: 0, - sseKeepaliveMs: 25, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, sseKeepaliveMs: 25 }); const base = `http://localhost:${server.port}`; try { @@ -316,15 +275,7 @@ describe("SSE keepalive", () => { const dir = mkdtempSync(join(tmpdir(), "factory-sse-keepalive-stop-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const runtime = makeAgentRuntime(adapter); - const services = createTestServices(); - const server = serve({ - runtime, - services, - db, - port: 0, - sseKeepaliveMs: 10, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, sseKeepaliveMs: 10 }); const base = `http://localhost:${server.port}`; try { @@ -364,14 +315,7 @@ describe("phase 3 HTTP API + SSE", () => { ], 1, ); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); const base = `http://localhost:${server.port}`; try { @@ -420,14 +364,7 @@ describe("phase 3 HTTP API + SSE", () => { ], 500, ); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); const base = `http://localhost:${server.port}`; try { @@ -463,14 +400,7 @@ describe("phase 3 HTTP API + SSE", () => { ], 60, ); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); const base = `http://localhost:${server.port}`; try { @@ -509,14 +439,7 @@ describe("phase 3 HTTP API + SSE", () => { ], 400, ); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); const base = `http://localhost:${server.port}`; try { @@ -554,14 +477,7 @@ describe("phase 3 HTTP API + SSE", () => { ], 1, ); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); const base = `http://localhost:${server.port}`; try { @@ -607,15 +523,7 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { 1, ); const config = await loadFactoryConfig(FIXTURE_CONFIG); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - config, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); const base = `http://localhost:${server.port}`; try { @@ -640,16 +548,8 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-d31-unknown-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const runtime = makeAgentRuntime(adapter); const config = await loadFactoryConfig(FIXTURE_CONFIG); - const services = createTestServices(); - const server = serve({ - runtime, - services, - db, - port: 0, - config, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); const base = `http://localhost:${server.port}`; try { @@ -695,15 +595,7 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { maxConcurrentRuns: 1, retainedWorkspaces: 10, }); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - config, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); const base = `http://localhost:${server.port}`; try { @@ -766,15 +658,7 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { maxConcurrentRuns: 3, retainedWorkspaces: 10, }); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - config, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); const base = `http://localhost:${server.port}`; try { @@ -841,15 +725,7 @@ describe("a scratch workflow through POST /api/runs (issue #13)", () => { maxConcurrentRuns: 3, retainedWorkspaces: 10, }); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); - const server = serve({ - runtime, - services, - db, - port: 0, - config, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); const base = `http://localhost:${server.port}`; try { @@ -920,12 +796,9 @@ describe("ctx.dispatch through POST /api/runs (issue #14)", () => { }, }); - const services = createTestServices(); - const runtime = makeAgentRuntime(adapter); const server = serve({ - runtime, - services, db, + runtime: makeAgentRuntime(adapter), port: 0, config: defineConfig({ repo: { @@ -1021,12 +894,9 @@ describe("GET /api/schedules (issue #17)", () => { workspace: { kind: "scratch" }, run: async () => ({}), }); - const services = createTestServices(); - const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ - runtime, - services, db, + runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config: schedulesConfig(root, workflow), }); @@ -1093,12 +963,9 @@ describe("GET /api/schedules (issue #17)", () => { }, ], }); - const services = createTestServices(); - const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ - runtime, - services, db, + runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1122,14 +989,7 @@ describe("GET /api/schedules (issue #17)", () => { test("serves an empty list on a legacy no-config server", async () => { const root = mkdtempSync(join(tmpdir(), "factory-schedules-empty-test-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); - const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); - const server = serve({ - runtime, - services, - db, - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0 }); const base = `http://localhost:${server.port}`; try { @@ -1171,12 +1031,9 @@ describe("GET /api/schedules (issue #17)", () => { }, ], }); - const services = createTestServices(); - const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ - runtime, - services, db, + runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1275,12 +1132,9 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }, ], }); - const services = createTestServices(); - const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ - runtime, - services, db, + runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1332,12 +1186,9 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { { id: "nightly-run", workflow: workflow.id, input: {}, cron: "0 3 * * *", timezone: "UTC" }, ], }); - const services = createTestServices(); - const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ - runtime, - services, db, + runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1378,12 +1229,9 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { { id: "slow-skip", workflow: workflow.id, input: {}, cron: "0 3 * * *", timezone: "UTC" }, ], }); - const services = createTestServices(); - const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ - runtime, - services, db, + runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1446,12 +1294,9 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }, ], }); - const services = createTestServices(); - const runtime = makeAgentRuntime(createSlowFakeAdapter([], 1)); const server = serve({ - runtime, - services, db, + runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1477,15 +1322,6 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }); }); -function createTestServices() { - return { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; -} - /** * #37 / #24: the legacy `{dir, clone}` POST resets a clone on `dir`, so the * adapter must prepare that dir; a bare `dir` is caller-managed and is not. @@ -1509,12 +1345,7 @@ describe("adapter.prepareWorkspace through the legacy POST /api/runs (#37)", () const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); const adapter = trackingAdapter(); - const server = serve({ - db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), - port: 0, - }); + const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); const base = `http://localhost:${server.port}`; try { diff --git a/src/server/http.ts b/src/server/http.ts index b792aa0..45c2abb 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -33,24 +33,29 @@ import type { Database } from "bun:sqlite"; import { isTerminal, type RunEvent } from "../events"; import type { FactoryConfig } from "../config"; -import { Schema, SchemaParser } from "effect"; +import { Schema, SchemaParser, type ManagedRuntime } from "effect"; import { resetClone, type GitIdentity } from "../lib/clone"; import { loadWorkflow } from "../lib/load-workflow"; import { getRunEvents, listRuns, type RunSummary } from "../persistence/store"; -import type { ManagedRuntime } from "effect"; import type { AgentRuntime } from "../runtime/agent-runtime"; import index from "../web/index.html"; import { admitRun } from "./admission"; import { DedupeKeyError } from "../lib/dedupe"; import { ConcurrencyLimitError, + startTrackedRun, type DaemonServices, type DispatchEnv, type StartTrackedRunOptions, - startTrackedRun, } from "./runs"; import { nextFireAt, toRuntimeSchedules } from "./scheduler"; +/** + * Issue #17: what `GET /api/schedules` serves for one schedule — the config's + * own record (id, workflow, input, cron, timezone, overlap, run-on-start), + * the next fire time computed from the stored cron, and the most recent run + * that carries the schedule's id. + */ export interface ScheduleSummary { readonly id: string; readonly workflowId: string; @@ -59,7 +64,9 @@ export interface ScheduleSummary { readonly timezone: string; readonly overlap: "skip" | "stack"; readonly runOnStart: boolean; + /** Epoch ms of the cron's next fire after now. */ readonly nextFireAt: number; + /** The schedule's latest run — scheduled *or* manually triggered. */ readonly lastRun: | { readonly runId: string; readonly status: string; readonly startedAt: number } | undefined; @@ -67,6 +74,7 @@ export interface ScheduleSummary { export interface ServerOptions { readonly db: Database; + /** Issue #38: this daemon's run registry, pubsub, dedupe registry and refresh gates. */ readonly services: DaemonServices; /** * Issue #36: the Effect managed runtime that provides the agent runtime @@ -74,18 +82,62 @@ export interface ServerOptions { * rather than threaded through `ServerOptions`. */ readonly runtime: ManagedRuntime.ManagedRuntime; + /** + * The loaded `factory.config.ts` (D27). Absent = the phase 3 path-based API + * behaves exactly as before (no limit, explicit dir+clone, `/api/workflows` + * serves an empty list) — the dispatcher legitimately keeps supplying + * filesystem paths (D31). + */ readonly config?: FactoryConfig; + /** + * How often an open SSE stream emits its keepalive comment frame. Exposed + * only so tests can turn it down far enough for a sub-second quiet gap to + * exercise the timer; production has no reason to set it. + */ readonly sseKeepaliveMs?: number; } +/** + * `Bun.serve`'s `idleTimeout` defaults to **10 seconds** and closes any + * connection with no traffic in that window. A run's SSE stream only carries + * traffic when the workflow emits an event, so any quiet gap longer than that — + * `ctx.exec` running a test suite, write-back's `git push` + `gh pr create`, an + * agent thinking before its first chunk — dropped the connection under a + * perfectly healthy run. The SPA has no way to tell that apart from a dead + * process and rendered the run "interrupted" until a manual refresh. + * + * A comment frame on a timer is the fix rather than a raised `idleTimeout`, + * because Bun caps `idleTimeout` at 255s: a single long agent step would still + * outlast it, whereas a keepalive holds a stream open for any gap length. + * SSE comment frames (`:`-prefixed, no `data:` line) are inert to every client + * we ship — the SPA's `dataLine` (`web/api.ts`) and the CLI's tail both skip + * frames without a `data:` line. + * + * 5s leaves 2x margin under the 10s default. It is deliberately not derived + * from `idleTimeout`: we do not set that option, so the default is the contract. + */ const DEFAULT_SSE_KEEPALIVE_MS = 5_000; +/** `RunSummary` plus this process's live-registry bit — what the SPA reads. */ export type RunSummaryResponse = RunSummary & { readonly active: boolean }; +/** + * A run with no terminal event is `"interrupted"` in the store, which cannot + * tell a crash apart from a run this process still holds. The runs page needs + * that bit to group live rows, so it is derived here from the in-memory + * registry (`server/runs.ts`) rather than stored (D24: active is process state). + */ function listSummaries(db: Database, services: DaemonServices): ReadonlyArray { return listRuns(db).map((run) => ({ ...run, active: services.registry.isActive(run.runId) })); } +/** + * Issue #14: the dispatch environment a run's `ctx.dispatch` shares with its + * child runs — the same config wiring the run itself gets (workspace root, + * repo, adapter, admission). Present only for config-backed servers; without + * it `ctx.dispatch` throws, because in-process/path-based legacy runs have + * no registry to start children from. + */ function dispatchEnvFor(config: FactoryConfig): DispatchEnv { return { workspace: { @@ -101,6 +153,12 @@ function dispatchEnvFor(config: FactoryConfig): DispatchEnv { }; } +/** + * The start options every config-backed `startTrackedRun` shares: D28's + * workspace allocation, the run environment, D29's admission limit and #14's + * dispatch env. Used by the registry POST path and, since issue #17, by the + * manual schedule trigger. + */ function configRunOptions( config: FactoryConfig, maxConcurrentRuns: number | undefined, @@ -130,6 +188,11 @@ interface StartRunBody { readonly workflowId?: unknown; readonly workflowPath?: unknown; readonly input?: unknown; + /** + * Issue #15: an optional dedupe key. While a non-terminal run holds it, + * another start with the same key is a 409 conflict naming the key and the + * holding run — never a silently dropped duplicate. + */ readonly dedupeKey?: unknown; readonly dir?: string; readonly clone?: { readonly sshUrl: string; readonly identity: GitIdentity }; @@ -142,6 +205,11 @@ function json(body: unknown, init?: { readonly status?: number }): Response { }); } +/** + * The SSE `Last-Event-ID` header, or `undefined` if absent or not a sequence + * number. `EventSource` sends it automatically on reconnect, and `seq` is the + * value we stamp on each frame's `id:` line, so it is the resume offset. + */ function parseLastEventId(req: Request): number | undefined { const raw = req.headers.get("last-event-id"); if (raw === null || !/^\d+$/.test(raw)) return undefined; @@ -157,6 +225,9 @@ function sseStream( ): ReadableStream { const encoder = new TextEncoder(); + // `closed`/`unsubscribe`/`keepalive` live on the stream (not just `start`) so + // the `cancel()` hook below can also mark them when a client disconnects + // without anyone having sent a terminal event. let closed = false; let unsubscribe: () => void = () => undefined; let keepalive: ReturnType | undefined; @@ -167,6 +238,9 @@ function sseStream( return new ReadableStream({ start(controller) { + // `Last-Event-ID` makes this resume: seed `lastSeq` from it and the + // persisted replay below skips everything the client already has, exactly + // as the live tail already does. `seq` is the offset (D20/D26). let lastSeq = lastEventId ?? -1; let draining = false; const buffered: Array = []; @@ -178,6 +252,8 @@ function sseStream( encoder.encode(`id: ${event.seq}\ndata: ${JSON.stringify(event)}\n\n`), ); } catch { + // The client is gone (navigated away, fetch aborted) — Bun has + // already closed this controller. Stop pushing and leave the fan-out. closed = true; stopKeepalive(); unsubscribe(); @@ -222,6 +298,10 @@ function sseStream( return; } + // The run is still live, so this stream stays open indefinitely and must + // beat `idleTimeout` on its own. The same enqueue guard as `send`: once + // Bun has closed the controller under us, stop the timer rather than + // throw outside request context. keepalive = setInterval(() => { if (closed) { stopKeepalive(); @@ -237,6 +317,10 @@ function sseStream( }, keepaliveMs); }, cancel(): void { + // A navigating/aborted client must drop its pubsub subscription and its + // keepalive timer; otherwise a later event (or tick) would enqueue into a + // controller Bun has already closed, throwing outside request context — + // which kills the whole serve() process. closed = true; stopKeepalive(); unsubscribe(); @@ -263,6 +347,11 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise } if (req.method === "POST" && url.pathname === "/api/runs") { + // D29: the one admission function, consulted here (refused atomically + // inside `startTrackedRun` — the registry slot is reserved before any + // await, so concurrent starts cannot lose the race) and on the dispatch + // path alike. 409 over the limit; the limit is only consulted when a + // config is present (the no-config legacy path kept its old behaviour). const maxConcurrentRuns = options.config?.maxConcurrentRuns; if ( maxConcurrentRuns !== undefined && @@ -385,6 +474,9 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise ...(runEnv !== undefined ? { dispatchEnv: dispatchEnvFor(runEnv) } : {}), input: body.input, ...(typeof body.dedupeKey === "string" ? { dedupeKey: body.dedupeKey } : {}), + ...(body.clone !== undefined && typeof body.dir === "string" + ? { prepareWorkspace: true } + : {}), }; let runId: string; try { @@ -410,9 +502,18 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise return json({ runId }, { status: 201 }); } + /** + * Issue #17: the schedules exactly as the running daemon carries them — + * read from `options.config`, the same object the scheduler loop rides on, + * never a separate persisted copy. Next fire computed from the stored cron + * (the payoff for keeping cron as cron), the last run matched on the + * schedule id recorded in `RunStarted`. + */ if (req.method === "GET" && url.pathname === "/api/schedules") { const config = options.config; const schedules = config?.schedules ?? []; + // `listRuns` is newest-first, so the first hit per schedule id is its + // most recent run. const lastRunBySchedule = new Map(); for (const run of listRuns(options.db)) { if (run.scheduleId !== undefined && !lastRunBySchedule.has(run.scheduleId)) { @@ -461,6 +562,8 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise } const workflow = options.config!.workflows.find((w) => w.id === schedule.workflowId)!; + // D29 admission, checked here exactly like POST /api/runs so a manual + // trigger over the limit is a visible 409 rather than a surprise. const maxConcurrentRuns = options.config.maxConcurrentRuns; if (!admitRun(maxConcurrentRuns, services.registry.activeRunIds().length)) { return json( @@ -469,6 +572,11 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise ); } + // Overlap policy (the epic's dispatch decision): under `"skip"` the + // manual run takes the schedule's own dedupe key, so a trigger while + // another run holds it throws `DedupeKeyError` — surfaced as a 409 + // naming the key and the holder, never a silent no-op. Under `"stack"` + // it fires regardless. let runId: string; try { runId = await startTrackedRun(options.runtime, options.db, services, workflow, { @@ -541,6 +649,11 @@ export function serve(options: ServerOptions & { port?: number }): ReturnType handler(req), "/*": index, }, + // Bun's `development: true` HMR mode crashes TanStack Router at boot + // (`Cannot read properties of null (reading 'replaceRouteChunk')` from + // router-core's dev-only prototype patch). Runtime bundling with + // `development: false` still serves the same HTML-route bundle, cached and + // minified, so the POC takes correctness over hot reload. See STATUS S2. development: false, }); } diff --git a/src/server/nested-runs.test.ts b/src/server/nested-runs.test.ts index 14c8e60..7c9e8ac 100644 --- a/src/server/nested-runs.test.ts +++ b/src/server/nested-runs.test.ts @@ -8,8 +8,6 @@ * carries the message). A validated child is then started fire-and-forget * with the parent's run id recorded on its `RunStarted.parentId`, and the * parent never exposes an awaitable child (D-epic 19). - * - * #38: tests create their own DaemonServices. */ import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; @@ -20,12 +18,9 @@ import { describe, expect, test } from "bun:test"; import { Schema } from "effect"; import { getRunEvents, listRuns, openStore } from "../persistence/store"; import { createSlowFakeAdapter } from "../replay/adapter"; -import { makeAgentRuntime } from "../runtime/agent-runtime"; -import { createRunRegistry, startTrackedRun, type DaemonServices } from "./runs"; -import { createPubSub } from "./pubsub"; -import { createDedupeRegistry } from "../lib/dedupe"; -import { createRefreshGates } from "../lib/workspace"; +import { activeRunIds, isActive, startTrackedRun } from "./runs"; import { defineWorkflow } from "../workflow"; +import { makeAgentRuntime } from "../runtime/agent-runtime"; const SLOW_ADAPTER = createSlowFakeAdapter( [ @@ -38,6 +33,8 @@ const SLOW_ADAPTER = createSlowFakeAdapter( const runtime = makeAgentRuntime(SLOW_ADAPTER); +// The child does a real ctx.exec so its outcome is observable in its log. +// Scratch (issue #13): no mirror refresh, so no git fixture is needed. const numberedChild = defineWorkflow("child-wf", { input: Schema.Struct({ n: Schema.Finite }), workspace: { kind: "scratch" }, @@ -67,15 +64,6 @@ const manyChildren = defineWorkflow("many-children-wf", { }, }); -function createTestServices(): DaemonServices { - return { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; -} - async function waitFor(predicate: () => boolean, timeoutMs = 10_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -94,7 +82,6 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-happy-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); const parent = defineWorkflow("parent-wf", { input: Schema.Struct({}), @@ -105,7 +92,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, services, parent, { + const parentRunId = await startTrackedRun(runtime, db, parent, { workspace: { workspaceRoot: join(root, "workspaces"), sshUrl: join(root, "seed-not-used"), @@ -123,10 +110,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor( - () => - !services.registry.isActive(parentRunId) && services.registry.activeRunIds().length === 0, - ); + await waitFor(() => !isActive(parentRunId) && activeRunIds().length === 0); const parentEvents = getRunEvents(db, parentRunId); const dispatched = parentEvents.find((e) => e.payload._tag === "RunDispatched"); @@ -149,6 +133,7 @@ describe("nested runs through the daemon (issue #14)", () => { : "missing", ).toBe("child-wf"); + // The child actually ran to completion under the daemon. const childSummary = listRuns(db).find((run) => run.runId === childRunId); expect(childSummary?.status).toBe("RunFinished"); @@ -160,9 +145,8 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-depth-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); - await startTrackedRun(runtime, db, services, selfDispatching, { + await startTrackedRun(runtime, db, selfDispatching, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -182,8 +166,9 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => services.registry.activeRunIds().length === 0); + await waitFor(() => activeRunIds().length === 0); + // The chain built to the cap, no farther: root + 5 children max. const selfRuns = listRuns(db).filter((run) => run.workflowId === "self-dispatch-wf"); const failedStates = new Set(["RunFailed", "RunCancelled", "interrupted"]); const completed = selfRuns.filter( @@ -211,9 +196,8 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-count-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); - const parentRunId = await startTrackedRun(runtime, db, services, manyChildren, { + const parentRunId = await startTrackedRun(runtime, db, manyChildren, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -234,7 +218,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => services.registry.activeRunIds().length === 0); + await waitFor(() => activeRunIds().length === 0); expect(tagsOf(db, parentRunId).filter((tag) => tag === "RunDispatched").length).toBe(5); const parentEvents = getRunEvents(db, parentRunId); @@ -253,7 +237,6 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-wip-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); const parent = defineWorkflow("parent-wip-wf", { input: Schema.Struct({}), @@ -264,7 +247,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, services, parent, { + const parentRunId = await startTrackedRun(runtime, db, parent, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -284,7 +267,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => !services.registry.isActive(parentRunId)); + await waitFor(() => !isActive(parentRunId)); const events = getRunEvents(db, parentRunId); expect(events.map((event) => event.payload._tag)).not.toContain("RunDispatched"); diff --git a/src/server/runs.test.ts b/src/server/runs.test.ts index db01e48..7e050bc 100644 --- a/src/server/runs.test.ts +++ b/src/server/runs.test.ts @@ -7,9 +7,6 @@ * - a `cancel` arriving while a run is only a reserved slot is deferred into * run start (L1): the run starts, is cancelled immediately, and ends as a * clean RunCancelled with no orphaned slot. - * - * #38: tests create their own DaemonServices instead of using module-level - * singletons or test seams. */ import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; @@ -25,13 +22,13 @@ import { makeAgentRuntime } from "../runtime/agent-runtime"; import { defineWorkflow, Schema } from "../workflow"; import { ConcurrencyLimitError, - createRunRegistry, + DispatchCapError, + activeRunIds, + cancelRegisteredRun, + getActiveHandle, + isActive, startTrackedRun, - type DaemonServices, } from "./runs"; -import { createPubSub } from "./pubsub"; -import { createDedupeRegistry } from "../lib/dedupe"; -import { createRefreshGates } from "../lib/workspace"; const SLOW_ADAPTER = createSlowFakeAdapter( [ @@ -57,13 +54,17 @@ function tmpRoot(): { root: string; finish: () => void } { return { root, finish: () => rmSync(root, { recursive: true, force: true }) }; } -function createTestServices(): DaemonServices { - return { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; +interface Gate { + readonly wait: () => Promise; + readonly release: () => void; +} + +function makeGate(): Gate { + let release: () => void = () => undefined; + const promise = new Promise((resolve) => { + release = resolve; + }); + return { wait: () => promise, release }; } async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { @@ -75,32 +76,47 @@ async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { + test("ConcurrencyLimitError carries _tag and maxConcurrentRuns", () => { + const err = ConcurrencyLimitError.of({ maxConcurrentRuns: 5 }); + expect(err._tag).toBe("ConcurrencyLimitError"); + expect(err.maxConcurrentRuns).toBe(5); + expect(err instanceof Error).toBe(true); + }); + + test("DispatchCapError carries _tag and message", () => { + const err = new DispatchCapError({ message: "depth exceeded" }); + expect(err._tag).toBe("DispatchCapError"); + expect(err.message).toContain("depth exceeded"); + expect(err instanceof Error).toBe(true); + }); + + test("ConcurrencyLimitError constructs its message, so the stack header names it", () => { + const err = ConcurrencyLimitError.of({ maxConcurrentRuns: 5 }); + expect(err.message).toBe("concurrency limit reached (max 5 concurrent runs)"); + expect(err.stack?.split("\n")[0]).toBe(`ConcurrencyLimitError: ${err.message}`); + }); +}); + describe("startTrackedRun admission (M1: the slot is reserved before any await)", () => { test("a second start while the first is still reserving is refused atomically, and the slot frees after", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const gate = makeGate(); - const seed = join(root, "seed"); - await Bun.$`git init -b main -q ${seed}`.quiet(); - await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); - - const first = startTrackedRun(runtime, db, services, echoWorkflow, { + const first = startTrackedRun(runtime, db, echoWorkflow, { runId: "run-first", - workspace: { - workspaceRoot: join(root, "workspaces"), - sshUrl: seed, - identity: { name: "T", email: "t@t.test" }, - retainedWorkspaces: 10, - }, + dir: join(root, "first-dir"), input: {}, maxConcurrentRuns: 1, + beforeStart: gate.wait, }); - await waitFor(() => services.registry.activeRunIds().length === 1); - expect(services.registry.activeRunIds()).toEqual(["run-first"]); + await waitFor(() => activeRunIds().length === 1); + expect(activeRunIds()).toEqual(["run-first"]); + expect(getActiveHandle("run-first")).toBeUndefined(); - const second = startTrackedRun(runtime, db, services, echoWorkflow, { + const second = startTrackedRun(runtime, db, echoWorkflow, { runId: "run-second", dir: join(root, "second-dir"), input: {}, @@ -114,10 +130,11 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" ), ).resolves.toBeInstanceOf(ConcurrencyLimitError); + gate.release(); const firstRunId = await first; expect(firstRunId).toBe("run-first"); - await waitFor(() => !services.registry.isActive(firstRunId)); - expect(services.registry.activeRunIds()).toEqual([]); + await waitFor(() => !isActive(firstRunId)); + expect(activeRunIds()).toEqual([]); db.close(); finish(); @@ -126,10 +143,9 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" test("a failed allocation releases the reserved slot", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); await expect( - startTrackedRun(runtime, db, services, echoWorkflow, { + startTrackedRun(runtime, db, echoWorkflow, { runId: "run-doomed", input: {}, maxConcurrentRuns: 1, @@ -142,15 +158,15 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" }), ).rejects.toThrow(/workspace/); - expect(services.registry.activeRunIds()).toEqual([]); + expect(activeRunIds()).toEqual([]); - const runId = await startTrackedRun(runtime, db, services, echoWorkflow, { + const runId = await startTrackedRun(runtime, db, echoWorkflow, { dir: join(root, "dir"), input: {}, maxConcurrentRuns: 1, }); - expect(services.registry.activeRunIds()).toEqual([runId]); - await waitFor(() => !services.registry.isActive(runId)); + expect(activeRunIds()).toEqual([runId]); + await waitFor(() => !isActive(runId)); db.close(); finish(); @@ -161,40 +177,35 @@ describe("cancel of a reserved-but-not-started run (L1)", () => { test("cancelling during the allocation window defers into run start; the run ends cancelled with no leak", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); - - const seed = join(root, "seed"); - await Bun.$`git init -b main -q ${seed}`.quiet(); - await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); + mkdirSync(join(root, "dir"), { recursive: true }); + const gate = makeGate(); - const starting = startTrackedRun(runtime, db, services, sleepWorkflow, { + const starting = startTrackedRun(runtime, db, sleepWorkflow, { runId: "run-gated", - workspace: { - workspaceRoot: join(root, "workspaces"), - sshUrl: seed, - identity: { name: "T", email: "t@t.test" }, - retainedWorkspaces: 10, - }, + dir: join(root, "dir"), input: {}, maxConcurrentRuns: 1, + beforeStart: gate.wait, }); - await waitFor(() => services.registry.activeRunIds().length === 1); + await waitFor(() => activeRunIds().length === 1); + expect(getActiveHandle("run-gated")).toBeUndefined(); - const cancelled = services.registry.cancelRegisteredRun("run-gated"); - expect(cancelled).toBeDefined(); + const cancelled = cancelRegisteredRun("run-gated"); + expect(cancelled).toEqual({ kind: "reserved" }); + gate.release(); const runId = await starting; expect(runId).toBe("run-gated"); - await waitFor(() => services.registry.activeRunIds().length === 0); + await waitFor(() => activeRunIds().length === 0); const events = getRunEvents(db, runId).map((e) => e.payload._tag); expect(events).toContain("RunStarted"); expect(events).toContain("RunCancelled"); - expect(services.registry.cancelRegisteredRun(runId)).toBeUndefined(); + expect(cancelRegisteredRun(runId)).toBeUndefined(); await Bun.sleep(50); - expect(services.registry.activeRunIds()).toEqual([]); + expect(activeRunIds()).toEqual([]); db.close(); finish(); @@ -237,17 +248,17 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { _tmp = root; const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "seed"), { recursive: true }); - const services = createTestServices(); - const runId = await startTrackedRun(runtime, db, services, failingScratchWorkflow, { + const runId = await startTrackedRun(runtime, db, failingScratchWorkflow, { runId: "run-scratch-empty", workspace: { ...workspaceSpec(), sshUrl: join(root, "no-such-remote") }, input: {}, }); - await waitFor(() => !services.registry.isActive(runId)); + await waitFor(() => !isActive(runId)); const dir = join(root, "workspaces", "run-scratch-empty"); expect(existsSync(dir)).toBe(true); + // failing, so the dir survives for inspection (a completed scratch dir is reaped). expect(readdirSync(dir)).toEqual([]); expect(existsSync(join(root, "workspaces", ".mirror.git"))).toBe(false); finish(); @@ -258,23 +269,22 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { _tmp = root; const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "seed"), { recursive: true }); - const services = createTestServices(); - const okId = await startTrackedRun(runtime, db, services, scratchWorkflow, { + const okId = await startTrackedRun(runtime, db, scratchWorkflow, { runId: "run-scratch-ok", workspace: workspaceSpec(), input: {}, }); - await waitFor(() => !services.registry.isActive(okId)); - await new Promise((resolve) => setTimeout(resolve, 50)); + await waitFor(() => !isActive(okId)); + await new Promise((resolve) => setTimeout(resolve, 50)); // reap lands async expect(existsSync(join(root, "workspaces", "run-scratch-ok"))).toBe(false); - const badId = await startTrackedRun(runtime, db, services, failingScratchWorkflow, { + const badId = await startTrackedRun(runtime, db, failingScratchWorkflow, { runId: "run-scratch-bad", workspace: workspaceSpec(), input: {}, }); - await waitFor(() => !services.registry.isActive(badId)); + await waitFor(() => !isActive(badId)); await new Promise((resolve) => setTimeout(resolve, 50)); expect(existsSync(join(root, "workspaces", "run-scratch-bad"))).toBe(true); finish(); @@ -287,6 +297,7 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); + // A kept (failed) scratch dir, already recorded in the log. appendEvent(db, { runId: "run-scratch-kept", seq: 0, @@ -300,14 +311,15 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { }, } satisfies RunEvent); mkdirSync(join(root, "workspaces", "run-scratch-kept"), { recursive: true }); - const services = createTestServices(); - await startTrackedRun(runtime, db, services, echoWorkflow, { + // retention = 1: with the scratch dir correctly excluded, the only clone + // survives: the leftover must not count toward retention. + await startTrackedRun(runtime, db, echoWorkflow, { runId: "run-clone", workspace: { ...workspaceSpec(), sshUrl: seed, retainedWorkspaces: 1 }, input: {}, }); - await waitFor(() => !services.registry.isActive("run-clone")); + await waitFor(() => !isActive("run-clone")); expect(existsSync(join(root, "workspaces", "run-clone"))).toBe(true); expect(existsSync(join(root, "workspaces", "run-scratch-kept"))).toBe(true); @@ -345,9 +357,8 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); const adapter = trackingAdapter(); - const services = createTestServices(); - const runId = await startTrackedRun(makeAgentRuntime(adapter), db, services, echoWorkflow, { + const runId = await startTrackedRun(makeAgentRuntime(adapter), db, echoWorkflow, { runId: "run-prep-clone", workspace: { workspaceRoot: join(root, "workspaces"), @@ -357,7 +368,7 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { }, input: {}, }); - await waitFor(() => !services.registry.isActive(runId)); + await waitFor(() => !isActive(runId)); expect(adapter.prepared).toEqual([join(root, "workspaces", "run-prep-clone")]); finish(); @@ -367,9 +378,8 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); const adapter = trackingAdapter(); - const services = createTestServices(); - const runId = await startTrackedRun(makeAgentRuntime(adapter), db, services, scratchWorkflow, { + const runId = await startTrackedRun(makeAgentRuntime(adapter), db, scratchWorkflow, { runId: "run-prep-scratch", workspace: { workspaceRoot: join(root, "workspaces"), @@ -379,7 +389,7 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { }, input: {}, }); - await waitFor(() => !services.registry.isActive(runId)); + await waitFor(() => !isActive(runId)); expect(adapter.prepared).toEqual([]); finish(); diff --git a/src/server/runs.ts b/src/server/runs.ts index b8da2d4..a5d20be 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -17,9 +17,9 @@ * coexist in one process without sharing state. */ +import { Schema, type ManagedRuntime } from "effect"; import type { Database } from "bun:sqlite"; import { rm } from "node:fs/promises"; -import { Schema, type ManagedRuntime } from "effect"; import { admitRun } from "./admission"; import { appendEvent, getRunEvents, listRuns } from "../persistence/store"; import type { RunRepo } from "../runtime/run"; @@ -35,10 +35,6 @@ interface ReservedSlot { cancelled: boolean; } -function isReserved(entry: RunHandle | ReservedSlot | undefined): boolean { - return entry !== undefined && !("result" in entry) && "cancelled" in entry; -} - /** * Issue #14: how deep a parent → child → grandchild chain may nest — the cap * that keeps a workflow which dispatches itself from filling the daemon. @@ -52,6 +48,10 @@ export class DispatchCapError extends Schema.TaggedError()("Di message: Schema.String, }) {} +function isReserved(entry: RunHandle | ReservedSlot | undefined): boolean { + return entry !== undefined && !("result" in entry) && "cancelled" in entry; +} + export class ConcurrencyLimitError extends Schema.TaggedError()( "ConcurrencyLimitError", { maxConcurrentRuns: Schema.Number, message: Schema.String }, @@ -199,6 +199,13 @@ export interface StartTrackedRunOptions { * through to the run's model precedence chain. */ readonly agentOverrides?: { readonly model?: string }; + /** + * ADR 0012 §3 (#37): when true, the runtime calls `adapter.prepareWorkspace` + * before the workflow runs. The caller sets this when it has done a + * `resetClone` on `dir`. Combined with the daemon's own allocation check, + * this covers both clone paths. + */ + readonly prepareWorkspace?: boolean; } /** @@ -414,6 +421,8 @@ export async function startTrackedRun( dir, ...(options.repo !== undefined ? { repo: options.repo } : {}), workspaceKind: kind, + prepareWorkspace: + options.prepareWorkspace === true || (workspaceAllocated && kind === "clone"), ...(dispatch !== undefined ? { dispatch } : {}), ...(options.parentRunId !== undefined ? { parentRunId: options.parentRunId } : {}), ...(options.dedupeKey !== undefined ? { dedupeKey: options.dedupeKey } : {}), diff --git a/src/server/scheduler.ts b/src/server/scheduler.ts index 6dcb9a2..afc8e8f 100644 --- a/src/server/scheduler.ts +++ b/src/server/scheduler.ts @@ -33,9 +33,9 @@ import type { FactoryConfig } from "../config"; import { DedupeKeyError, type DedupeRegistry } from "../lib/dedupe"; import type { WorkflowDefinition } from "../workflow"; import { - startTrackedRun, ConcurrencyLimitError, DispatchCapError, + startTrackedRun, type DaemonServices, type DispatchEnv, type WorkspaceSpec, @@ -47,6 +47,7 @@ export class SchedulerError extends Schema.TaggedError()("Schedu cause: Schema.Defect(), }) {} +/** A config schedule resolved into the runtime's terms: workflow definition in hand, cron parsed. */ export interface RuntimeSchedule { readonly id: string; readonly workflowId: string; @@ -62,10 +63,17 @@ export function scheduleDedupeKey(schedule: RuntimeSchedule): string { return `schedule:${schedule.id}`; } +/** + * Issue #17: when the schedule fires next, computed from the stored cron as + * of `now` — the concrete payoff for keeping cron as a cron string. The same + * `Cron.next` the loop itself uses, exposed for `GET /api/schedules` and the + * UI's next-fire display. + */ export function nextFireAt(schedule: RuntimeSchedule, now: number): number { return Cron.next(schedule.cron, new Date(now)).getTime(); } +/** Resolves config's schedules (already validated at load) against the registry and parses each cron. */ export function toRuntimeSchedules(config: FactoryConfig): ReadonlyArray { return config.schedules.map((schedule) => { const workflow = config.workflows.find((w) => w.id === schedule.workflowId); @@ -89,16 +97,26 @@ export function toRuntimeSchedules(config: FactoryConfig): ReadonlyArray; + /** Starts the scheduled run; the daemon wires it to `startTrackedRun`. */ readonly fire: (schedule: RuntimeSchedule) => Promise; + /** Issue #38: the daemon's own dedupe registry — where schedule keys are held. */ readonly dedupeRegistry: DedupeRegistry; + /** Issue #38: the wall clock windows are judged against; tests pass a `TestClock`. */ readonly clock: Clock.Clock; } export interface SchedulerState { + /** Per schedule: the wall-clock the last tick observed (initialized to session start). */ readonly lastTick: Map; + /** Schedules still owed their single run-on-start fire this session. */ readonly runOnStartPending: Set; } +/** + * The state a fresh daemon session starts with: `lastTick` = now — the + * no-catch-up rule — and every `runOnStart` schedule marked pending exactly + * once per session. + */ export function createSchedulerState(deps: SchedulerDeps): SchedulerState { const now = deps.clock.currentTimeMillisUnsafe(); const schedules = deps.schedules; @@ -167,6 +185,10 @@ export async function tickOnce( const lastTick = state.lastTick.get(schedule.id) ?? now; state.lastTick.set(schedule.id, now); + // runOnStart fires on the first tick of the session, before any cron + // window; once attempted it is consumed whether or not the fire + // succeeded (the next cron window is the retry, not a second + // start-fire). const onStart = state.runOnStartPending.has(schedule.id); if (onStart) state.runOnStartPending.delete(schedule.id); @@ -199,6 +221,7 @@ export async function tickOnce( return results; } +/** The daemon's scheduler loop — `Effect.repeat` around the plain `tickOnce`. */ export function runSchedulerLoop( deps: SchedulerDeps, state: SchedulerState, @@ -225,6 +248,12 @@ export function runSchedulerLoop( return Effect.repeat(tick, Schedule.spaced(intervalMs)); } +/** + * The daemon's fire function: a scheduled run is a config-backed run started + * like any other — registry input, workspace, dispatch environment — made + * self-deduping with `schedule:` (issue #15's registry) so the overlap + * policy and the trigger record both come for free. + */ export function makeScheduleFire(options: { readonly db: Database; readonly runtime: ManagedRuntime.ManagedRuntime; @@ -239,7 +268,7 @@ export function makeScheduleFire(options: { return startTrackedRun(env.runtime, env.db, env.services, schedule.workflow, { input: schedule.input, repo: env.repo, - maxConcurrentRuns: env.maxConcurrentRuns, + maxConcurrentRuns: options.maxConcurrentRuns, workspace: env.workspace, dispatchEnv: env.dispatchEnv, scheduleId: schedule.id, From 8008b28212cd89ace0b26ada6672ebf1fe1e1e07 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 09:32:07 +0000 Subject: [PATCH 18/26] test(server): restore the invariant assertions the per-daemon rewrite weakened The reserved-but-not-started window is now held deterministically by pre-seeding the daemon's refresh gate for the workspace mirror, replacing the removed `beforeStart` seam rather than relying on git clone timing. Restores the strong M1/L1 assertions, the #34 TaggedError block, the DedupeKeyError `_tag` check and the deleted test comments; adds a synchronous second-start refusal check; drops the stale `adapter` DispatchEnv cast; and makes the two-daemon test check dedupe holding and pubsub isolation while runs are live. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/workspace.test.ts | 16 --- src/server/dedupe.test.ts | 198 ++++++++++++++++++++++----------- src/server/http.test.ts | 156 +++++++++++++++++++++++--- src/server/nested-runs.test.ts | 39 +++++-- src/server/runs.test.ts | 160 +++++++++++++++++++------- src/server/scheduler.test.ts | 10 ++ src/server/two-daemons.test.ts | 77 +++++++++++-- 7 files changed, 496 insertions(+), 160 deletions(-) diff --git a/src/lib/workspace.test.ts b/src/lib/workspace.test.ts index f45ef68..2a107bf 100644 --- a/src/lib/workspace.test.ts +++ b/src/lib/workspace.test.ts @@ -30,7 +30,6 @@ describe("allocateWorkspace (D28)", () => { const dir = await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-a", workspaceRoot, sshUrl: seed, @@ -56,7 +55,6 @@ describe("allocateWorkspace (D28)", () => { await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-one", workspaceRoot, sshUrl: seed, @@ -70,7 +68,6 @@ describe("allocateWorkspace (D28)", () => { const dir = await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-two", workspaceRoot, sshUrl: seed, @@ -90,7 +87,6 @@ describe("allocateWorkspace (D28)", () => { for (const runId of ["run-1", "run-2", "run-3"]) { await allocateWorkspace({ refreshGates: createRefreshGates(), - runId, workspaceRoot, sshUrl: seed, @@ -144,7 +140,6 @@ describe("allocateWorkspace (D28)", () => { const dir = await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-a", workspaceRoot, sshUrl: seed, @@ -173,7 +168,6 @@ describe("allocateWorkspace (D28)", () => { const dir = await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-a", workspaceRoot, sshUrl: remote, @@ -219,7 +213,6 @@ describe("allocateWorkspace (D28)", () => { await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-1", workspaceRoot, sshUrl: seed, @@ -229,7 +222,6 @@ describe("allocateWorkspace (D28)", () => { await Bun.sleep(5); await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-2", workspaceRoot, sshUrl: seed, @@ -240,7 +232,6 @@ describe("allocateWorkspace (D28)", () => { await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-3", workspaceRoot, sshUrl: seed, @@ -263,7 +254,6 @@ describe("allocateWorkspace (D28)", () => { await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-1", workspaceRoot, sshUrl: seed, @@ -273,7 +263,6 @@ describe("allocateWorkspace (D28)", () => { await Bun.sleep(5); await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-2", workspaceRoot, sshUrl: seed, @@ -284,7 +273,6 @@ describe("allocateWorkspace (D28)", () => { await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-3", workspaceRoot, sshUrl: seed, @@ -308,7 +296,6 @@ describe("scratch workspaces (issue #13)", () => { const dir = await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-scratch", workspaceRoot, sshUrl: seed, @@ -334,7 +321,6 @@ describe("scratch workspaces (issue #13)", () => { for (const runId of ["scratch-1", "scratch-2"]) { await allocateWorkspace({ refreshGates: createRefreshGates(), - runId, workspaceRoot, sshUrl: seed, @@ -345,7 +331,6 @@ describe("scratch workspaces (issue #13)", () => { } await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "clone-1", workspaceRoot, sshUrl: seed, @@ -401,7 +386,6 @@ describe("host exec injection (issue #13)", () => { const exit0: ExecResult = { command: "fake", exitCode: 0, stdout: "", stderr: "" }; const dir = await allocateWorkspace({ refreshGates: createRefreshGates(), - runId: "run-fake", workspaceRoot: workspaceRoot2, sshUrl: seed, diff --git a/src/server/dedupe.test.ts b/src/server/dedupe.test.ts index 845d8ac..e198066 100644 --- a/src/server/dedupe.test.ts +++ b/src/server/dedupe.test.ts @@ -4,6 +4,10 @@ * releases it when the run reaches any terminal state. Starting a second run * with a key a non-terminal run already holds throws a `DedupeKeyError` * naming the key and the holding run. + * + * #38: tests create their own DaemonServices instead of using module-level + * singletons. A holder is kept in its reserved-but-not-started window by + * pre-seeding the daemon's refresh gate for its workspace mirror. */ import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; @@ -15,13 +19,24 @@ import { createSlowFakeAdapter } from "../replay/adapter"; import { makeAgentRuntime } from "../runtime/agent-runtime"; import { defineWorkflow, Schema } from "../workflow"; import { createDedupeRegistry, DedupeKeyError } from "../lib/dedupe"; -import { activeRunIds, getActiveHandle, isActive, startTrackedRun } from "./runs"; +import { createRunRegistry, startTrackedRun, type DaemonServices } from "./runs"; +import { createPubSub } from "./pubsub"; +import { createRefreshGates } from "../lib/workspace"; import { serve } from "./http"; import { defineConfig } from "../config"; /** Fire-and-forget children keep filling the registry; drain before closing the store. */ -async function drain(timeoutMs = 10_000): Promise { - await waitFor(() => activeRunIds().length === 0, timeoutMs); +async function drain(services: DaemonServices, timeoutMs = 10_000): Promise { + await waitFor(() => services.registry.activeRunIds().length === 0, timeoutMs); +} + +function createTestServices(): DaemonServices { + return { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; } const SLOW_ADAPTER = createSlowFakeAdapter( @@ -44,7 +59,7 @@ const echoWorkflow = defineWorkflow("echo-wf", { }); interface Gate { - readonly wait: () => Promise; + readonly promise: Promise; readonly release: () => void; } @@ -53,7 +68,7 @@ function makeGate(): Gate { const promise = new Promise((resolve) => { release = resolve; }); - return { wait: () => promise, release }; + return { promise, release }; } async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { @@ -69,23 +84,48 @@ interface StartOpts { readonly runId?: string; readonly dir: string; readonly dedupeKey?: string; - readonly beforeStart?: () => Promise; - /** Shared holder registry when one test needs two starts to agree. */ - readonly registry?: ReturnType; + /** The daemon services both starts share, so they agree on holders. */ + readonly services: DaemonServices; + /** + * Holds the run in its reserved-but-not-started window until released: the + * run allocates a clone workspace under `dir` whose mirror refresh is + * queued behind the gate. + */ + readonly gate?: Gate; } -function start(db: ReturnType, opts: StartOpts): Promise { - return startTrackedRun(runtime, db, echoWorkflow, { - dir: opts.dir, +async function start(db: ReturnType, opts: StartOpts): Promise { + const location = + opts.gate === undefined + ? { dir: opts.dir } + : { workspace: await gatedWorkspace(opts.dir, opts.services, opts.gate) }; + return startTrackedRun(runtime, db, opts.services, echoWorkflow, { + ...location, input: {}, - dedupeRegistry: opts.registry ?? createDedupeRegistry(), maxConcurrentRuns: 4, ...(opts.runId !== undefined ? { runId: opts.runId } : {}), ...(opts.dedupeKey !== undefined ? { dedupeKey: opts.dedupeKey } : {}), - ...(opts.beforeStart !== undefined ? { beforeStart: opts.beforeStart } : {}), }); } +async function gatedWorkspace( + root: string, + services: DaemonServices, + gate: Gate, +): Promise { + const seed = join(root, "seed"); + await Bun.$`git init -b main -q ${seed}`.quiet(); + await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); + const workspaceRoot = join(root, "workspaces"); + services.refreshGates.set(join(workspaceRoot, ".mirror.git"), gate.promise); + return { + workspaceRoot, + sshUrl: seed, + identity: { name: "Test Bot", email: "test@factory.local" }, + retainedWorkspaces: 10, + }; +} + /** The `DedupeKeyError` a rejected start produced — else a placeholder string. */ async function errorOf(promise: Promise): Promise { return promise.then( @@ -130,6 +170,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { test("a ctx.dispatch collision throws into the parent and is recorded as DispatchCollision", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-dispatch-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); mkdirSync(join(root, "workspaces"), { recursive: true }); const parent = defineWorkflow("parent-wf", { @@ -147,14 +188,14 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, parent, { + const parentRunId = await startTrackedRun(runtime, db, services, parent, { input: { wait: 3 }, workspace: workspaces(join(root)), maxConcurrentRuns: 10, dispatchEnv: withWorkspaces(join(root)), }); - await waitFor(() => !isActive(parentRunId)); + await waitFor(() => !services.registry.isActive(parentRunId)); const events = getRunEvents(db, parentRunId); const collided = events.find((event) => event.payload._tag === "DispatchCollision"); @@ -170,7 +211,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { failed !== undefined && failed.payload._tag === "RunFailed" ? failed.payload.message : "", ).toMatch(/dedupe key held: "item:7"/); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -178,15 +219,16 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { test("the key is released when the holding child run finishes, so a later dispatch succeeds", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-dispatch-release-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); mkdirSync(join(root, "workspaces"), { recursive: true }); - let release: () => void = () => undefined; const gate = makeGate(); + const parentGate = makeGate(); const heldWorkflow = defineWorkflow("held-wf", { input: Schema.Struct({}), workspace: { kind: "scratch" }, run: async () => { - await gate.wait; + await gate.promise; return {}; }, }); @@ -197,16 +239,14 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { run: async (ctx) => { const ids = []; ids.push(await ctx.dispatch(heldWorkflow, {}, { dedupeKey: "item:8" })); - await new Promise((r) => { - release = r; - }); + await parentGate.promise; // The first child is terminal by now; the key must be free again. ids.push(await ctx.dispatch(heldWorkflow, {}, { dedupeKey: "item:8" })); return { ids }; }, }); - const parentRunId = await startTrackedRun(runtime, db, parent, { + const parentRunId = await startTrackedRun(runtime, db, services, parent, { input: {}, workspace: workspaces(join(root)), maxConcurrentRuns: 10, @@ -218,16 +258,26 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { ); await Bun.sleep(150); // The first child dispatched, still holding while the gate is shut — one - // more dispatch inside the parent would collide. Then let the child end. + // more dispatch inside the parent would collide. Then let the child end, + // and only once it is terminal let the parent dispatch again. + const firstChild = getRunEvents(db, parentRunId).find( + (event) => event.payload._tag === "RunDispatched", + ); + const firstChildRunId = + firstChild !== undefined && firstChild.payload._tag === "RunDispatched" + ? firstChild.payload.childRunId + : ""; + expect(services.dedupeRegistry.holderOf("item:8")).toBe(firstChildRunId); gate.release(); - if (typeof release === "function") release(); + await waitFor(() => !services.registry.isActive(firstChildRunId)); + parentGate.release(); await waitFor( () => getRunEvents(db, parentRunId).filter((event) => event.payload._tag === "RunDispatched") .length >= 2, 10_000, ); - await waitFor(() => !isActive(parentRunId)); + await waitFor(() => !services.registry.isActive(parentRunId)); const dispatchedCount = getRunEvents(db, parentRunId).filter( (event) => event.payload._tag === "RunDispatched", @@ -237,7 +287,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { getRunEvents(db, parentRunId).some((event) => event.payload._tag === "DispatchCollision"), ).toBe(false); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -248,19 +298,19 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-hold-")); const db = openStore(join(root, "factory.db")); const gate = makeGate(); - const registry = createDedupeRegistry(); + const services = createTestServices(); const first = start(db, { runId: "run-holder", dir: join(root, "dir-1"), dedupeKey: "item:41", - beforeStart: gate.wait, - registry, + gate, + services, }); - await waitFor(() => isActive("run-holder")); + await waitFor(() => services.registry.isActive("run-holder")); const err = (await errorOf( - start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:41", registry }), + start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:41", services }), )) as DedupeKeyError | string; expect(err).toBeInstanceOf(DedupeKeyError); if (err instanceof DedupeKeyError) { @@ -271,9 +321,9 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { gate.release(); await first; - await waitFor(() => !isActive("run-holder")); + await waitFor(() => !services.registry.isActive("run-holder")); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -282,24 +332,24 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-log-")); const db = openStore(join(root, "factory.db")); const gate = makeGate(); - const registry = createDedupeRegistry(); + const services = createTestServices(); const first = start(db, { runId: "run-holder", dir: join(root, "dir-1"), dedupeKey: "item:42", - beforeStart: gate.wait, - registry, + gate, + services, }); - await waitFor(() => isActive("run-holder")); + await waitFor(() => services.registry.isActive("run-holder")); await errorOf( - start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:42", registry }), + start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:42", services }), ); gate.release(); await first; - await waitFor(() => !isActive("run-holder")); + await waitFor(() => !services.registry.isActive("run-holder")); const holderStarted = getRunEvents(db, "run-holder").find( (e) => e.payload._tag === "RunStarted", @@ -314,7 +364,7 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { // A dropped start never started: the collision run's log has no rows. expect(getRunEvents(db, "run-collide")).toHaveLength(0); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -322,16 +372,25 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("the key is released when the holding run reaches its terminal state", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-finish-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); - const firstRunId = await start(db, { dir: join(root, "dir-1"), dedupeKey: "item:43" }); + const firstRunId = await start(db, { + dir: join(root, "dir-1"), + dedupeKey: "item:43", + services, + }); expect(firstRunId).toBeTypeOf("string"); - await waitFor(() => !isActive(firstRunId)); + await waitFor(() => !services.registry.isActive(firstRunId)); - const retryRunId = await start(db, { dir: join(root, "dir-2"), dedupeKey: "item:43" }); - await waitFor(() => !isActive(retryRunId)); + const retryRunId = await start(db, { + dir: join(root, "dir-2"), + dedupeKey: "item:43", + services, + }); + await waitFor(() => !services.registry.isActive(retryRunId)); expect(getRunEvents(db, retryRunId).some((e) => e.payload._tag === "RunStarted")).toBe(true); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -339,21 +398,25 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a failed start releases its claim so a corrected retry can start", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-fail-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); // allocation of an explicitly undefined dir fails after the claim await expect( - startTrackedRun(runtime, db, echoWorkflow, { + startTrackedRun(runtime, db, services, echoWorkflow, { input: {}, dedupeKey: "item:44", - dedupeRegistry: createDedupeRegistry(), }), ).rejects.toThrow(/needs `dir` or `workspace`/); - const retryRunId = await start(db, { dir: join(root, "dir-1"), dedupeKey: "item:44" }); - await waitFor(() => !isActive(retryRunId)); + const retryRunId = await start(db, { + dir: join(root, "dir-1"), + dedupeKey: "item:44", + services, + }); + await waitFor(() => !services.registry.isActive(retryRunId)); expect(getRunEvents(db, retryRunId).some((e) => e.payload._tag === "RunStarted")).toBe(true); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -361,31 +424,29 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a cancelled run releases its key (RunCancelled is a terminal state)", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-cancel-")); const db = openStore(join(root, "factory.db")); - const registry = createDedupeRegistry(); + const services = createTestServices(); - const holderRunId = await startTrackedRun(runtime, db, echoWorkflow, { + const holderRunId = await startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-cancelled", dir: join(root, "dir-1"), input: {}, dedupeKey: "item:45", - dedupeRegistry: registry, }); - await waitFor(() => !isActive(holderRunId) || true); - const handle = getActiveHandle("run-cancelled"); + await waitFor(() => !services.registry.isActive(holderRunId) || true); + const handle = services.registry.getActiveHandle("run-cancelled"); expect(handle).toBeDefined(); if (handle !== undefined) await handle.cancel(); - await waitFor(() => !isActive(holderRunId)); + await waitFor(() => !services.registry.isActive(holderRunId)); const started = getRunEvents(db, holderRunId).some((e) => e.payload._tag === "RunStarted"); - const retryRunId = await startTrackedRun(runtime, db, echoWorkflow, { + const retryRunId = await startTrackedRun(runtime, db, services, echoWorkflow, { dir: join(root, "dir-2"), input: {}, dedupeKey: "item:45", - dedupeRegistry: registry, }); expect(started).toBe(true); expect(retryRunId).toBeTypeOf("string"); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); @@ -394,14 +455,17 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("runs started without a key behave exactly as before", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-none-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); - const firstRunId = await start(db, { dir: join(root, "dir-1") }); - const secondRunId = await start(db, { dir: join(root, "dir-2") }); + const firstRunId = await start(db, { dir: join(root, "dir-1"), services }); + const secondRunId = await start(db, { dir: join(root, "dir-2"), services }); expect(firstRunId).toBeTypeOf("string"); expect(secondRunId).toBeTypeOf("string"); - await waitFor(() => !isActive(firstRunId) && !isActive(secondRunId)); + await waitFor( + () => !services.registry.isActive(firstRunId) && !services.registry.isActive(secondRunId), + ); - await drain(); + await drain(services); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -425,6 +489,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { test("a key held by a running run surfaces as a 409 conflict naming the key and the holder", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-http-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); const slowStartWorkflow = defineWorkflow("dedupe-http-slow", { input: Schema.Struct({}), @@ -439,6 +504,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const server = serve({ db, runtime: makeAgentRuntime(SLOW_ADAPTER), + services, port: 0, config: defineConfig({ repo: { @@ -494,7 +560,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const { runId: retryRunId } = (await retryRes.json()) as { runId: string }; await waitForTerminalStatus(db, retryRunId); } finally { - await drain(); + await drain(services); await server.stop(true); db.close(); rmSync(root, { recursive: true, force: true }); @@ -504,6 +570,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { test("a non-string dedupeKey is a 400, and omitting one behaves as today", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-http-shape-")); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); const plainWorkflow = defineWorkflow("dedupe-http-plain", { input: Schema.Struct({}), workspace: { kind: "scratch" }, @@ -513,6 +580,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const server = serve({ db, runtime: makeAgentRuntime(SLOW_ADAPTER), + services, port: 0, config: defineConfig({ repo: { @@ -543,7 +611,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const { runId } = (await plain.json()) as { runId: string }; await waitForTerminalStatus(db, runId); } finally { - await drain(); + await drain(services); await server.stop(true); db.close(); rmSync(root, { recursive: true, force: true }); diff --git a/src/server/http.test.ts b/src/server/http.test.ts index 30e7676..46b1d29 100644 --- a/src/server/http.test.ts +++ b/src/server/http.test.ts @@ -12,6 +12,10 @@ import { defineConfig, loadFactoryConfig } from "../config"; import registryWorkflow from "../../test/fixtures/registry-workflow"; import registryScratchWorkflow from "../../test/fixtures/registry-scratch-workflow"; import { serve } from "./http"; +import { createRunRegistry, type DaemonServices } from "./runs"; +import { createPubSub } from "./pubsub"; +import { createDedupeRegistry } from "../lib/dedupe"; +import { createRefreshGates } from "../lib/workspace"; const ECHO_WORKFLOW = `${import.meta.dir}/../../test/fixtures/echo-workflow.ts`; const QUIET_GAP_WORKFLOW = `${import.meta.dir}/../../test/fixtures/quiet-gap-workflow.ts`; @@ -85,13 +89,29 @@ async function readSseUntilTerminal( return frames; } +/** Issue #38: each server under test gets its own per-daemon services. */ +function createTestServices(): DaemonServices { + return { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; +} + describe("GET /api/workflows (D30)", () => { test("lists the config's workflows with id and JSON Schema input", async () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); const config = await loadFactoryConfig(FIXTURE_CONFIG); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -116,7 +136,12 @@ describe("GET /api/workflows (D30)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-empty-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -136,7 +161,12 @@ describe("GET /api/workflows (D30)", () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-empty-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -156,7 +186,12 @@ describe("phase 4 SPA serving", () => { const dir = mkdtempSync(join(tmpdir(), "factory-spa-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([{ type: "TEXT_MESSAGE_START" }], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -239,7 +274,13 @@ describe("SSE keepalive", () => { const dir = mkdtempSync(join(tmpdir(), "factory-sse-keepalive-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, sseKeepaliveMs: 25 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + sseKeepaliveMs: 25, + }); const base = `http://localhost:${server.port}`; try { @@ -275,7 +316,13 @@ describe("SSE keepalive", () => { const dir = mkdtempSync(join(tmpdir(), "factory-sse-keepalive-stop-test-")); const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, sseKeepaliveMs: 10 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + sseKeepaliveMs: 10, + }); const base = `http://localhost:${server.port}`; try { @@ -315,7 +362,12 @@ describe("phase 3 HTTP API + SSE", () => { ], 1, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -364,7 +416,12 @@ describe("phase 3 HTTP API + SSE", () => { ], 500, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -400,7 +457,12 @@ describe("phase 3 HTTP API + SSE", () => { ], 60, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -439,7 +501,12 @@ describe("phase 3 HTTP API + SSE", () => { ], 400, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -477,7 +544,12 @@ describe("phase 3 HTTP API + SSE", () => { ], 1, ); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -523,7 +595,13 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { 1, ); const config = await loadFactoryConfig(FIXTURE_CONFIG); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -549,7 +627,13 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { const db = openStore(join(dir, "factory.db")); const adapter = createSlowFakeAdapter([], 1); const config = await loadFactoryConfig(FIXTURE_CONFIG); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -595,7 +679,13 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { maxConcurrentRuns: 1, retainedWorkspaces: 10, }); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -658,7 +748,13 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { maxConcurrentRuns: 3, retainedWorkspaces: 10, }); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -725,7 +821,13 @@ describe("a scratch workflow through POST /api/runs (issue #13)", () => { maxConcurrentRuns: 3, retainedWorkspaces: 10, }); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0, config }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + config, + }); const base = `http://localhost:${server.port}`; try { @@ -799,6 +901,7 @@ describe("ctx.dispatch through POST /api/runs (issue #14)", () => { const server = serve({ db, runtime: makeAgentRuntime(adapter), + services: createTestServices(), port: 0, config: defineConfig({ repo: { @@ -897,6 +1000,7 @@ describe("GET /api/schedules (issue #17)", () => { const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), + services: createTestServices(), port: 0, config: schedulesConfig(root, workflow), }); @@ -966,6 +1070,7 @@ describe("GET /api/schedules (issue #17)", () => { const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), + services: createTestServices(), port: 0, config, }); @@ -989,7 +1094,12 @@ describe("GET /api/schedules (issue #17)", () => { test("serves an empty list on a legacy no-config server", async () => { const root = mkdtempSync(join(tmpdir(), "factory-schedules-empty-test-")); const db = openStore(join(root, "factory.db")); - const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { @@ -1034,6 +1144,7 @@ describe("GET /api/schedules (issue #17)", () => { const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), + services: createTestServices(), port: 0, config, }); @@ -1135,6 +1246,7 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), + services: createTestServices(), port: 0, config, }); @@ -1189,6 +1301,7 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), + services: createTestServices(), port: 0, config, }); @@ -1232,6 +1345,7 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), + services: createTestServices(), port: 0, config, }); @@ -1297,6 +1411,7 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { const server = serve({ db, runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), + services: createTestServices(), port: 0, config, }); @@ -1345,7 +1460,12 @@ describe("adapter.prepareWorkspace through the legacy POST /api/runs (#37)", () const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); const adapter = trackingAdapter(); - const server = serve({ db, runtime: makeAgentRuntime(adapter), port: 0 }); + const server = serve({ + db, + runtime: makeAgentRuntime(adapter), + services: createTestServices(), + port: 0, + }); const base = `http://localhost:${server.port}`; try { diff --git a/src/server/nested-runs.test.ts b/src/server/nested-runs.test.ts index 7c9e8ac..3b74d93 100644 --- a/src/server/nested-runs.test.ts +++ b/src/server/nested-runs.test.ts @@ -8,6 +8,8 @@ * carries the message). A validated child is then started fire-and-forget * with the parent's run id recorded on its `RunStarted.parentId`, and the * parent never exposes an awaitable child (D-epic 19). + * + * #38: tests create their own DaemonServices. */ import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; @@ -18,7 +20,10 @@ import { describe, expect, test } from "bun:test"; import { Schema } from "effect"; import { getRunEvents, listRuns, openStore } from "../persistence/store"; import { createSlowFakeAdapter } from "../replay/adapter"; -import { activeRunIds, isActive, startTrackedRun } from "./runs"; +import { createRunRegistry, startTrackedRun, type DaemonServices } from "./runs"; +import { createPubSub } from "./pubsub"; +import { createDedupeRegistry } from "../lib/dedupe"; +import { createRefreshGates } from "../lib/workspace"; import { defineWorkflow } from "../workflow"; import { makeAgentRuntime } from "../runtime/agent-runtime"; @@ -64,6 +69,15 @@ const manyChildren = defineWorkflow("many-children-wf", { }, }); +function createTestServices(): DaemonServices { + return { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; +} + async function waitFor(predicate: () => boolean, timeoutMs = 10_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -82,6 +96,7 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-happy-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); const parent = defineWorkflow("parent-wf", { input: Schema.Struct({}), @@ -92,7 +107,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, parent, { + const parentRunId = await startTrackedRun(runtime, db, services, parent, { workspace: { workspaceRoot: join(root, "workspaces"), sshUrl: join(root, "seed-not-used"), @@ -110,7 +125,10 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => !isActive(parentRunId) && activeRunIds().length === 0); + await waitFor( + () => + !services.registry.isActive(parentRunId) && services.registry.activeRunIds().length === 0, + ); const parentEvents = getRunEvents(db, parentRunId); const dispatched = parentEvents.find((e) => e.payload._tag === "RunDispatched"); @@ -145,8 +163,9 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-depth-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); - await startTrackedRun(runtime, db, selfDispatching, { + await startTrackedRun(runtime, db, services, selfDispatching, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -166,7 +185,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => activeRunIds().length === 0); + await waitFor(() => services.registry.activeRunIds().length === 0); // The chain built to the cap, no farther: root + 5 children max. const selfRuns = listRuns(db).filter((run) => run.workflowId === "self-dispatch-wf"); @@ -196,8 +215,9 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-count-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); - const parentRunId = await startTrackedRun(runtime, db, manyChildren, { + const parentRunId = await startTrackedRun(runtime, db, services, manyChildren, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -218,7 +238,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => activeRunIds().length === 0); + await waitFor(() => services.registry.activeRunIds().length === 0); expect(tagsOf(db, parentRunId).filter((tag) => tag === "RunDispatched").length).toBe(5); const parentEvents = getRunEvents(db, parentRunId); @@ -237,6 +257,7 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-wip-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); + const services = createTestServices(); const parent = defineWorkflow("parent-wip-wf", { input: Schema.Struct({}), @@ -247,7 +268,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, parent, { + const parentRunId = await startTrackedRun(runtime, db, services, parent, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -267,7 +288,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => !isActive(parentRunId)); + await waitFor(() => !services.registry.isActive(parentRunId)); const events = getRunEvents(db, parentRunId); expect(events.map((event) => event.payload._tag)).not.toContain("RunDispatched"); diff --git a/src/server/runs.test.ts b/src/server/runs.test.ts index 7e050bc..4152a3f 100644 --- a/src/server/runs.test.ts +++ b/src/server/runs.test.ts @@ -7,6 +7,11 @@ * - a `cancel` arriving while a run is only a reserved slot is deferred into * run start (L1): the run starts, is cancelled immediately, and ends as a * clean RunCancelled with no orphaned slot. + * + * #38: tests create their own DaemonServices instead of using module-level + * singletons. The reserved-but-not-started window is held open by pre-seeding + * the daemon's refresh gate for the workspace mirror, so allocation blocks + * until the test releases it. */ import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; @@ -23,12 +28,14 @@ import { defineWorkflow, Schema } from "../workflow"; import { ConcurrencyLimitError, DispatchCapError, - activeRunIds, - cancelRegisteredRun, - getActiveHandle, - isActive, + createRunRegistry, startTrackedRun, + type DaemonServices, + type WorkspaceSpec, } from "./runs"; +import { createPubSub } from "./pubsub"; +import { createDedupeRegistry } from "../lib/dedupe"; +import { createRefreshGates } from "../lib/workspace"; const SLOW_ADAPTER = createSlowFakeAdapter( [ @@ -54,8 +61,17 @@ function tmpRoot(): { root: string; finish: () => void } { return { root, finish: () => rmSync(root, { recursive: true, force: true }) }; } +function createTestServices(): DaemonServices { + return { + registry: createRunRegistry(), + pubsub: createPubSub(), + dedupeRegistry: createDedupeRegistry(), + refreshGates: createRefreshGates(), + }; +} + interface Gate { - readonly wait: () => Promise; + readonly promise: Promise; readonly release: () => void; } @@ -64,7 +80,30 @@ function makeGate(): Gate { const promise = new Promise((resolve) => { release = resolve; }); - return { wait: () => promise, release }; + return { promise, release }; +} + +/** + * A clone workspace whose allocation blocks until `gate` is released: the + * gate is pre-seeded as the mirror's in-flight refresh, so `allocateWorkspace` + * queues behind it — the run holds a reserved slot and nothing more. + */ +async function gatedWorkspace( + root: string, + services: DaemonServices, + gate: Gate, +): Promise { + const seed = join(root, "seed"); + await Bun.$`git init -b main -q ${seed}`.quiet(); + await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); + const workspaceRoot = join(root, "workspaces"); + services.refreshGates.set(join(workspaceRoot, ".mirror.git"), gate.promise); + return { + workspaceRoot, + sshUrl: seed, + identity: { name: "T", email: "t@t.test" }, + retainedWorkspaces: 10, + }; } async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { @@ -102,21 +141,21 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" test("a second start while the first is still reserving is refused atomically, and the slot frees after", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); const gate = makeGate(); - const first = startTrackedRun(runtime, db, echoWorkflow, { + const first = startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-first", - dir: join(root, "first-dir"), + workspace: await gatedWorkspace(root, services, gate), input: {}, maxConcurrentRuns: 1, - beforeStart: gate.wait, }); - await waitFor(() => activeRunIds().length === 1); - expect(activeRunIds()).toEqual(["run-first"]); - expect(getActiveHandle("run-first")).toBeUndefined(); + await waitFor(() => services.registry.activeRunIds().length === 1); + expect(services.registry.activeRunIds()).toEqual(["run-first"]); + expect(services.registry.getActiveHandle("run-first")).toBeUndefined(); - const second = startTrackedRun(runtime, db, echoWorkflow, { + const second = startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-second", dir: join(root, "second-dir"), input: {}, @@ -133,8 +172,40 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" gate.release(); const firstRunId = await first; expect(firstRunId).toBe("run-first"); - await waitFor(() => !isActive(firstRunId)); - expect(activeRunIds()).toEqual([]); + await waitFor(() => !services.registry.isActive(firstRunId)); + expect(services.registry.activeRunIds()).toEqual([]); + + db.close(); + finish(); + }); + + test("a second start issued synchronously, before the first is awaited, is refused", async () => { + const { root, finish } = tmpRoot(); + const db = openStore(join(root, "factory.db")); + const services = createTestServices(); + const gate = makeGate(); + const workspace = await gatedWorkspace(root, services, gate); + + // No await between the two calls: the first must already hold its slot. + const first = startTrackedRun(runtime, db, services, echoWorkflow, { + runId: "run-sync-first", + workspace, + input: {}, + maxConcurrentRuns: 1, + }); + const second = startTrackedRun(runtime, db, services, echoWorkflow, { + runId: "run-sync-second", + workspace, + input: {}, + maxConcurrentRuns: 1, + }); + + expect(await second.catch((err: unknown) => err)).toBeInstanceOf(ConcurrencyLimitError); + expect(services.registry.activeRunIds()).toEqual(["run-sync-first"]); + + gate.release(); + const firstRunId = await first; + await waitFor(() => !services.registry.isActive(firstRunId)); db.close(); finish(); @@ -143,9 +214,10 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" test("a failed allocation releases the reserved slot", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); await expect( - startTrackedRun(runtime, db, echoWorkflow, { + startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-doomed", input: {}, maxConcurrentRuns: 1, @@ -158,15 +230,15 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" }), ).rejects.toThrow(/workspace/); - expect(activeRunIds()).toEqual([]); + expect(services.registry.activeRunIds()).toEqual([]); - const runId = await startTrackedRun(runtime, db, echoWorkflow, { + const runId = await startTrackedRun(runtime, db, services, echoWorkflow, { dir: join(root, "dir"), input: {}, maxConcurrentRuns: 1, }); - expect(activeRunIds()).toEqual([runId]); - await waitFor(() => !isActive(runId)); + expect(services.registry.activeRunIds()).toEqual([runId]); + await waitFor(() => !services.registry.isActive(runId)); db.close(); finish(); @@ -177,35 +249,34 @@ describe("cancel of a reserved-but-not-started run (L1)", () => { test("cancelling during the allocation window defers into run start; the run ends cancelled with no leak", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - mkdirSync(join(root, "dir"), { recursive: true }); + const services = createTestServices(); const gate = makeGate(); - const starting = startTrackedRun(runtime, db, sleepWorkflow, { + const starting = startTrackedRun(runtime, db, services, sleepWorkflow, { runId: "run-gated", - dir: join(root, "dir"), + workspace: await gatedWorkspace(root, services, gate), input: {}, maxConcurrentRuns: 1, - beforeStart: gate.wait, }); - await waitFor(() => activeRunIds().length === 1); - expect(getActiveHandle("run-gated")).toBeUndefined(); + await waitFor(() => services.registry.activeRunIds().length === 1); + expect(services.registry.getActiveHandle("run-gated")).toBeUndefined(); - const cancelled = cancelRegisteredRun("run-gated"); + const cancelled = services.registry.cancelRegisteredRun("run-gated"); expect(cancelled).toEqual({ kind: "reserved" }); gate.release(); const runId = await starting; expect(runId).toBe("run-gated"); - await waitFor(() => activeRunIds().length === 0); + await waitFor(() => services.registry.activeRunIds().length === 0); const events = getRunEvents(db, runId).map((e) => e.payload._tag); expect(events).toContain("RunStarted"); expect(events).toContain("RunCancelled"); - expect(cancelRegisteredRun(runId)).toBeUndefined(); + expect(services.registry.cancelRegisteredRun(runId)).toBeUndefined(); await Bun.sleep(50); - expect(activeRunIds()).toEqual([]); + expect(services.registry.activeRunIds()).toEqual([]); db.close(); finish(); @@ -247,14 +318,15 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { const { root, finish } = tmpRoot(); _tmp = root; const db = openStore(join(root, "factory.db")); + const services = createTestServices(); mkdirSync(join(root, "seed"), { recursive: true }); - const runId = await startTrackedRun(runtime, db, failingScratchWorkflow, { + const runId = await startTrackedRun(runtime, db, services, failingScratchWorkflow, { runId: "run-scratch-empty", workspace: { ...workspaceSpec(), sshUrl: join(root, "no-such-remote") }, input: {}, }); - await waitFor(() => !isActive(runId)); + await waitFor(() => !services.registry.isActive(runId)); const dir = join(root, "workspaces", "run-scratch-empty"); expect(existsSync(dir)).toBe(true); @@ -268,23 +340,24 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { const { root, finish } = tmpRoot(); _tmp = root; const db = openStore(join(root, "factory.db")); + const services = createTestServices(); mkdirSync(join(root, "seed"), { recursive: true }); - const okId = await startTrackedRun(runtime, db, scratchWorkflow, { + const okId = await startTrackedRun(runtime, db, services, scratchWorkflow, { runId: "run-scratch-ok", workspace: workspaceSpec(), input: {}, }); - await waitFor(() => !isActive(okId)); + await waitFor(() => !services.registry.isActive(okId)); await new Promise((resolve) => setTimeout(resolve, 50)); // reap lands async expect(existsSync(join(root, "workspaces", "run-scratch-ok"))).toBe(false); - const badId = await startTrackedRun(runtime, db, failingScratchWorkflow, { + const badId = await startTrackedRun(runtime, db, services, failingScratchWorkflow, { runId: "run-scratch-bad", workspace: workspaceSpec(), input: {}, }); - await waitFor(() => !isActive(badId)); + await waitFor(() => !services.registry.isActive(badId)); await new Promise((resolve) => setTimeout(resolve, 50)); expect(existsSync(join(root, "workspaces", "run-scratch-bad"))).toBe(true); finish(); @@ -294,6 +367,7 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { const { root, finish } = tmpRoot(); _tmp = root; const db = openStore(join(root, "factory.db")); + const services = createTestServices(); const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); @@ -314,12 +388,12 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { // retention = 1: with the scratch dir correctly excluded, the only clone // survives: the leftover must not count toward retention. - await startTrackedRun(runtime, db, echoWorkflow, { + await startTrackedRun(runtime, db, services, echoWorkflow, { runId: "run-clone", workspace: { ...workspaceSpec(), sshUrl: seed, retainedWorkspaces: 1 }, input: {}, }); - await waitFor(() => !isActive("run-clone")); + await waitFor(() => !services.registry.isActive("run-clone")); expect(existsSync(join(root, "workspaces", "run-clone"))).toBe(true); expect(existsSync(join(root, "workspaces", "run-scratch-kept"))).toBe(true); @@ -354,11 +428,12 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { test("an allocated clone workspace is prepared on the allocated dir", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); const adapter = trackingAdapter(); - const runId = await startTrackedRun(makeAgentRuntime(adapter), db, echoWorkflow, { + const runId = await startTrackedRun(makeAgentRuntime(adapter), db, services, echoWorkflow, { runId: "run-prep-clone", workspace: { workspaceRoot: join(root, "workspaces"), @@ -368,7 +443,7 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { }, input: {}, }); - await waitFor(() => !isActive(runId)); + await waitFor(() => !services.registry.isActive(runId)); expect(adapter.prepared).toEqual([join(root, "workspaces", "run-prep-clone")]); finish(); @@ -377,9 +452,10 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { test("a scratch workspace is not prepared", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); + const services = createTestServices(); const adapter = trackingAdapter(); - const runId = await startTrackedRun(makeAgentRuntime(adapter), db, scratchWorkflow, { + const runId = await startTrackedRun(makeAgentRuntime(adapter), db, services, scratchWorkflow, { runId: "run-prep-scratch", workspace: { workspaceRoot: join(root, "workspaces"), @@ -389,7 +465,7 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { }, input: {}, }); - await waitFor(() => !isActive(runId)); + await waitFor(() => !services.registry.isActive(runId)); expect(adapter.prepared).toEqual([]); finish(); diff --git a/src/server/scheduler.test.ts b/src/server/scheduler.test.ts index 5b4882d..f9c2e23 100644 --- a/src/server/scheduler.test.ts +++ b/src/server/scheduler.test.ts @@ -76,13 +76,16 @@ interface Fixture { deps: () => SchedulerDeps; readonly fired: Array; readonly registry: ReturnType; + /** Per-schedule injected fire failures. */ readonly fireError: Map; + /** The fixture's injectable clock. */ readonly setTime: (t: number) => Promise; } async function fixture(schedules: ReadonlyArray): Promise { const registry = createDedupeRegistry(); const fired: Array = []; + // Per-schedule injected fire failures. const fireError: Map = new Map(); const { clock, setTime } = await createTestClock(0); @@ -114,6 +117,7 @@ const DAY01_0400 = Date.parse("2026-01-01T04:00:00Z"); describe("scheduler tick (issue #16)", () => { test("fires when its cron window fell between lastTick and now", async () => { const fx = await fixture([schedule()]); + // state created before the window (03:05), tick after it (04:00) await fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); @@ -140,6 +144,7 @@ describe("scheduler tick (issue #16)", () => { const fx = await fixture([schedule()]); await fx.setTime(DAY01_0259); const state = createSchedulerState(fx.deps()); + // A previous run of this schedule is still going. fx.registry.claim("schedule:nightly", "run-previous"); await fx.setTime(DAY01_0400); @@ -171,12 +176,16 @@ describe("scheduler tick (issue #16)", () => { const results = await tickOnce(fx.deps(), state); expect(results).toEqual([{ scheduleId: "nightly", action: "skipped-concurrency" }]); + // lastTick advanced regardless: the missed window is not refired later. fx.fireError.delete("nightly"); const again = await tickOnce(fx.deps(), state); expect(again).toEqual([{ scheduleId: "nightly", action: "skipped-not-due" }]); }); test("windows missed while the daemon was down are not replayed", async () => { + // Every-minute cron, three windows pass between two ticks after the + // daemon "came up": one fire, not three — and the pre-start windows + // (12:00:00–12:00:30 before the state existed) never fire at all. const fx = await fixture([schedule({ cron: Cron.parseUnsafe("* * * * *", "UTC") })]); const start = Date.parse("2026-01-01T12:00:00Z"); await fx.setTime(start); @@ -200,6 +209,7 @@ describe("scheduler tick (issue #16)", () => { expect(fx.fired).toEqual(["nightly"]); expect(state.runOnStartPending).toEqual(new Set()); + // Pending is consumed whether or not the fire succeeded. const after = await tickOnce(fx.deps(), state); expect(after.every((r) => r.action === "skipped-not-due")).toBe(true); expect(fx.fired).toEqual(["nightly"]); diff --git a/src/server/two-daemons.test.ts b/src/server/two-daemons.test.ts index 0ae03c1..1fe4890 100644 --- a/src/server/two-daemons.test.ts +++ b/src/server/two-daemons.test.ts @@ -2,7 +2,8 @@ * #38 acceptance criterion: two daemons can run in one process without * sharing run state. This test starts two daemons on different ports, starts * a run on each, and verifies that each daemon's registry, pubsub, and dedupe - * state is independent. + * state is independent. Runs are held open on a gate so the cross-daemon + * checks happen while both are still live. */ import { mkdtempSync, rmSync } from "node:fs"; @@ -14,12 +15,23 @@ import { defineWorkflow } from "../workflow"; import { defineConfig } from "../config"; import { createSlowFakeAdapter } from "../replay/adapter"; import { startDaemon } from "./daemon"; -import { Effect, Fiber } from "effect"; +import { isTerminal, type RunEvent } from "../events"; +import type { DaemonHandle } from "./daemon"; + +let releaseRuns: () => void = () => undefined; +let runsGate = Promise.resolve(); + +function closeGate(): void { + runsGate = new Promise((resolve) => { + releaseRuns = resolve; + }); +} const scratchWorkflow = defineWorkflow("two-daemon-test", { input: Schema.Struct({ marker: Schema.String }), workspace: { kind: "scratch" }, run: async (ctx, input) => { + await runsGate; await ctx.exec(["sh", "-c", "sleep 0.1"]); return { marker: input.marker }; }, @@ -36,6 +48,29 @@ async function waitForTerminal(port: number, runId: string, timeoutMs = 10_000): throw new Error(`run ${runId} did not reach terminal state within ${timeoutMs}ms`); } +/** + * Subscribes `runId` on both daemons' pubsubs: the owner's events land in + * `own`, anything the other daemon publishes for it lands in `foreign`. + */ +function watch( + owner: DaemonHandle, + other: DaemonHandle, + runId: string, +): { readonly own: Array; readonly foreign: Array; readonly stop: () => void } { + const own: Array = []; + const foreign: Array = []; + const unsubOwn = owner.services.pubsub.subscribe(runId, (event) => own.push(event)); + const unsubOther = other.services.pubsub.subscribe(runId, (event) => foreign.push(event)); + return { + own, + foreign, + stop: () => { + unsubOwn(); + unsubOther(); + }, + }; +} + describe("two daemons in one process (#38)", () => { test("two daemons have independent run registries, pubsub, and dedupe state", async () => { const root = mkdtempSync(join(tmpdir(), "factory-two-daemons-")); @@ -84,6 +119,7 @@ describe("two daemons in one process (#38)", () => { }); try { + closeGate(); const port1 = daemon1.server.port; const port2 = daemon2.server.port; if (port1 === undefined || port2 === undefined) { @@ -106,6 +142,12 @@ describe("two daemons in one process (#38)", () => { expect(runId1).not.toBe(runId2); + const watch1 = watch(daemon1, daemon2, runId1); + const watch2 = watch(daemon2, daemon1, runId2); + + expect(daemon1.services.registry.activeRunIds()).toEqual([runId1]); + expect(daemon2.services.registry.activeRunIds()).toEqual([runId2]); + const list1 = (await fetch(`http://localhost:${port1}/api/runs`).then((r) => r.json(), )) as Array<{ runId: string }>; @@ -122,13 +164,25 @@ describe("two daemons in one process (#38)", () => { const detail2FromDaemon1 = await fetch(`http://localhost:${port1}/api/runs/${runId2}`); expect(detail2FromDaemon1.status).toBe(404); + releaseRuns(); await waitForTerminal(port1, runId1); await waitForTerminal(port2, runId2); + // Each run's live events reached its own daemon's pubsub, never the other's. + expect(watch1.own.some((event) => isTerminal(event.payload))).toBe(true); + expect(watch2.own.some((event) => isTerminal(event.payload))).toBe(true); + expect(watch1.own.every((event) => event.runId === runId1)).toBe(true); + expect(watch2.own.every((event) => event.runId === runId2)).toBe(true); + expect(watch1.foreign).toEqual([]); + expect(watch2.foreign).toEqual([]); + watch1.stop(); + watch2.stop(); + expect(daemon1.services.registry.activeRunIds()).toEqual([]); expect(daemon2.services.registry.activeRunIds()).toEqual([]); const dedupeKey = "shared-key"; + closeGate(); const res3 = await fetch(`http://localhost:${port1}/api/runs`, { method: "POST", body: JSON.stringify({ workflowId: "two-daemon-test", input: { marker: "d1" }, dedupeKey }), @@ -136,6 +190,11 @@ describe("two daemons in one process (#38)", () => { expect(res3.status).toBe(201); const { runId: runId3 } = (await res3.json()) as { runId: string }; + // Daemon 1 still holds the key (run 3 is gated) when daemon 2 is asked + // for the same key — and daemon 2 knows nothing of it. + expect(daemon1.services.dedupeRegistry.holderOf(dedupeKey)).toBe(runId3); + expect(daemon2.services.dedupeRegistry.holderOf(dedupeKey)).toBeUndefined(); + const res4 = await fetch(`http://localhost:${port2}/api/runs`, { method: "POST", body: JSON.stringify({ workflowId: "two-daemon-test", input: { marker: "d2" }, dedupeKey }), @@ -144,18 +203,16 @@ describe("two daemons in one process (#38)", () => { const { runId: runId4 } = (await res4.json()) as { runId: string }; expect(runId3).not.toBe(runId4); + expect(daemon1.services.dedupeRegistry.holderOf(dedupeKey)).toBe(runId3); + expect(daemon2.services.dedupeRegistry.holderOf(dedupeKey)).toBe(runId4); + releaseRuns(); await waitForTerminal(port1, runId3); await waitForTerminal(port2, runId4); } finally { - if (daemon1.schedulerFiber !== undefined) { - Effect.runFork(Fiber.interrupt(daemon1.schedulerFiber)); - } - if (daemon2.schedulerFiber !== undefined) { - Effect.runFork(Fiber.interrupt(daemon2.schedulerFiber)); - } - daemon1.server.stop(true); - daemon2.server.stop(true); + releaseRuns(); + await daemon1.stop(); + await daemon2.stop(); rmSync(root, { recursive: true, force: true }); } }); From 82f4f5d68e6d3c08dfdeaf191165c966db412bad Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 09:33:19 +0000 Subject: [PATCH 19/26] fix(runs): decide admission before claiming the dedupe key A start refused by the concurrency limit threw after claiming its dedupe key and outside the try that releases it, so the key stayed held by a run that never started. A scheduled fire with `overlap: "skip"` at the limit then reported `skipped-overlap` forever. Admission is now checked first; claim and slot reservation still happen with no await in between. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/server/runs.test.ts | 32 ++++++++++++++++++++++++++++++++ src/server/runs.ts | 14 ++++++++------ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/server/runs.test.ts b/src/server/runs.test.ts index 4152a3f..9a32551 100644 --- a/src/server/runs.test.ts +++ b/src/server/runs.test.ts @@ -211,6 +211,38 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" finish(); }); + test("a start refused at the limit does not keep its dedupe key", async () => { + const { root, finish } = tmpRoot(); + const db = openStore(join(root, "factory.db")); + const services = createTestServices(); + const gate = makeGate(); + + const first = startTrackedRun(runtime, db, services, echoWorkflow, { + runId: "run-holding-slot", + workspace: await gatedWorkspace(root, services, gate), + input: {}, + maxConcurrentRuns: 1, + }); + + // e.g. a scheduled fire with `overlap: "skip"` while the daemon is full. + const refused = startTrackedRun(runtime, db, services, echoWorkflow, { + runId: "run-refused", + dir: join(root, "refused-dir"), + input: {}, + maxConcurrentRuns: 1, + dedupeKey: "schedule:nightly", + }); + expect(await refused.catch((err: unknown) => err)).toBeInstanceOf(ConcurrencyLimitError); + expect(services.dedupeRegistry.holderOf("schedule:nightly")).toBeUndefined(); + + gate.release(); + const firstRunId = await first; + await waitFor(() => !services.registry.isActive(firstRunId)); + + db.close(); + finish(); + }); + test("a failed allocation releases the reserved slot", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); diff --git a/src/server/runs.ts b/src/server/runs.ts index a5d20be..4555417 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -351,6 +351,13 @@ export async function startTrackedRun( throw new Error(`run ${runId} is already active`); } + // D29 admission is decided before the dedupe claim, so a refused start + // throws having claimed nothing — otherwise its key would stay held by a + // run that never started (and a `"skip"` schedule would skip forever). + const limit = existing === undefined ? options.maxConcurrentRuns : undefined; + if (limit !== undefined && !admitRun(limit, registry.size)) { + throw ConcurrencyLimitError.of({ maxConcurrentRuns: limit }); + } // Issue #15: claim the dedupe key synchronously — check-then-claim with no // `await` in between, the same atomicity the registry slot reservation has — // so two near-simultaneous starts on the same key cannot both slip past. A @@ -358,12 +365,7 @@ export async function startTrackedRun( if (options.dedupeKey !== undefined && options.dedupeKeyClaimed !== true) { services.dedupeRegistry.claim(options.dedupeKey, runId); } - if (existing === undefined && options.maxConcurrentRuns !== undefined) { - if (!admitRun(options.maxConcurrentRuns, registry.size)) { - throw ConcurrencyLimitError.of({ maxConcurrentRuns: options.maxConcurrentRuns }); - } - registry.reserve(runId); - } + if (limit !== undefined) registry.reserve(runId); try { // Issue #13: a scratch workspace takes its kind from the workflow and From 2c94d22c3ae3f99694d97c97f537d27d4288114e Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 10:42:28 +0000 Subject: [PATCH 20/26] refactor(daemon): compose per-daemon state as Effect layers The run registry, run-event pubsub, dedupe registry and mirror refresh gates become Context.Service classes with layers that build fresh state per layer build. DaemonLayer composes them with AgentRuntimeLayer into the daemon's ManagedRuntime (server/daemon-runtime.ts), replacing the hand-threaded DaemonServices bag. The plain-async boundaries (HTTP handlers, startTrackedRun, ctx.dispatch, the scheduler's fire) take the daemon runtime and resolve services with serviceOf, a synchronous runSync lookup over all-synchronous layers, so startTrackedRun still reserves its slot and claims its dedupe key with no await between check and set. The scheduler loop is forked on the daemon runtime and resolves its dedupe registry and Clock from context; the unused DaemonOptions.clock seam is gone. startRun accepts any runtime that can run AgentRuntime effects (AgentRunner), since ManagedRuntime is invariant in its services. Tests build a fresh daemon runtime per case (createTestDaemon); the two-daemon test also asserts no service instance is shared. Refs #38 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/dedupe.ts | 26 +++++- src/lib/workspace.ts | 28 ++++++- src/runtime/run.ts | 14 +++- src/server/concurrency.test.ts | 33 +------- src/server/daemon-runtime.ts | 56 +++++++++++++ src/server/daemon.ts | 82 +++++++------------ src/server/dedupe.test.ts | 143 +++++++++++++++------------------ src/server/http.test.ts | 96 +++++++--------------- src/server/http.ts | 78 +++++++++--------- src/server/nested-runs.test.ts | 45 ++++------- src/server/pubsub.ts | 14 +++- src/server/runs.test.ts | 134 ++++++++++++++---------------- src/server/runs.ts | 95 ++++++++++++---------- src/server/scheduler.ts | 42 +++++++--- src/server/test-daemon.ts | 33 ++++++++ src/server/two-daemons.test.ts | 39 ++++++--- 16 files changed, 516 insertions(+), 442 deletions(-) create mode 100644 src/server/daemon-runtime.ts create mode 100644 src/server/test-daemon.ts diff --git a/src/lib/dedupe.ts b/src/lib/dedupe.ts index 32e72d1..41a35c9 100644 --- a/src/lib/dedupe.ts +++ b/src/lib/dedupe.ts @@ -16,9 +16,15 @@ * again — a dead run must not hold a key forever. Release happens the moment * the holding run reaches any terminal state (including cancelled/failed); * see `startTrackedRun` in server/runs.ts. + * + * #38: "per-process" means per daemon. The registry is a `Context.Service` + * whose layer builds a fresh map each time a daemon runtime is built + * (`server/daemon-runtime.ts`), so two daemons in one process never share + * keys. `createDedupeRegistry` stays exported for unit tests of the plain + * map logic. */ -import { Schema } from "effect"; +import { Context, Layer, Schema } from "effect"; export class DedupeKeyError extends Schema.TaggedError()("DedupeKeyError", { key: Schema.String, @@ -44,7 +50,7 @@ export class DedupeKeyError extends Schema.TaggedError()("Dedupe } } -export interface DedupeRegistry { +export interface DedupeRegistryShape { /** The run id currently holding `key`, or `undefined`. */ holderOf(key: string): string | undefined; /** @@ -57,7 +63,7 @@ export interface DedupeRegistry { release(key: string, runId: string): void; } -export function createDedupeRegistry(): DedupeRegistry { +export function createDedupeRegistry(): DedupeRegistryShape { const held = new Map(); return { holderOf: (key) => held.get(key), @@ -72,3 +78,17 @@ export function createDedupeRegistry(): DedupeRegistry { }, }; } + +/** + * The dedupe registry as a daemon service (#38, ADR 0009 §5). Resolve it from + * the daemon runtime's context; provide it with `DedupeRegistryLayer`. + */ +export class DedupeRegistry extends Context.Service()( + "DedupeRegistry", +) {} + +/** A fresh, empty registry per layer build — i.e. per daemon runtime. */ +export const DedupeRegistryLayer: Layer.Layer = Layer.sync( + DedupeRegistry, + createDedupeRegistry, +); diff --git a/src/lib/workspace.ts b/src/lib/workspace.ts index 15c89ac..d2868df 100644 --- a/src/lib/workspace.ts +++ b/src/lib/workspace.ts @@ -22,6 +22,7 @@ * non-host workspace is plumbing or a rewrite. */ +import { Context, Layer } from "effect"; import { existsSync, readdirSync, statSync } from "node:fs"; import { mkdir, rm } from "node:fs/promises"; import { join } from "node:path"; @@ -31,12 +32,18 @@ import type { WorkspaceKind } from "../workflow"; const MIRROR_DIR = ".mirror.git"; -export interface RefreshGates { +/** + * The mirror-refresh promise queue, keyed by mirror path (see the + * concurrency note above). #38: per daemon, not per module — it is the + * `RefreshGates` service below, and `allocateWorkspace` takes the resolved + * value as a plain argument. + */ +export interface RefreshGatesShape { get(mirrorPath: string): Promise | undefined; set(mirrorPath: string, promise: Promise): void; } -export function createRefreshGates(): RefreshGates { +export function createRefreshGates(): RefreshGatesShape { const gates = new Map>(); return { get: (mirrorPath) => gates.get(mirrorPath), @@ -46,6 +53,19 @@ export function createRefreshGates(): RefreshGates { }; } +/** + * The refresh gates as a daemon service (#38, ADR 0009 §5); provided by + * `RefreshGatesLayer`, one fresh queue per daemon runtime. + */ +export class RefreshGates extends Context.Service()( + "RefreshGates", +) {} + +export const RefreshGatesLayer: Layer.Layer = Layer.sync( + RefreshGates, + createRefreshGates, +); + export interface WorkspaceAllocationInput { readonly runId: string; readonly workspaceRoot: string; @@ -72,11 +92,11 @@ export interface WorkspaceAllocationInput { /** Injectable host-exec seam; defaults to `hostExec` (no behaviour change). */ readonly exec?: ExecFn; /** Per-daemon refresh gates for serializing mirror maintenance. */ - readonly refreshGates: RefreshGates; + readonly refreshGates: RefreshGatesShape; } function enqueueRefresh( - refreshGates: RefreshGates, + refreshGates: RefreshGatesShape, mirrorPath: string, task: () => Promise, ): Promise { diff --git a/src/runtime/run.ts b/src/runtime/run.ts index 8387371..d6d05fd 100644 --- a/src/runtime/run.ts +++ b/src/runtime/run.ts @@ -142,6 +142,18 @@ function resolveOutput( } } +/** + * What `startRun` needs from a runtime: forking and running effects that + * require `AgentRuntime`. Any `ManagedRuntime` that provides at least the + * agent runtime satisfies it — the CLI's agent-only runtime and the daemon's + * runtime (#38), which provides more. (`ManagedRuntime` itself is invariant + * in its services, so naming the two methods is what lets both through.) + */ +export type AgentRunner = Pick< + ManagedRuntime.ManagedRuntime, + "runFork" | "runPromise" +>; + /** * `runtime` is the composition root's `ManagedRuntime` (issue #36): agent * steps resolve their adapter from its `AgentRuntime` service and their @@ -150,7 +162,7 @@ function resolveOutput( */ export function startRun( workflow: WorkflowDefinition, - runtime: ManagedRuntime.ManagedRuntime, + runtime: AgentRunner, options: StartRunOptions, ): RunHandle { let seq = 0; diff --git a/src/server/concurrency.test.ts b/src/server/concurrency.test.ts index 3d2c2d6..f2691a0 100644 --- a/src/server/concurrency.test.ts +++ b/src/server/concurrency.test.ts @@ -19,13 +19,9 @@ import { describe, expect, test } from "bun:test"; import { defineConfig } from "../config"; import { getRunEvents, openStore } from "../persistence/store"; import { createSlowFakeAdapter } from "../replay/adapter"; -import { makeAgentRuntime } from "../runtime/agent-runtime"; import { admitRun } from "./admission"; import { serve } from "./http"; -import { createRunRegistry } from "./runs"; -import { createPubSub } from "./pubsub"; -import { createDedupeRegistry } from "../lib/dedupe"; -import { createRefreshGates } from "../lib/workspace"; +import { makeDaemonRuntime } from "./daemon-runtime"; const TREE_WORKFLOW = `${import.meta.dir}/../../test/fixtures/tree-workflow.ts`; @@ -75,16 +71,9 @@ describe("phase 5 P1: per-run working trees (D28) over the real server", () => { const workspaceRoot = join(root, "workspaces"); const db = openStore(join(root, "factory.db")); - const services = { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; const server = serve({ - services, db, - runtime: makeAgentRuntime( + runtime: makeDaemonRuntime( createSlowFakeAdapter( [ { type: "TEXT_MESSAGE_START" }, @@ -171,16 +160,9 @@ describe("phase 5 P1: per-run working trees (D28) over the real server", () => { await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); const db = openStore(join(root, "factory.db")); - const services = { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; const server = serve({ - services, db, - runtime: makeAgentRuntime( + runtime: makeDaemonRuntime( createSlowFakeAdapter( [ { type: "TEXT_MESSAGE_START" }, @@ -240,16 +222,9 @@ describe("phase 5 P1: per-run working trees (D28) over the real server", () => { const workspaceRoot = join(root, "workspaces"); const db = openStore(join(root, "factory.db")); - const services = { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; const server = serve({ - services, db, - runtime: makeAgentRuntime( + runtime: makeDaemonRuntime( createSlowFakeAdapter( [ { type: "TEXT_MESSAGE_START" }, diff --git a/src/server/daemon-runtime.ts b/src/server/daemon-runtime.ts new file mode 100644 index 0000000..e12d26b --- /dev/null +++ b/src/server/daemon-runtime.ts @@ -0,0 +1,56 @@ +/** + * The daemon's composition root as a value (#38, ADR 0009 §5, following + * #36's `AgentRuntime`): every per-daemon singleton is a `Context.Service`, + * and `DaemonLayer` composes their layers with `AgentRuntimeLayer` into one + * `ManagedRuntime`. Building the runtime builds the state — a fresh run + * registry, pubsub, dedupe registry and refresh-gate queue — so two daemons + * are two runtimes, and a test that wants clean state builds a new one. + * + * The boundaries (HTTP handlers, the scheduler's fire function, `ctx.dispatch`) + * stay plain async per ADR 0009 §5 and resolve services from the runtime with + * `serviceOf`. Every layer here is synchronous, so `serviceOf` is a + * synchronous lookup — which is what lets `startTrackedRun` resolve its + * services and still reserve its slot with no `await` between check and set. + */ + +import { Effect, Layer, ManagedRuntime, type Context } from "effect"; +import { DedupeRegistry, DedupeRegistryLayer } from "../lib/dedupe"; +import { RefreshGates, RefreshGatesLayer } from "../lib/workspace"; +import type { AgentAdapter } from "../runtime/agent-adapter"; +import { AgentRuntime, AgentRuntimeLayer } from "../runtime/agent-runtime"; +import { RunPubSub, RunPubSubLayer } from "./pubsub"; +import { RunRegistry, RunRegistryLayer } from "./runs"; + +/** Every service a daemon runtime provides. */ +export type DaemonServices = AgentRuntime | RunRegistry | RunPubSub | DedupeRegistry | RefreshGates; + +/** The daemon's `ManagedRuntime`: the agent runtime plus its per-daemon state. */ +export type DaemonRuntime = ManagedRuntime.ManagedRuntime; + +/** The daemon's layer: `adapter` (or the opencode default) plus fresh per-daemon state. */ +export const DaemonLayer = (adapter?: AgentAdapter): Layer.Layer => + Layer.mergeAll( + AgentRuntimeLayer(adapter), + RunRegistryLayer, + RunPubSubLayer, + DedupeRegistryLayer, + RefreshGatesLayer, + ); + +/** + * Build a daemon runtime. The caller owns it and must `dispose()` it; tests + * build one per case for isolated state. + */ +export const makeDaemonRuntime = (adapter?: AgentAdapter): DaemonRuntime => + ManagedRuntime.make(DaemonLayer(adapter)); + +/** + * Resolve one of the daemon's services from its runtime, synchronously — the + * plain-async boundary's way into the context. + */ +export function serviceOf( + runtime: DaemonRuntime, + service: Context.Key, +): S { + return runtime.runSync(Effect.service(service)); +} diff --git a/src/server/daemon.ts b/src/server/daemon.ts index 6285259..94bf6a3 100644 --- a/src/server/daemon.ts +++ b/src/server/daemon.ts @@ -11,35 +11,24 @@ * `runtime/agent-step.ts` instead of being threaded through every options * type. `DaemonHandle.stop` disposes it. * - * #38: the daemon creates per-instance services (run registry, pubsub, - * dedupe registry, refresh gates) so two daemons can coexist in one process - * without sharing state. + * #38: that runtime is the daemon runtime (`server/daemon-runtime.ts`): the + * agent runtime layer composed with the per-daemon state layers (run + * registry, pubsub, dedupe registry, refresh gates). Building it builds fresh + * state, so two daemons are two runtimes and coexist in one process without + * sharing anything. The scheduler loop is forked on it and resolves its + * dedupe registry and clock from its context. */ import { mkdir } from "node:fs/promises"; import { dirname } from "node:path"; -import { Clock, Effect, Fiber } from "effect"; +import { Effect, Fiber } from "effect"; type AnyFiber = Fiber.Fiber; import type { FactoryConfig } from "../config"; import { openStore } from "../persistence/store"; import { serve } from "./http"; -import { - createRunRegistry, - type DaemonServices, - type DispatchEnv, - type WorkspaceSpec, -} from "./runs"; -import { createPubSub } from "./pubsub"; -import { createDedupeRegistry } from "../lib/dedupe"; -import { createRefreshGates } from "../lib/workspace"; -import { - createSchedulerState, - makeScheduleFire, - runSchedulerLoop, - toRuntimeSchedules, - type SchedulerDeps, -} from "./scheduler"; -import { makeAgentRuntime } from "../runtime/agent-runtime"; +import type { DispatchEnv, WorkspaceSpec } from "./runs"; +import { makeScheduleFire, runSchedulerLoop, toRuntimeSchedules } from "./scheduler"; +import { makeDaemonRuntime, type DaemonRuntime } from "./daemon-runtime"; export interface DaemonOptions { readonly dbPath: string; @@ -53,16 +42,18 @@ export interface DaemonOptions { * Absent, the legacy per-request `{dir, clone}` behaviour is kept. */ readonly config?: FactoryConfig; - /** Issue #38: the scheduler's clock, over Effect's default wall clock. */ - readonly clock?: Clock.Clock; } export interface DaemonHandle { readonly server: ReturnType; /** Issue #16: the loop that fires the config's schedules, when it has any. */ readonly schedulerFiber: AnyFiber | undefined; - /** Issue #38: this daemon's own registry, pubsub, dedupe registry and refresh gates. */ - readonly services: DaemonServices; + /** + * Issue #38: this daemon's runtime — its agent runtime plus its own + * registry, pubsub, dedupe registry and refresh gates, resolvable with + * `serviceOf`. + */ + readonly runtime: DaemonRuntime; /** * Shut the daemon down: interrupt the scheduler, stop the HTTP server, and * dispose the agent runtime (issue #36). @@ -77,19 +68,11 @@ export async function startDaemon(options: DaemonOptions): Promise await mkdir(dirname(options.dbPath), { recursive: true }); const db = openStore(options.dbPath); - const runtime = makeAgentRuntime(options.config?.agent.adapter); - - const services: DaemonServices = { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; + const runtime = makeDaemonRuntime(options.config?.agent.adapter); const server = serve({ db, runtime, - services, port: options.port, ...(options.config !== undefined ? { config: options.config } : {}), }); @@ -113,25 +96,18 @@ export async function startDaemon(options: DaemonOptions): Promise maxDispatchDepth: config.maxDispatchDepth, maxChildrenPerRun: config.maxChildrenPerRun, }; - const deps: SchedulerDeps = { - schedules: toRuntimeSchedules(config), - fire: makeScheduleFire({ - db, - runtime, - services, - maxConcurrentRuns, - workspace, - repo, - dispatchEnv, - }), - dedupeRegistry: services.dedupeRegistry, - clock: options.clock ?? Clock.Clock.defaultValue(), - }; - const state = createSchedulerState(deps); - return Effect.runFork( + const fire = makeScheduleFire({ + db, + runtime, + maxConcurrentRuns, + workspace, + repo, + dispatchEnv, + }); + return runtime.runFork( runSchedulerLoop( - deps, - state, + toRuntimeSchedules(config), + fire, options.schedulerIntervalMs ?? DEFAULT_SCHEDULER_INTERVAL_MS, ), ); @@ -144,5 +120,5 @@ export async function startDaemon(options: DaemonOptions): Promise await runtime.dispose(); }; - return { server, schedulerFiber, services, stop }; + return { server, schedulerFiber, runtime, stop }; } diff --git a/src/server/dedupe.test.ts b/src/server/dedupe.test.ts index e198066..934e0c4 100644 --- a/src/server/dedupe.test.ts +++ b/src/server/dedupe.test.ts @@ -5,8 +5,8 @@ * with a key a non-terminal run already holds throws a `DedupeKeyError` * naming the key and the holding run. * - * #38: tests create their own DaemonServices instead of using module-level - * singletons. A holder is kept in its reserved-but-not-started window by + * #38: each test builds its own daemon runtime (`createTestDaemon`) instead + * of sharing module-level singletons. A holder is kept in its reserved-but-not-started window by * pre-seeding the daemon's refresh gate for its workspace mirror. */ @@ -16,27 +16,16 @@ import { join } from "node:path"; import { describe, expect, test } from "bun:test"; import { openStore, getRunEvents } from "../persistence/store"; import { createSlowFakeAdapter } from "../replay/adapter"; -import { makeAgentRuntime } from "../runtime/agent-runtime"; import { defineWorkflow, Schema } from "../workflow"; -import { createDedupeRegistry, DedupeKeyError } from "../lib/dedupe"; -import { createRunRegistry, startTrackedRun, type DaemonServices } from "./runs"; -import { createPubSub } from "./pubsub"; -import { createRefreshGates } from "../lib/workspace"; +import { DedupeKeyError } from "../lib/dedupe"; +import { startTrackedRun } from "./runs"; +import { createTestDaemon, type TestDaemon } from "./test-daemon"; import { serve } from "./http"; import { defineConfig } from "../config"; /** Fire-and-forget children keep filling the registry; drain before closing the store. */ -async function drain(services: DaemonServices, timeoutMs = 10_000): Promise { - await waitFor(() => services.registry.activeRunIds().length === 0, timeoutMs); -} - -function createTestServices(): DaemonServices { - return { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; +async function drain(daemon: TestDaemon, timeoutMs = 10_000): Promise { + await waitFor(() => daemon.registry.activeRunIds().length === 0, timeoutMs); } const SLOW_ADAPTER = createSlowFakeAdapter( @@ -48,8 +37,6 @@ const SLOW_ADAPTER = createSlowFakeAdapter( 25, ); -const runtime = makeAgentRuntime(SLOW_ADAPTER); - const echoWorkflow = defineWorkflow("echo-wf", { input: Schema.Struct({}), run: async (ctx) => { @@ -84,8 +71,8 @@ interface StartOpts { readonly runId?: string; readonly dir: string; readonly dedupeKey?: string; - /** The daemon services both starts share, so they agree on holders. */ - readonly services: DaemonServices; + /** The daemon daemon both starts share, so they agree on holders. */ + readonly daemon: TestDaemon; /** * Holds the run in its reserved-but-not-started window until released: the * run allocates a clone workspace under `dir` whose mirror refresh is @@ -98,8 +85,8 @@ async function start(db: ReturnType, opts: StartOpts): Promise const location = opts.gate === undefined ? { dir: opts.dir } - : { workspace: await gatedWorkspace(opts.dir, opts.services, opts.gate) }; - return startTrackedRun(runtime, db, opts.services, echoWorkflow, { + : { workspace: await gatedWorkspace(opts.dir, opts.daemon, opts.gate) }; + return startTrackedRun(opts.daemon.runtime, db, echoWorkflow, { ...location, input: {}, maxConcurrentRuns: 4, @@ -110,14 +97,14 @@ async function start(db: ReturnType, opts: StartOpts): Promise async function gatedWorkspace( root: string, - services: DaemonServices, + daemon: TestDaemon, gate: Gate, ): Promise { const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); const workspaceRoot = join(root, "workspaces"); - services.refreshGates.set(join(workspaceRoot, ".mirror.git"), gate.promise); + daemon.refreshGates.set(join(workspaceRoot, ".mirror.git"), gate.promise); return { workspaceRoot, sshUrl: seed, @@ -170,7 +157,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { test("a ctx.dispatch collision throws into the parent and is recorded as DispatchCollision", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-dispatch-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); mkdirSync(join(root, "workspaces"), { recursive: true }); const parent = defineWorkflow("parent-wf", { @@ -188,14 +175,14 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, services, parent, { + const parentRunId = await startTrackedRun(daemon.runtime, db, parent, { input: { wait: 3 }, workspace: workspaces(join(root)), maxConcurrentRuns: 10, dispatchEnv: withWorkspaces(join(root)), }); - await waitFor(() => !services.registry.isActive(parentRunId)); + await waitFor(() => !daemon.registry.isActive(parentRunId)); const events = getRunEvents(db, parentRunId); const collided = events.find((event) => event.payload._tag === "DispatchCollision"); @@ -211,7 +198,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { failed !== undefined && failed.payload._tag === "RunFailed" ? failed.payload.message : "", ).toMatch(/dedupe key held: "item:7"/); - await drain(services); + await drain(daemon); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -219,7 +206,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { test("the key is released when the holding child run finishes, so a later dispatch succeeds", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-dispatch-release-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); mkdirSync(join(root, "workspaces"), { recursive: true }); const gate = makeGate(); @@ -246,7 +233,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, services, parent, { + const parentRunId = await startTrackedRun(daemon.runtime, db, parent, { input: {}, workspace: workspaces(join(root)), maxConcurrentRuns: 10, @@ -267,9 +254,9 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { firstChild !== undefined && firstChild.payload._tag === "RunDispatched" ? firstChild.payload.childRunId : ""; - expect(services.dedupeRegistry.holderOf("item:8")).toBe(firstChildRunId); + expect(daemon.dedupeRegistry.holderOf("item:8")).toBe(firstChildRunId); gate.release(); - await waitFor(() => !services.registry.isActive(firstChildRunId)); + await waitFor(() => !daemon.registry.isActive(firstChildRunId)); parentGate.release(); await waitFor( () => @@ -277,7 +264,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { .length >= 2, 10_000, ); - await waitFor(() => !services.registry.isActive(parentRunId)); + await waitFor(() => !daemon.registry.isActive(parentRunId)); const dispatchedCount = getRunEvents(db, parentRunId).filter( (event) => event.payload._tag === "RunDispatched", @@ -287,7 +274,7 @@ describe("dedupe keys through ctx.dispatch (issue #15)", () => { getRunEvents(db, parentRunId).some((event) => event.payload._tag === "DispatchCollision"), ).toBe(false); - await drain(services); + await drain(daemon); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -298,19 +285,19 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-hold-")); const db = openStore(join(root, "factory.db")); const gate = makeGate(); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const first = start(db, { runId: "run-holder", dir: join(root, "dir-1"), dedupeKey: "item:41", gate, - services, + daemon, }); - await waitFor(() => services.registry.isActive("run-holder")); + await waitFor(() => daemon.registry.isActive("run-holder")); const err = (await errorOf( - start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:41", services }), + start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:41", daemon }), )) as DedupeKeyError | string; expect(err).toBeInstanceOf(DedupeKeyError); if (err instanceof DedupeKeyError) { @@ -321,9 +308,9 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { gate.release(); await first; - await waitFor(() => !services.registry.isActive("run-holder")); + await waitFor(() => !daemon.registry.isActive("run-holder")); - await drain(services); + await drain(daemon); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -332,24 +319,24 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-log-")); const db = openStore(join(root, "factory.db")); const gate = makeGate(); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const first = start(db, { runId: "run-holder", dir: join(root, "dir-1"), dedupeKey: "item:42", gate, - services, + daemon, }); - await waitFor(() => services.registry.isActive("run-holder")); + await waitFor(() => daemon.registry.isActive("run-holder")); await errorOf( - start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:42", services }), + start(db, { runId: "run-collide", dir: join(root, "dir-2"), dedupeKey: "item:42", daemon }), ); gate.release(); await first; - await waitFor(() => !services.registry.isActive("run-holder")); + await waitFor(() => !daemon.registry.isActive("run-holder")); const holderStarted = getRunEvents(db, "run-holder").find( (e) => e.payload._tag === "RunStarted", @@ -364,7 +351,7 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { // A dropped start never started: the collision run's log has no rows. expect(getRunEvents(db, "run-collide")).toHaveLength(0); - await drain(services); + await drain(daemon); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -372,25 +359,25 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("the key is released when the holding run reaches its terminal state", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-finish-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const firstRunId = await start(db, { dir: join(root, "dir-1"), dedupeKey: "item:43", - services, + daemon, }); expect(firstRunId).toBeTypeOf("string"); - await waitFor(() => !services.registry.isActive(firstRunId)); + await waitFor(() => !daemon.registry.isActive(firstRunId)); const retryRunId = await start(db, { dir: join(root, "dir-2"), dedupeKey: "item:43", - services, + daemon, }); - await waitFor(() => !services.registry.isActive(retryRunId)); + await waitFor(() => !daemon.registry.isActive(retryRunId)); expect(getRunEvents(db, retryRunId).some((e) => e.payload._tag === "RunStarted")).toBe(true); - await drain(services); + await drain(daemon); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -398,11 +385,11 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a failed start releases its claim so a corrected retry can start", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-fail-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); // allocation of an explicitly undefined dir fails after the claim await expect( - startTrackedRun(runtime, db, services, echoWorkflow, { + startTrackedRun(daemon.runtime, db, echoWorkflow, { input: {}, dedupeKey: "item:44", }), @@ -411,12 +398,12 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { const retryRunId = await start(db, { dir: join(root, "dir-1"), dedupeKey: "item:44", - services, + daemon, }); - await waitFor(() => !services.registry.isActive(retryRunId)); + await waitFor(() => !daemon.registry.isActive(retryRunId)); expect(getRunEvents(db, retryRunId).some((e) => e.payload._tag === "RunStarted")).toBe(true); - await drain(services); + await drain(daemon); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -424,29 +411,29 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("a cancelled run releases its key (RunCancelled is a terminal state)", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-cancel-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); - const holderRunId = await startTrackedRun(runtime, db, services, echoWorkflow, { + const holderRunId = await startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-cancelled", dir: join(root, "dir-1"), input: {}, dedupeKey: "item:45", }); - await waitFor(() => !services.registry.isActive(holderRunId) || true); - const handle = services.registry.getActiveHandle("run-cancelled"); + await waitFor(() => !daemon.registry.isActive(holderRunId) || true); + const handle = daemon.registry.getActiveHandle("run-cancelled"); expect(handle).toBeDefined(); if (handle !== undefined) await handle.cancel(); - await waitFor(() => !services.registry.isActive(holderRunId)); + await waitFor(() => !daemon.registry.isActive(holderRunId)); const started = getRunEvents(db, holderRunId).some((e) => e.payload._tag === "RunStarted"); - const retryRunId = await startTrackedRun(runtime, db, services, echoWorkflow, { + const retryRunId = await startTrackedRun(daemon.runtime, db, echoWorkflow, { dir: join(root, "dir-2"), input: {}, dedupeKey: "item:45", }); expect(started).toBe(true); expect(retryRunId).toBeTypeOf("string"); - await drain(services); + await drain(daemon); db.close(); rmSync(root, { recursive: true, force: true }); @@ -455,17 +442,17 @@ describe("dedupe keys through startTrackedRun (issue #15)", () => { test("runs started without a key behave exactly as before", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-none-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); - const firstRunId = await start(db, { dir: join(root, "dir-1"), services }); - const secondRunId = await start(db, { dir: join(root, "dir-2"), services }); + const firstRunId = await start(db, { dir: join(root, "dir-1"), daemon }); + const secondRunId = await start(db, { dir: join(root, "dir-2"), daemon }); expect(firstRunId).toBeTypeOf("string"); expect(secondRunId).toBeTypeOf("string"); await waitFor( - () => !services.registry.isActive(firstRunId) && !services.registry.isActive(secondRunId), + () => !daemon.registry.isActive(firstRunId) && !daemon.registry.isActive(secondRunId), ); - await drain(services); + await drain(daemon); db.close(); rmSync(root, { recursive: true, force: true }); }); @@ -489,7 +476,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { test("a key held by a running run surfaces as a 409 conflict naming the key and the holder", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-http-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const slowStartWorkflow = defineWorkflow("dedupe-http-slow", { input: Schema.Struct({}), @@ -503,8 +490,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const server = serve({ db, - runtime: makeAgentRuntime(SLOW_ADAPTER), - services, + runtime: daemon.runtime, port: 0, config: defineConfig({ repo: { @@ -560,7 +546,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const { runId: retryRunId } = (await retryRes.json()) as { runId: string }; await waitForTerminalStatus(db, retryRunId); } finally { - await drain(services); + await drain(daemon); await server.stop(true); db.close(); rmSync(root, { recursive: true, force: true }); @@ -570,7 +556,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { test("a non-string dedupeKey is a 400, and omitting one behaves as today", async () => { const root = mkdtempSync(join(tmpdir(), "factory-dedupe-http-shape-")); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const plainWorkflow = defineWorkflow("dedupe-http-plain", { input: Schema.Struct({}), workspace: { kind: "scratch" }, @@ -579,8 +565,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const server = serve({ db, - runtime: makeAgentRuntime(SLOW_ADAPTER), - services, + runtime: daemon.runtime, port: 0, config: defineConfig({ repo: { @@ -611,7 +596,7 @@ describe("dedupe keys over POST /api/runs (issue #15)", () => { const { runId } = (await plain.json()) as { runId: string }; await waitForTerminalStatus(db, runId); } finally { - await drain(services); + await drain(daemon); await server.stop(true); db.close(); rmSync(root, { recursive: true, force: true }); diff --git a/src/server/http.test.ts b/src/server/http.test.ts index 46b1d29..780950e 100644 --- a/src/server/http.test.ts +++ b/src/server/http.test.ts @@ -6,16 +6,14 @@ import { Schema } from "effect"; import { defineWorkflow, type WorkflowDefinition } from "../workflow"; import { createSlowFakeAdapter } from "../replay/adapter"; import type { AgentAdapter } from "../runtime/agent-adapter"; -import { makeAgentRuntime } from "../runtime/agent-runtime"; import { getRunEvents, openStore } from "../persistence/store"; import { defineConfig, loadFactoryConfig } from "../config"; import registryWorkflow from "../../test/fixtures/registry-workflow"; import registryScratchWorkflow from "../../test/fixtures/registry-scratch-workflow"; import { serve } from "./http"; -import { createRunRegistry, type DaemonServices } from "./runs"; -import { createPubSub } from "./pubsub"; -import { createDedupeRegistry } from "../lib/dedupe"; -import { createRefreshGates } from "../lib/workspace"; +// Issue #38: each server under test gets its own daemon runtime, so its own +// registry, pubsub, dedupe registry and refresh gates. +import { makeDaemonRuntime } from "./daemon-runtime"; const ECHO_WORKFLOW = `${import.meta.dir}/../../test/fixtures/echo-workflow.ts`; const QUIET_GAP_WORKFLOW = `${import.meta.dir}/../../test/fixtures/quiet-gap-workflow.ts`; @@ -89,16 +87,6 @@ async function readSseUntilTerminal( return frames; } -/** Issue #38: each server under test gets its own per-daemon services. */ -function createTestServices(): DaemonServices { - return { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; -} - describe("GET /api/workflows (D30)", () => { test("lists the config's workflows with id and JSON Schema input", async () => { const dir = mkdtempSync(join(tmpdir(), "factory-workflows-test-")); @@ -107,8 +95,7 @@ describe("GET /api/workflows (D30)", () => { const config = await loadFactoryConfig(FIXTURE_CONFIG); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, config, }); @@ -138,8 +125,7 @@ describe("GET /api/workflows (D30)", () => { const adapter = createSlowFakeAdapter([], 1); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, }); const base = `http://localhost:${server.port}`; @@ -163,8 +149,7 @@ describe("GET /api/workflows (D30)", () => { const adapter = createSlowFakeAdapter([], 1); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, }); const base = `http://localhost:${server.port}`; @@ -188,8 +173,7 @@ describe("phase 4 SPA serving", () => { const adapter = createSlowFakeAdapter([{ type: "TEXT_MESSAGE_START" }], 1); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, }); const base = `http://localhost:${server.port}`; @@ -276,8 +260,7 @@ describe("SSE keepalive", () => { const adapter = createSlowFakeAdapter([], 1); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, sseKeepaliveMs: 25, }); @@ -318,8 +301,7 @@ describe("SSE keepalive", () => { const adapter = createSlowFakeAdapter([], 1); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, sseKeepaliveMs: 10, }); @@ -364,8 +346,7 @@ describe("phase 3 HTTP API + SSE", () => { ); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, }); const base = `http://localhost:${server.port}`; @@ -418,8 +399,7 @@ describe("phase 3 HTTP API + SSE", () => { ); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, }); const base = `http://localhost:${server.port}`; @@ -459,8 +439,7 @@ describe("phase 3 HTTP API + SSE", () => { ); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, }); const base = `http://localhost:${server.port}`; @@ -503,8 +482,7 @@ describe("phase 3 HTTP API + SSE", () => { ); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, }); const base = `http://localhost:${server.port}`; @@ -546,8 +524,7 @@ describe("phase 3 HTTP API + SSE", () => { ); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, }); const base = `http://localhost:${server.port}`; @@ -597,8 +574,7 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { const config = await loadFactoryConfig(FIXTURE_CONFIG); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, config, }); @@ -629,8 +605,7 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { const config = await loadFactoryConfig(FIXTURE_CONFIG); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, config, }); @@ -681,8 +656,7 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, config, }); @@ -750,8 +724,7 @@ describe("POST /api/runs {workflowId, input} (D31)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, config, }); @@ -823,8 +796,7 @@ describe("a scratch workflow through POST /api/runs (issue #13)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, config, }); @@ -900,8 +872,7 @@ describe("ctx.dispatch through POST /api/runs (issue #14)", () => { const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, config: defineConfig({ repo: { @@ -999,8 +970,7 @@ describe("GET /api/schedules (issue #17)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), - services: createTestServices(), + runtime: makeDaemonRuntime(createSlowFakeAdapter([], 1)), port: 0, config: schedulesConfig(root, workflow), }); @@ -1069,8 +1039,7 @@ describe("GET /api/schedules (issue #17)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), - services: createTestServices(), + runtime: makeDaemonRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1096,8 +1065,7 @@ describe("GET /api/schedules (issue #17)", () => { const db = openStore(join(root, "factory.db")); const server = serve({ db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), - services: createTestServices(), + runtime: makeDaemonRuntime(createSlowFakeAdapter([], 1)), port: 0, }); const base = `http://localhost:${server.port}`; @@ -1143,8 +1111,7 @@ describe("GET /api/schedules (issue #17)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), - services: createTestServices(), + runtime: makeDaemonRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1245,8 +1212,7 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), - services: createTestServices(), + runtime: makeDaemonRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1300,8 +1266,7 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), - services: createTestServices(), + runtime: makeDaemonRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1344,8 +1309,7 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), - services: createTestServices(), + runtime: makeDaemonRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1410,8 +1374,7 @@ describe("POST /api/schedules/:id/run (issue #17)", () => { }); const server = serve({ db, - runtime: makeAgentRuntime(createSlowFakeAdapter([], 1)), - services: createTestServices(), + runtime: makeDaemonRuntime(createSlowFakeAdapter([], 1)), port: 0, config, }); @@ -1462,8 +1425,7 @@ describe("adapter.prepareWorkspace through the legacy POST /api/runs (#37)", () const adapter = trackingAdapter(); const server = serve({ db, - runtime: makeAgentRuntime(adapter), - services: createTestServices(), + runtime: makeDaemonRuntime(adapter), port: 0, }); const base = `http://localhost:${server.port}`; diff --git a/src/server/http.ts b/src/server/http.ts index 45c2abb..27bb1db 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -26,26 +26,30 @@ * read completes. Without this ordering a live event emitted between the * subscribe and the read could be lost. * - * #38: the handler receives DaemonServices (registry, pubsub, dedupe) as - * required parameters rather than accessing module-level singletons. + * #38: the handler resolves the daemon's registry and pubsub from the daemon + * runtime it is given (`server/daemon-runtime.ts`) rather than reading + * module-level singletons. The handlers themselves stay plain functions + * (ADR 0009 §5) — the runtime is only the way into the context. */ import type { Database } from "bun:sqlite"; import { isTerminal, type RunEvent } from "../events"; import type { FactoryConfig } from "../config"; -import { Schema, SchemaParser, type ManagedRuntime } from "effect"; +import { Schema, SchemaParser } from "effect"; import { resetClone, type GitIdentity } from "../lib/clone"; import { loadWorkflow } from "../lib/load-workflow"; import { getRunEvents, listRuns, type RunSummary } from "../persistence/store"; -import type { AgentRuntime } from "../runtime/agent-runtime"; import index from "../web/index.html"; import { admitRun } from "./admission"; import { DedupeKeyError } from "../lib/dedupe"; +import { serviceOf, type DaemonRuntime } from "./daemon-runtime"; +import { RunPubSub, type PubSub } from "./pubsub"; import { ConcurrencyLimitError, + RunRegistry, startTrackedRun, - type DaemonServices, type DispatchEnv, + type RunRegistryShape, type StartTrackedRunOptions, } from "./runs"; import { nextFireAt, toRuntimeSchedules } from "./scheduler"; @@ -74,14 +78,14 @@ export interface ScheduleSummary { export interface ServerOptions { readonly db: Database; - /** Issue #38: this daemon's run registry, pubsub, dedupe registry and refresh gates. */ - readonly services: DaemonServices; /** * Issue #36: the Effect managed runtime that provides the agent runtime * service. The adapter is resolved from its context inside the agent step - * rather than threaded through `ServerOptions`. + * rather than threaded through `ServerOptions`. #38: the same runtime + * provides this daemon's run registry, pubsub, dedupe registry and refresh + * gates. */ - readonly runtime: ManagedRuntime.ManagedRuntime; + readonly runtime: DaemonRuntime; /** * The loaded `factory.config.ts` (D27). Absent = the phase 3 path-based API * behaves exactly as before (no limit, explicit dir+clone, `/api/workflows` @@ -127,8 +131,11 @@ export type RunSummaryResponse = RunSummary & { readonly active: boolean }; * that bit to group live rows, so it is derived here from the in-memory * registry (`server/runs.ts`) rather than stored (D24: active is process state). */ -function listSummaries(db: Database, services: DaemonServices): ReadonlyArray { - return listRuns(db).map((run) => ({ ...run, active: services.registry.isActive(run.runId) })); +function listSummaries( + db: Database, + registry: RunRegistryShape, +): ReadonlyArray { + return listRuns(db).map((run) => ({ ...run, active: registry.isActive(run.runId) })); } /** @@ -218,7 +225,8 @@ function parseLastEventId(req: Request): number | undefined { function sseStream( db: Database, - services: DaemonServices, + registry: RunRegistryShape, + pubsub: PubSub, runId: string, lastEventId?: number, keepaliveMs: number = DEFAULT_SSE_KEEPALIVE_MS, @@ -270,7 +278,7 @@ function sseStream( controller.close(); }; - unsubscribe = services.pubsub.subscribe(runId, (event) => { + unsubscribe = pubsub.subscribe(runId, (event) => { if (!draining) { buffered.push(event); return; @@ -293,7 +301,7 @@ function sseStream( if (isTerminal(event.payload)) sawTerminal = true; } - if (sawTerminal || !services.registry.isActive(runId)) { + if (sawTerminal || !registry.isActive(runId)) { finish(); return; } @@ -329,12 +337,13 @@ function sseStream( } export function createHandler(options: ServerOptions): (req: Request) => Promise { - const services = options.services; + const registry = serviceOf(options.runtime, RunRegistry); + const pubsub = serviceOf(options.runtime, RunPubSub); return async (req: Request): Promise => { const url = new URL(req.url); if (req.method === "GET" && url.pathname === "/api/runs") { - return json(listSummaries(options.db, services)); + return json(listSummaries(options.db, registry)); } if (req.method === "GET" && url.pathname === "/api/workflows") { @@ -355,7 +364,7 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise const maxConcurrentRuns = options.config?.maxConcurrentRuns; if ( maxConcurrentRuns !== undefined && - !admitRun(maxConcurrentRuns, services.registry.activeRunIds().length) + !admitRun(maxConcurrentRuns, registry.activeRunIds().length) ) { return json( { error: ConcurrencyLimitError.of({ maxConcurrentRuns }).message }, @@ -405,13 +414,7 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise }; let runId: string; try { - runId = await startTrackedRun( - options.runtime, - options.db, - services, - workflow, - startOptions, - ); + runId = await startTrackedRun(options.runtime, options.db, workflow, startOptions); } catch (err) { if (err instanceof ConcurrencyLimitError) { return json({ error: err.message }, { status: 409 }); @@ -480,13 +483,7 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise }; let runId: string; try { - runId = await startTrackedRun( - options.runtime, - options.db, - services, - workflow, - startOptions, - ); + runId = await startTrackedRun(options.runtime, options.db, workflow, startOptions); } catch (err) { if (err instanceof ConcurrencyLimitError) { return json({ error: err.message }, { status: 409 }); @@ -565,7 +562,7 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise // D29 admission, checked here exactly like POST /api/runs so a manual // trigger over the limit is a visible 409 rather than a surprise. const maxConcurrentRuns = options.config.maxConcurrentRuns; - if (!admitRun(maxConcurrentRuns, services.registry.activeRunIds().length)) { + if (!admitRun(maxConcurrentRuns, registry.activeRunIds().length)) { return json( { error: ConcurrencyLimitError.of({ maxConcurrentRuns }).message }, { status: 409 }, @@ -579,7 +576,7 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise // it fires regardless. let runId: string; try { - runId = await startTrackedRun(options.runtime, options.db, services, workflow, { + runId = await startTrackedRun(options.runtime, options.db, workflow, { ...configRunOptions(options.config, maxConcurrentRuns, { scheduleId: schedule.id, ...(schedule.overlap === "skip" ? { dedupeKey: `schedule:${schedule.id}` } : {}), @@ -605,7 +602,7 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise const cancelMatch = /^\/api\/runs\/([^/]+)\/cancel$/.exec(url.pathname); if (req.method === "POST" && cancelMatch) { const runId = cancelMatch[1] as string; - const target = services.registry.cancelRegisteredRun(runId); + const target = registry.cancelRegisteredRun(runId); if (target === undefined) return json({ error: "run not active" }, { status: 409 }); if (target.kind === "handle") await target.handle.cancel(); return json({ runId, cancelled: true }); @@ -614,10 +611,17 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise const eventsMatch = /^\/api\/runs\/([^/]+)\/events$/.exec(url.pathname); if (req.method === "GET" && eventsMatch) { const runId = eventsMatch[1] as string; - const exists = listSummaries(options.db, services).some((r) => r.runId === runId); + const exists = listSummaries(options.db, registry).some((r) => r.runId === runId); if (!exists) return json({ error: "not found" }, { status: 404 }); return new Response( - sseStream(options.db, services, runId, parseLastEventId(req), options.sseKeepaliveMs), + sseStream( + options.db, + registry, + pubsub, + runId, + parseLastEventId(req), + options.sseKeepaliveMs, + ), { headers: { "content-type": "text/event-stream", @@ -631,7 +635,7 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise const runMatch = /^\/api\/runs\/([^/]+)$/.exec(url.pathname); if (req.method === "GET" && runMatch) { const runId = runMatch[1] as string; - const run = listSummaries(options.db, services).find((r) => r.runId === runId); + const run = listSummaries(options.db, registry).find((r) => r.runId === runId); if (run === undefined) return json({ error: "not found" }, { status: 404 }); return json(run); } diff --git a/src/server/nested-runs.test.ts b/src/server/nested-runs.test.ts index 3b74d93..4fe57fa 100644 --- a/src/server/nested-runs.test.ts +++ b/src/server/nested-runs.test.ts @@ -9,7 +9,7 @@ * with the parent's run id recorded on its `RunStarted.parentId`, and the * parent never exposes an awaitable child (D-epic 19). * - * #38: tests create their own DaemonServices. + * #38: each test builds its own daemon runtime (`createTestDaemon`). */ import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; @@ -20,12 +20,9 @@ import { describe, expect, test } from "bun:test"; import { Schema } from "effect"; import { getRunEvents, listRuns, openStore } from "../persistence/store"; import { createSlowFakeAdapter } from "../replay/adapter"; -import { createRunRegistry, startTrackedRun, type DaemonServices } from "./runs"; -import { createPubSub } from "./pubsub"; -import { createDedupeRegistry } from "../lib/dedupe"; -import { createRefreshGates } from "../lib/workspace"; +import { startTrackedRun } from "./runs"; +import { createTestDaemon } from "./test-daemon"; import { defineWorkflow } from "../workflow"; -import { makeAgentRuntime } from "../runtime/agent-runtime"; const SLOW_ADAPTER = createSlowFakeAdapter( [ @@ -36,8 +33,6 @@ const SLOW_ADAPTER = createSlowFakeAdapter( 25, ); -const runtime = makeAgentRuntime(SLOW_ADAPTER); - // The child does a real ctx.exec so its outcome is observable in its log. // Scratch (issue #13): no mirror refresh, so no git fixture is needed. const numberedChild = defineWorkflow("child-wf", { @@ -69,15 +64,6 @@ const manyChildren = defineWorkflow("many-children-wf", { }, }); -function createTestServices(): DaemonServices { - return { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; -} - async function waitFor(predicate: () => boolean, timeoutMs = 10_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -96,7 +82,7 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-happy-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const parent = defineWorkflow("parent-wf", { input: Schema.Struct({}), @@ -107,7 +93,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, services, parent, { + const parentRunId = await startTrackedRun(daemon.runtime, db, parent, { workspace: { workspaceRoot: join(root, "workspaces"), sshUrl: join(root, "seed-not-used"), @@ -126,8 +112,7 @@ describe("nested runs through the daemon (issue #14)", () => { }); await waitFor( - () => - !services.registry.isActive(parentRunId) && services.registry.activeRunIds().length === 0, + () => !daemon.registry.isActive(parentRunId) && daemon.registry.activeRunIds().length === 0, ); const parentEvents = getRunEvents(db, parentRunId); @@ -163,9 +148,9 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-depth-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); - await startTrackedRun(runtime, db, services, selfDispatching, { + await startTrackedRun(daemon.runtime, db, selfDispatching, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -185,7 +170,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => services.registry.activeRunIds().length === 0); + await waitFor(() => daemon.registry.activeRunIds().length === 0); // The chain built to the cap, no farther: root + 5 children max. const selfRuns = listRuns(db).filter((run) => run.workflowId === "self-dispatch-wf"); @@ -215,9 +200,9 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-count-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); - const parentRunId = await startTrackedRun(runtime, db, services, manyChildren, { + const parentRunId = await startTrackedRun(daemon.runtime, db, manyChildren, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -238,7 +223,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => services.registry.activeRunIds().length === 0); + await waitFor(() => daemon.registry.activeRunIds().length === 0); expect(tagsOf(db, parentRunId).filter((tag) => tag === "RunDispatched").length).toBe(5); const parentEvents = getRunEvents(db, parentRunId); @@ -257,7 +242,7 @@ describe("nested runs through the daemon (issue #14)", () => { const root = mkdtempSync(join(tmpdir(), "factory-nested-wip-")); const db = openStore(join(root, "factory.db")); mkdirSync(join(root, "workspaces"), { recursive: true }); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const parent = defineWorkflow("parent-wip-wf", { input: Schema.Struct({}), @@ -268,7 +253,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - const parentRunId = await startTrackedRun(runtime, db, services, parent, { + const parentRunId = await startTrackedRun(daemon.runtime, db, parent, { input: {}, workspace: { workspaceRoot: join(root, "workspaces"), @@ -288,7 +273,7 @@ describe("nested runs through the daemon (issue #14)", () => { }, }); - await waitFor(() => !services.registry.isActive(parentRunId)); + await waitFor(() => !daemon.registry.isActive(parentRunId)); const events = getRunEvents(db, parentRunId); expect(events.map((event) => event.payload._tag)).not.toContain("RunDispatched"); diff --git a/src/server/pubsub.ts b/src/server/pubsub.ts index 8757752..f837789 100644 --- a/src/server/pubsub.ts +++ b/src/server/pubsub.ts @@ -5,10 +5,12 @@ * (`src/persistence/store.ts`); this is the gap between "last flushed to * sqlite" and "just happened", not a second source of truth. * - * Each daemon creates its own PubSub via `createPubSub()` so two daemons in - * one process do not share subscriber state (#38). + * #38: the subscriber map is per daemon, not per module. It is the + * `RunPubSub` service below; its layer builds a fresh map each time a daemon + * runtime is built, so two daemons in one process do not share subscribers. */ +import { Context, Layer } from "effect"; import type { RunEvent } from "../events"; type Listener = (event: RunEvent) => void; @@ -38,3 +40,11 @@ export function createPubSub(): PubSub { }, }; } + +/** + * The live run-event fan-out as a daemon service (#38, ADR 0009 §5). Named + * `RunPubSub` rather than `PubSub` so it never reads as Effect's own module. + */ +export class RunPubSub extends Context.Service()("RunPubSub") {} + +export const RunPubSubLayer: Layer.Layer = Layer.sync(RunPubSub, createPubSub); diff --git a/src/server/runs.test.ts b/src/server/runs.test.ts index 9a32551..3591f0b 100644 --- a/src/server/runs.test.ts +++ b/src/server/runs.test.ts @@ -8,8 +8,8 @@ * run start (L1): the run starts, is cancelled immediately, and ends as a * clean RunCancelled with no orphaned slot. * - * #38: tests create their own DaemonServices instead of using module-level - * singletons. The reserved-but-not-started window is held open by pre-seeding + * #38: each test builds its own daemon runtime (`createTestDaemon`) instead + * of sharing module-level singletons. The reserved-but-not-started window is held open by pre-seeding * the daemon's refresh gate for the workspace mirror, so allocation blocks * until the test releases it. */ @@ -23,19 +23,14 @@ import type { AgentAdapter } from "../runtime/agent-adapter"; import echoWorkflow from "../../test/fixtures/echo-workflow"; import { appendEvent, openStore, getRunEvents } from "../persistence/store"; import { createSlowFakeAdapter } from "../replay/adapter"; -import { makeAgentRuntime } from "../runtime/agent-runtime"; import { defineWorkflow, Schema } from "../workflow"; import { ConcurrencyLimitError, DispatchCapError, - createRunRegistry, startTrackedRun, - type DaemonServices, type WorkspaceSpec, } from "./runs"; -import { createPubSub } from "./pubsub"; -import { createDedupeRegistry } from "../lib/dedupe"; -import { createRefreshGates } from "../lib/workspace"; +import { createTestDaemon, type TestDaemon } from "./test-daemon"; const SLOW_ADAPTER = createSlowFakeAdapter( [ @@ -46,8 +41,6 @@ const SLOW_ADAPTER = createSlowFakeAdapter( 25, ); -const runtime = makeAgentRuntime(SLOW_ADAPTER); - const sleepWorkflow = defineWorkflow("sleep-test", { input: Schema.Struct({}), run: async (ctx) => { @@ -61,15 +54,6 @@ function tmpRoot(): { root: string; finish: () => void } { return { root, finish: () => rmSync(root, { recursive: true, force: true }) }; } -function createTestServices(): DaemonServices { - return { - registry: createRunRegistry(), - pubsub: createPubSub(), - dedupeRegistry: createDedupeRegistry(), - refreshGates: createRefreshGates(), - }; -} - interface Gate { readonly promise: Promise; readonly release: () => void; @@ -90,14 +74,14 @@ function makeGate(): Gate { */ async function gatedWorkspace( root: string, - services: DaemonServices, + daemon: TestDaemon, gate: Gate, ): Promise { const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); const workspaceRoot = join(root, "workspaces"); - services.refreshGates.set(join(workspaceRoot, ".mirror.git"), gate.promise); + daemon.refreshGates.set(join(workspaceRoot, ".mirror.git"), gate.promise); return { workspaceRoot, sshUrl: seed, @@ -141,21 +125,21 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" test("a second start while the first is still reserving is refused atomically, and the slot frees after", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const gate = makeGate(); - const first = startTrackedRun(runtime, db, services, echoWorkflow, { + const first = startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-first", - workspace: await gatedWorkspace(root, services, gate), + workspace: await gatedWorkspace(root, daemon, gate), input: {}, maxConcurrentRuns: 1, }); - await waitFor(() => services.registry.activeRunIds().length === 1); - expect(services.registry.activeRunIds()).toEqual(["run-first"]); - expect(services.registry.getActiveHandle("run-first")).toBeUndefined(); + await waitFor(() => daemon.registry.activeRunIds().length === 1); + expect(daemon.registry.activeRunIds()).toEqual(["run-first"]); + expect(daemon.registry.getActiveHandle("run-first")).toBeUndefined(); - const second = startTrackedRun(runtime, db, services, echoWorkflow, { + const second = startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-second", dir: join(root, "second-dir"), input: {}, @@ -172,8 +156,8 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" gate.release(); const firstRunId = await first; expect(firstRunId).toBe("run-first"); - await waitFor(() => !services.registry.isActive(firstRunId)); - expect(services.registry.activeRunIds()).toEqual([]); + await waitFor(() => !daemon.registry.isActive(firstRunId)); + expect(daemon.registry.activeRunIds()).toEqual([]); db.close(); finish(); @@ -182,18 +166,18 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" test("a second start issued synchronously, before the first is awaited, is refused", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const gate = makeGate(); - const workspace = await gatedWorkspace(root, services, gate); + const workspace = await gatedWorkspace(root, daemon, gate); // No await between the two calls: the first must already hold its slot. - const first = startTrackedRun(runtime, db, services, echoWorkflow, { + const first = startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-sync-first", workspace, input: {}, maxConcurrentRuns: 1, }); - const second = startTrackedRun(runtime, db, services, echoWorkflow, { + const second = startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-sync-second", workspace, input: {}, @@ -201,11 +185,11 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" }); expect(await second.catch((err: unknown) => err)).toBeInstanceOf(ConcurrencyLimitError); - expect(services.registry.activeRunIds()).toEqual(["run-sync-first"]); + expect(daemon.registry.activeRunIds()).toEqual(["run-sync-first"]); gate.release(); const firstRunId = await first; - await waitFor(() => !services.registry.isActive(firstRunId)); + await waitFor(() => !daemon.registry.isActive(firstRunId)); db.close(); finish(); @@ -214,18 +198,18 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" test("a start refused at the limit does not keep its dedupe key", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const gate = makeGate(); - const first = startTrackedRun(runtime, db, services, echoWorkflow, { + const first = startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-holding-slot", - workspace: await gatedWorkspace(root, services, gate), + workspace: await gatedWorkspace(root, daemon, gate), input: {}, maxConcurrentRuns: 1, }); // e.g. a scheduled fire with `overlap: "skip"` while the daemon is full. - const refused = startTrackedRun(runtime, db, services, echoWorkflow, { + const refused = startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-refused", dir: join(root, "refused-dir"), input: {}, @@ -233,11 +217,11 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" dedupeKey: "schedule:nightly", }); expect(await refused.catch((err: unknown) => err)).toBeInstanceOf(ConcurrencyLimitError); - expect(services.dedupeRegistry.holderOf("schedule:nightly")).toBeUndefined(); + expect(daemon.dedupeRegistry.holderOf("schedule:nightly")).toBeUndefined(); gate.release(); const firstRunId = await first; - await waitFor(() => !services.registry.isActive(firstRunId)); + await waitFor(() => !daemon.registry.isActive(firstRunId)); db.close(); finish(); @@ -246,10 +230,10 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" test("a failed allocation releases the reserved slot", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); await expect( - startTrackedRun(runtime, db, services, echoWorkflow, { + startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-doomed", input: {}, maxConcurrentRuns: 1, @@ -262,15 +246,15 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" }), ).rejects.toThrow(/workspace/); - expect(services.registry.activeRunIds()).toEqual([]); + expect(daemon.registry.activeRunIds()).toEqual([]); - const runId = await startTrackedRun(runtime, db, services, echoWorkflow, { + const runId = await startTrackedRun(daemon.runtime, db, echoWorkflow, { dir: join(root, "dir"), input: {}, maxConcurrentRuns: 1, }); - expect(services.registry.activeRunIds()).toEqual([runId]); - await waitFor(() => !services.registry.isActive(runId)); + expect(daemon.registry.activeRunIds()).toEqual([runId]); + await waitFor(() => !daemon.registry.isActive(runId)); db.close(); finish(); @@ -281,34 +265,34 @@ describe("cancel of a reserved-but-not-started run (L1)", () => { test("cancelling during the allocation window defers into run start; the run ends cancelled with no leak", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const gate = makeGate(); - const starting = startTrackedRun(runtime, db, services, sleepWorkflow, { + const starting = startTrackedRun(daemon.runtime, db, sleepWorkflow, { runId: "run-gated", - workspace: await gatedWorkspace(root, services, gate), + workspace: await gatedWorkspace(root, daemon, gate), input: {}, maxConcurrentRuns: 1, }); - await waitFor(() => services.registry.activeRunIds().length === 1); - expect(services.registry.getActiveHandle("run-gated")).toBeUndefined(); + await waitFor(() => daemon.registry.activeRunIds().length === 1); + expect(daemon.registry.getActiveHandle("run-gated")).toBeUndefined(); - const cancelled = services.registry.cancelRegisteredRun("run-gated"); + const cancelled = daemon.registry.cancelRegisteredRun("run-gated"); expect(cancelled).toEqual({ kind: "reserved" }); gate.release(); const runId = await starting; expect(runId).toBe("run-gated"); - await waitFor(() => services.registry.activeRunIds().length === 0); + await waitFor(() => daemon.registry.activeRunIds().length === 0); const events = getRunEvents(db, runId).map((e) => e.payload._tag); expect(events).toContain("RunStarted"); expect(events).toContain("RunCancelled"); - expect(services.registry.cancelRegisteredRun(runId)).toBeUndefined(); + expect(daemon.registry.cancelRegisteredRun(runId)).toBeUndefined(); await Bun.sleep(50); - expect(services.registry.activeRunIds()).toEqual([]); + expect(daemon.registry.activeRunIds()).toEqual([]); db.close(); finish(); @@ -350,15 +334,15 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { const { root, finish } = tmpRoot(); _tmp = root; const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); mkdirSync(join(root, "seed"), { recursive: true }); - const runId = await startTrackedRun(runtime, db, services, failingScratchWorkflow, { + const runId = await startTrackedRun(daemon.runtime, db, failingScratchWorkflow, { runId: "run-scratch-empty", workspace: { ...workspaceSpec(), sshUrl: join(root, "no-such-remote") }, input: {}, }); - await waitFor(() => !services.registry.isActive(runId)); + await waitFor(() => !daemon.registry.isActive(runId)); const dir = join(root, "workspaces", "run-scratch-empty"); expect(existsSync(dir)).toBe(true); @@ -372,24 +356,24 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { const { root, finish } = tmpRoot(); _tmp = root; const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); mkdirSync(join(root, "seed"), { recursive: true }); - const okId = await startTrackedRun(runtime, db, services, scratchWorkflow, { + const okId = await startTrackedRun(daemon.runtime, db, scratchWorkflow, { runId: "run-scratch-ok", workspace: workspaceSpec(), input: {}, }); - await waitFor(() => !services.registry.isActive(okId)); + await waitFor(() => !daemon.registry.isActive(okId)); await new Promise((resolve) => setTimeout(resolve, 50)); // reap lands async expect(existsSync(join(root, "workspaces", "run-scratch-ok"))).toBe(false); - const badId = await startTrackedRun(runtime, db, services, failingScratchWorkflow, { + const badId = await startTrackedRun(daemon.runtime, db, failingScratchWorkflow, { runId: "run-scratch-bad", workspace: workspaceSpec(), input: {}, }); - await waitFor(() => !services.registry.isActive(badId)); + await waitFor(() => !daemon.registry.isActive(badId)); await new Promise((resolve) => setTimeout(resolve, 50)); expect(existsSync(join(root, "workspaces", "run-scratch-bad"))).toBe(true); finish(); @@ -399,7 +383,7 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { const { root, finish } = tmpRoot(); _tmp = root; const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const daemon = createTestDaemon(SLOW_ADAPTER); const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); @@ -420,12 +404,12 @@ describe("scratch workspaces through startTrackedRun (issue #13)", () => { // retention = 1: with the scratch dir correctly excluded, the only clone // survives: the leftover must not count toward retention. - await startTrackedRun(runtime, db, services, echoWorkflow, { + await startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-clone", workspace: { ...workspaceSpec(), sshUrl: seed, retainedWorkspaces: 1 }, input: {}, }); - await waitFor(() => !services.registry.isActive("run-clone")); + await waitFor(() => !daemon.registry.isActive("run-clone")); expect(existsSync(join(root, "workspaces", "run-clone"))).toBe(true); expect(existsSync(join(root, "workspaces", "run-scratch-kept"))).toBe(true); @@ -460,12 +444,12 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { test("an allocated clone workspace is prepared on the allocated dir", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); + const adapter = trackingAdapter(); const seed = join(root, "seed"); await Bun.$`git init -b main -q ${seed}`.quiet(); - const adapter = trackingAdapter(); + const daemon = createTestDaemon(adapter); - const runId = await startTrackedRun(makeAgentRuntime(adapter), db, services, echoWorkflow, { + const runId = await startTrackedRun(daemon.runtime, db, echoWorkflow, { runId: "run-prep-clone", workspace: { workspaceRoot: join(root, "workspaces"), @@ -475,7 +459,7 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { }, input: {}, }); - await waitFor(() => !services.registry.isActive(runId)); + await waitFor(() => !daemon.registry.isActive(runId)); expect(adapter.prepared).toEqual([join(root, "workspaces", "run-prep-clone")]); finish(); @@ -484,10 +468,10 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { test("a scratch workspace is not prepared", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); - const services = createTestServices(); const adapter = trackingAdapter(); + const daemon = createTestDaemon(adapter); - const runId = await startTrackedRun(makeAgentRuntime(adapter), db, services, scratchWorkflow, { + const runId = await startTrackedRun(daemon.runtime, db, scratchWorkflow, { runId: "run-prep-scratch", workspace: { workspaceRoot: join(root, "workspaces"), @@ -497,7 +481,7 @@ describe("adapter.prepareWorkspace through startTrackedRun (#37)", () => { }, input: {}, }); - await waitFor(() => !services.registry.isActive(runId)); + await waitFor(() => !daemon.registry.isActive(runId)); expect(adapter.prepared).toEqual([]); finish(); diff --git a/src/server/runs.ts b/src/server/runs.ts index 4555417..fc1018f 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -12,12 +12,17 @@ * `cancel` that arrives while a run is still reserving is deferred into run * start rather than dropped. * - * #38: the active-run registry, pubsub, and dedupe registry are now - * per-daemon instances passed through `DaemonServices`, so two daemons can - * coexist in one process without sharing state. + * #38 (ADR 0009 §5): the registry is a `Context.Service` (`RunRegistry`) + * whose layer builds a fresh map per daemon runtime, alongside the pubsub, + * dedupe registry and refresh gates. Nothing here holds module-level state: + * `startTrackedRun` resolves the services from the daemon runtime it is + * handed (`server/daemon-runtime.ts`), so two daemons can coexist in one + * process without sharing state. Resolution happens up front, synchronously, + * so the check-then-set below is still one uninterrupted stretch of plain + * code over plain maps. */ -import { Schema, type ManagedRuntime } from "effect"; +import { Context, Layer, Schema } from "effect"; import type { Database } from "bun:sqlite"; import { rm } from "node:fs/promises"; import { admitRun } from "./admission"; @@ -25,11 +30,11 @@ import { appendEvent, getRunEvents, listRuns } from "../persistence/store"; import type { RunRepo } from "../runtime/run"; import { startRun, type RunHandle } from "../runtime/run"; import type { GitIdentity } from "../lib/clone"; -import { allocateWorkspace, type RefreshGates } from "../lib/workspace"; -import type { DedupeRegistry } from "../lib/dedupe"; -import type { AgentRuntime } from "../runtime/agent-runtime"; +import { allocateWorkspace, RefreshGates } from "../lib/workspace"; +import { DedupeRegistry } from "../lib/dedupe"; import type { DispatchChildFn, WorkflowDefinition, WorkspaceKind } from "../workflow"; -import type { PubSub } from "./pubsub"; +import { serviceOf, type DaemonRuntime } from "./daemon-runtime"; +import { RunPubSub } from "./pubsub"; interface ReservedSlot { cancelled: boolean; @@ -74,14 +79,7 @@ export class ConcurrencyLimitError extends Schema.TaggedError; getActiveHandle(runId: string): RunHandle | undefined; @@ -104,7 +102,7 @@ export interface RunRegistry { size: number; } -export function createRunRegistry(): RunRegistry { +export function createRunRegistry(): RunRegistryShape { const active = new Map | ReservedSlot>(); return { isActive: (runId) => active.has(runId), @@ -141,6 +139,18 @@ export function createRunRegistry(): RunRegistry { }; } +/** + * The active-run registry as a daemon service (#38, ADR 0009 §5). Resolve it + * from the daemon runtime's context; provide it with `RunRegistryLayer`. + */ +export class RunRegistry extends Context.Service()("RunRegistry") {} + +/** A fresh, empty registry per layer build — i.e. per daemon runtime. */ +export const RunRegistryLayer: Layer.Layer = Layer.sync( + RunRegistry, + createRunRegistry, +); + export interface WorkspaceSpec { readonly workspaceRoot: string; readonly sshUrl: string; @@ -260,9 +270,8 @@ function dispatchDepth(db: Database, runId: string): number { * parent's. */ async function dispatchChildRun( - runtime: ManagedRuntime.ManagedRuntime, + runtime: DaemonRuntime, db: Database, - services: DaemonServices, env: DispatchEnv, parentRunId: string, child: WorkflowDefinition, @@ -271,7 +280,8 @@ async function dispatchChildRun( ): Promise { const maxDepth = env.maxDispatchDepth ?? DEFAULT_MAX_DISPATCH_DEPTH; const maxChildren = env.maxChildrenPerRun ?? DEFAULT_MAX_CHILDREN_PER_RUN; - const registry = services.registry; + const registry = serviceOf(runtime, RunRegistry); + const dedupeRegistry = serviceOf(runtime, DedupeRegistry); if ( env.maxConcurrentRuns !== undefined && @@ -299,10 +309,10 @@ async function dispatchChildRun( // and *before* the fire-and-forget start, so a collision throws into the // parent here instead of being swallowed by the un-awaited start's catch. const childRunId = `run-${crypto.randomUUID()}`; - if (opts?.dedupeKey !== undefined) services.dedupeRegistry.claim(opts.dedupeKey, childRunId); + if (opts?.dedupeKey !== undefined) dedupeRegistry.claim(opts.dedupeKey, childRunId); void (async () => { - await startTrackedRun(runtime, db, services, child, { + await startTrackedRun(runtime, db, child, { runId: childRunId, ...(env.workspace !== undefined ? { workspace: env.workspace } : {}), ...(env.repo !== undefined ? { repo: env.repo } : {}), @@ -314,7 +324,7 @@ async function dispatchChildRun( ...(opts?.dedupeKey !== undefined ? { dedupeKeyClaimed: true } : {}), }).catch((err: unknown) => { if (opts?.dedupeKey !== undefined) { - services.dedupeRegistry.release(opts.dedupeKey, childRunId); + dedupeRegistry.release(opts.dedupeKey, childRunId); } console.error( `nested run start failed (parent ${parentRunId}, child ${childRunId}):` + @@ -330,11 +340,17 @@ function countDispatchedChildren(db: Database, runId: string): number { return getRunEvents(db, runId).filter((event) => event.payload._tag === "RunDispatched").length; } -/** Starts a run, persists+publishes every event, and tracks it until terminal. */ +/** + * Starts a run, persists+publishes every event, and tracks it until terminal. + * + * `runtime` is the daemon's composition root (#36/#38): the run's agent steps + * resolve their adapter from it, and this function resolves the registry, + * pubsub, dedupe registry and refresh gates from it — synchronously, before + * the admission check, so resolution never opens a gap inside it. + */ export async function startTrackedRun( - runtime: ManagedRuntime.ManagedRuntime, + runtime: DaemonRuntime, db: Database, - services: DaemonServices, workflow: WorkflowDefinition, options: StartTrackedRunOptions, ): Promise { @@ -344,7 +360,10 @@ export async function startTrackedRun( // unrelated runs — one run's SSE watcher can then see the other's terminal event and close // its own db while its real run is still writing to it. const runId = options.runId ?? `run-${crypto.randomUUID()}`; - const registry = services.registry; + const registry = serviceOf(runtime, RunRegistry); + const dedupeRegistry = serviceOf(runtime, DedupeRegistry); + const pubsub = serviceOf(runtime, RunPubSub); + const refreshGates = serviceOf(runtime, RefreshGates); const existing = registry.get(runId); if (existing !== undefined && !isReserved(existing)) { @@ -363,7 +382,7 @@ export async function startTrackedRun( // so two near-simultaneous starts on the same key cannot both slip past. A // collision throws before anything is started, leaving no trace. if (options.dedupeKey !== undefined && options.dedupeKeyClaimed !== true) { - services.dedupeRegistry.claim(options.dedupeKey, runId); + dedupeRegistry.claim(options.dedupeKey, runId); } if (limit !== undefined) registry.reserve(runId); @@ -391,7 +410,7 @@ export async function startTrackedRun( kind, ...(scratchEntries !== undefined ? { scratchEntries } : {}), protectedEntries: [runId, ...registry.activeRunIds()], - refreshGates: services.refreshGates, + refreshGates, })); if (dir === undefined) throw new Error("startTrackedRun needs `dir` or `workspace`"); @@ -407,16 +426,7 @@ export async function startTrackedRun( options.dispatchEnv === undefined ? undefined : (child, input, opts) => - dispatchChildRun( - runtime, - db, - services, - options.dispatchEnv!, - runId, - child, - input, - opts, - ); + dispatchChildRun(runtime, db, options.dispatchEnv!, runId, child, input, opts); const handle = startRun(workflow, runtime, { runId, @@ -433,7 +443,7 @@ export async function startTrackedRun( input: options.input, onEvent: (event) => { appendEvent(db, event); - services.pubsub.publish(runId, event); + pubsub.publish(runId, event); }, }); @@ -453,8 +463,7 @@ export async function startTrackedRun( // releases the run's key. (An interrupted run — process death — is // covered by the registry being per-daemon: a new daemon holds // nothing.) - if (options.dedupeKey !== undefined) - services.dedupeRegistry.release(options.dedupeKey, runId); + if (options.dedupeKey !== undefined) dedupeRegistry.release(options.dedupeKey, runId); }); // Issue #13: a scratch dir is reaped when the run succeeds — there is no @@ -471,7 +480,7 @@ export async function startTrackedRun( return runId; } catch (err) { - if (options.dedupeKey !== undefined) services.dedupeRegistry.release(options.dedupeKey, runId); + if (options.dedupeKey !== undefined) dedupeRegistry.release(options.dedupeKey, runId); const current = registry.get(runId); if (current === undefined || isReserved(current)) registry.delete(runId); throw err; diff --git a/src/server/scheduler.ts b/src/server/scheduler.ts index afc8e8f..79bd710 100644 --- a/src/server/scheduler.ts +++ b/src/server/scheduler.ts @@ -25,23 +25,25 @@ * * #38: the clock comes from Effect's Clock service. Tests use TestClock or * provide a custom Clock instance rather than an injected `now` function. + * In the daemon, `runSchedulerLoop` resolves both the clock and the dedupe + * registry from the context of the daemon runtime it is forked on; the plain + * `tickOnce` receives them as ordinary arguments in `SchedulerDeps`. */ -import { Clock, Cron, Effect, Schedule, Schema, type ManagedRuntime } from "effect"; +import { Clock, Cron, Effect, Schedule, Schema } from "effect"; import type { Database } from "bun:sqlite"; import type { FactoryConfig } from "../config"; -import { DedupeKeyError, type DedupeRegistry } from "../lib/dedupe"; +import { DedupeKeyError, DedupeRegistry, type DedupeRegistryShape } from "../lib/dedupe"; import type { WorkflowDefinition } from "../workflow"; import { ConcurrencyLimitError, DispatchCapError, startTrackedRun, - type DaemonServices, type DispatchEnv, type WorkspaceSpec, } from "./runs"; import { RunCancelledSignal, type RunRepo } from "../runtime/run"; -import type { AgentRuntime } from "../runtime/agent-runtime"; +import type { DaemonRuntime } from "./daemon-runtime"; export class SchedulerError extends Schema.TaggedError()("SchedulerError", { cause: Schema.Defect(), @@ -100,7 +102,7 @@ export interface SchedulerDeps { /** Starts the scheduled run; the daemon wires it to `startTrackedRun`. */ readonly fire: (schedule: RuntimeSchedule) => Promise; /** Issue #38: the daemon's own dedupe registry — where schedule keys are held. */ - readonly dedupeRegistry: DedupeRegistry; + readonly dedupeRegistry: DedupeRegistryShape; /** Issue #38: the wall clock windows are judged against; tests pass a `TestClock`. */ readonly clock: Clock.Clock; } @@ -221,8 +223,31 @@ export async function tickOnce( return results; } -/** The daemon's scheduler loop — `Effect.repeat` around the plain `tickOnce`. */ +/** + * The daemon's scheduler loop — `Effect.repeat` around the plain `tickOnce`. + * The dedupe registry and the clock come from the context it runs in (#38): + * forked on the daemon runtime, that is the daemon's own registry and + * Effect's wall clock. The session state (`lastTick` = now) is taken when the + * loop starts. + */ export function runSchedulerLoop( + schedules: ReadonlyArray, + fire: SchedulerDeps["fire"], + intervalMs: number, +): Effect.Effect { + return Effect.gen(function* () { + const deps: SchedulerDeps = { + schedules, + fire, + dedupeRegistry: yield* DedupeRegistry, + clock: yield* Clock.Clock, + }; + const state = createSchedulerState(deps); + return yield* schedulerTicks(deps, state, intervalMs); + }); +} + +function schedulerTicks( deps: SchedulerDeps, state: SchedulerState, intervalMs: number, @@ -256,8 +281,7 @@ export function runSchedulerLoop( */ export function makeScheduleFire(options: { readonly db: Database; - readonly runtime: ManagedRuntime.ManagedRuntime; - readonly services: DaemonServices; + readonly runtime: DaemonRuntime; readonly maxConcurrentRuns: number; readonly workspace: WorkspaceSpec; readonly repo: RunRepo; @@ -265,7 +289,7 @@ export function makeScheduleFire(options: { }): (schedule: RuntimeSchedule) => Promise { const env = options; return async (schedule: RuntimeSchedule): Promise => { - return startTrackedRun(env.runtime, env.db, env.services, schedule.workflow, { + return startTrackedRun(env.runtime, env.db, schedule.workflow, { input: schedule.input, repo: env.repo, maxConcurrentRuns: options.maxConcurrentRuns, diff --git a/src/server/test-daemon.ts b/src/server/test-daemon.ts new file mode 100644 index 0000000..b93cec8 --- /dev/null +++ b/src/server/test-daemon.ts @@ -0,0 +1,33 @@ +/** + * Test support (#38): a fresh daemon runtime per test, with its per-daemon + * services already resolved for assertions. A new runtime is new state — the + * tests' isolation comes from building one each, not from resetting + * singletons. + */ + +import type { DedupeRegistryShape } from "../lib/dedupe"; +import { DedupeRegistry } from "../lib/dedupe"; +import { RefreshGates, type RefreshGatesShape } from "../lib/workspace"; +import type { AgentAdapter } from "../runtime/agent-adapter"; +import { makeDaemonRuntime, serviceOf, type DaemonRuntime } from "./daemon-runtime"; +import { RunPubSub, type PubSub } from "./pubsub"; +import { RunRegistry, type RunRegistryShape } from "./runs"; + +export interface TestDaemon { + readonly runtime: DaemonRuntime; + readonly registry: RunRegistryShape; + readonly pubsub: PubSub; + readonly dedupeRegistry: DedupeRegistryShape; + readonly refreshGates: RefreshGatesShape; +} + +export function createTestDaemon(adapter?: AgentAdapter): TestDaemon { + const runtime = makeDaemonRuntime(adapter); + return { + runtime, + registry: serviceOf(runtime, RunRegistry), + pubsub: serviceOf(runtime, RunPubSub), + dedupeRegistry: serviceOf(runtime, DedupeRegistry), + refreshGates: serviceOf(runtime, RefreshGates), + }; +} diff --git a/src/server/two-daemons.test.ts b/src/server/two-daemons.test.ts index 1fe4890..82ad6a8 100644 --- a/src/server/two-daemons.test.ts +++ b/src/server/two-daemons.test.ts @@ -17,6 +17,16 @@ import { createSlowFakeAdapter } from "../replay/adapter"; import { startDaemon } from "./daemon"; import { isTerminal, type RunEvent } from "../events"; import type { DaemonHandle } from "./daemon"; +import { serviceOf } from "./daemon-runtime"; +import { RunRegistry } from "./runs"; +import { RunPubSub } from "./pubsub"; +import { DedupeRegistry } from "../lib/dedupe"; +import { RefreshGates } from "../lib/workspace"; + +/** Each daemon's own services, resolved from its own runtime's context. */ +const registryOf = (daemon: DaemonHandle) => serviceOf(daemon.runtime, RunRegistry); +const pubsubOf = (daemon: DaemonHandle) => serviceOf(daemon.runtime, RunPubSub); +const dedupeOf = (daemon: DaemonHandle) => serviceOf(daemon.runtime, DedupeRegistry); let releaseRuns: () => void = () => undefined; let runsGate = Promise.resolve(); @@ -59,8 +69,8 @@ function watch( ): { readonly own: Array; readonly foreign: Array; readonly stop: () => void } { const own: Array = []; const foreign: Array = []; - const unsubOwn = owner.services.pubsub.subscribe(runId, (event) => own.push(event)); - const unsubOther = other.services.pubsub.subscribe(runId, (event) => foreign.push(event)); + const unsubOwn = pubsubOf(owner).subscribe(runId, (event) => own.push(event)); + const unsubOther = pubsubOf(other).subscribe(runId, (event) => foreign.push(event)); return { own, foreign, @@ -119,6 +129,15 @@ describe("two daemons in one process (#38)", () => { }); try { + // Two runtimes, two builds of every per-daemon layer: no service + // instance is shared. + expect(registryOf(daemon1)).not.toBe(registryOf(daemon2)); + expect(pubsubOf(daemon1)).not.toBe(pubsubOf(daemon2)); + expect(dedupeOf(daemon1)).not.toBe(dedupeOf(daemon2)); + expect(serviceOf(daemon1.runtime, RefreshGates)).not.toBe( + serviceOf(daemon2.runtime, RefreshGates), + ); + closeGate(); const port1 = daemon1.server.port; const port2 = daemon2.server.port; @@ -145,8 +164,8 @@ describe("two daemons in one process (#38)", () => { const watch1 = watch(daemon1, daemon2, runId1); const watch2 = watch(daemon2, daemon1, runId2); - expect(daemon1.services.registry.activeRunIds()).toEqual([runId1]); - expect(daemon2.services.registry.activeRunIds()).toEqual([runId2]); + expect(registryOf(daemon1).activeRunIds()).toEqual([runId1]); + expect(registryOf(daemon2).activeRunIds()).toEqual([runId2]); const list1 = (await fetch(`http://localhost:${port1}/api/runs`).then((r) => r.json(), @@ -178,8 +197,8 @@ describe("two daemons in one process (#38)", () => { watch1.stop(); watch2.stop(); - expect(daemon1.services.registry.activeRunIds()).toEqual([]); - expect(daemon2.services.registry.activeRunIds()).toEqual([]); + expect(registryOf(daemon1).activeRunIds()).toEqual([]); + expect(registryOf(daemon2).activeRunIds()).toEqual([]); const dedupeKey = "shared-key"; closeGate(); @@ -192,8 +211,8 @@ describe("two daemons in one process (#38)", () => { // Daemon 1 still holds the key (run 3 is gated) when daemon 2 is asked // for the same key — and daemon 2 knows nothing of it. - expect(daemon1.services.dedupeRegistry.holderOf(dedupeKey)).toBe(runId3); - expect(daemon2.services.dedupeRegistry.holderOf(dedupeKey)).toBeUndefined(); + expect(dedupeOf(daemon1).holderOf(dedupeKey)).toBe(runId3); + expect(dedupeOf(daemon2).holderOf(dedupeKey)).toBeUndefined(); const res4 = await fetch(`http://localhost:${port2}/api/runs`, { method: "POST", @@ -203,8 +222,8 @@ describe("two daemons in one process (#38)", () => { const { runId: runId4 } = (await res4.json()) as { runId: string }; expect(runId3).not.toBe(runId4); - expect(daemon1.services.dedupeRegistry.holderOf(dedupeKey)).toBe(runId3); - expect(daemon2.services.dedupeRegistry.holderOf(dedupeKey)).toBe(runId4); + expect(dedupeOf(daemon1).holderOf(dedupeKey)).toBe(runId3); + expect(dedupeOf(daemon2).holderOf(dedupeKey)).toBe(runId4); releaseRuns(); await waitForTerminal(port1, runId3); From 6c44b3379c97860aa9e11532a873572e8f2ef6f5 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 10:45:45 +0000 Subject: [PATCH 21/26] feat(daemon): cancel active runs on shutdown DaemonHandle.stop() (and so factory serve's SIGINT/SIGTERM handler) now shuts the run registry down before the runtime goes away: it refuses new starts (DaemonShuttingDownError, 503 over HTTP), cancels every running handle, marks every reserved slot for cancel-on-start, and waits up to shutdownTimeoutMs (default 5s) for the runs to settle. Runs therefore persist RunCancelled and their ctx.exec children are killed, instead of being orphaned and read back as interrupted after a restart. Order: interrupt the scheduler, shut the registry down while the daemon runtime is still live, stop the HTTP server (so SSE tails see the cancellations), dispose the runtime. stop() is idempotent. The registry layer's finalizer runs the same shutdown as a safety net when a runtime is disposed without stop(). Every start now reserves its registry slot, not only admission-limited ones, so a run still allocating is visible to shutdown either way. Refs #38 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/cli-commands.ts | 8 +- src/server/daemon.ts | 42 +++++-- src/server/http.ts | 10 ++ src/server/runs.ts | 106 ++++++++++++++++- src/server/shutdown.test.ts | 219 ++++++++++++++++++++++++++++++++++++ 5 files changed, 369 insertions(+), 16 deletions(-) create mode 100644 src/server/shutdown.test.ts diff --git a/src/cli-commands.ts b/src/cli-commands.ts index b1d2b95..0e03f67 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -83,8 +83,12 @@ export const serveCommand = Command.make("serve", serveConfig, (config) => : "scheduler: none (no schedules in config)", ), ); - // The daemon's shutdown path: stop the scheduler and server and dispose - // the agent runtime before exiting, instead of dying with them live. + // The daemon's shutdown path: stop the scheduler, cancel every active run + // and wait (bounded) for each to record `RunCancelled` and kill its + // `ctx.exec` children, then stop the server and dispose the runtime — + // instead of dying with them live and leaving orphaned processes and + // "interrupted" runs behind. `once`: a second signal falls through to the + // default handler and kills the process outright. yield* Effect.sync(() => { const shutdown = (signal: NodeJS.Signals) => { void handle.stop().finally(() => process.exit(signal === "SIGINT" ? 130 : 143)); diff --git a/src/server/daemon.ts b/src/server/daemon.ts index 94bf6a3..8d45111 100644 --- a/src/server/daemon.ts +++ b/src/server/daemon.ts @@ -17,6 +17,13 @@ * state, so two daemons are two runtimes and coexist in one process without * sharing anything. The scheduler loop is forked on it and resolves its * dedupe registry and clock from its context. + * + * Shutdown order (`DaemonHandle.stop`, which `factory serve`'s SIGINT/SIGTERM + * handler calls): interrupt the scheduler so nothing new fires; shut the run + * registry down — refuse new starts, cancel every active run and reserved + * slot, and wait (bounded) for them to persist `RunCancelled` and kill their + * `ctx.exec` children; only then stop the HTTP server (so live SSE tails see + * the cancellations) and dispose the runtime. */ import { mkdir } from "node:fs/promises"; @@ -26,9 +33,14 @@ type AnyFiber = Fiber.Fiber; import type { FactoryConfig } from "../config"; import { openStore } from "../persistence/store"; import { serve } from "./http"; -import type { DispatchEnv, WorkspaceSpec } from "./runs"; +import { + DEFAULT_SHUTDOWN_TIMEOUT_MS, + RunRegistry, + type DispatchEnv, + type WorkspaceSpec, +} from "./runs"; import { makeScheduleFire, runSchedulerLoop, toRuntimeSchedules } from "./scheduler"; -import { makeDaemonRuntime, type DaemonRuntime } from "./daemon-runtime"; +import { makeDaemonRuntime, serviceOf, type DaemonRuntime } from "./daemon-runtime"; export interface DaemonOptions { readonly dbPath: string; @@ -42,6 +54,11 @@ export interface DaemonOptions { * Absent, the legacy per-request `{dir, clone}` behaviour is kept. */ readonly config?: FactoryConfig; + /** + * How long `stop()` waits for cancelled runs to settle before abandoning + * them, over `DEFAULT_SHUTDOWN_TIMEOUT_MS`. + */ + readonly shutdownTimeoutMs?: number; } export interface DaemonHandle { @@ -55,8 +72,10 @@ export interface DaemonHandle { */ readonly runtime: DaemonRuntime; /** - * Shut the daemon down: interrupt the scheduler, stop the HTTP server, and - * dispose the agent runtime (issue #36). + * Shut the daemon down: interrupt the scheduler, cancel every active run + * and wait (bounded) for them to settle, stop the HTTP server, and dispose + * the daemon runtime (issue #36). Idempotent: every call returns the first + * call's promise. */ readonly stop: () => Promise; } @@ -114,11 +133,16 @@ export async function startDaemon(options: DaemonOptions): Promise })() : undefined; - const stop = async (): Promise => { - if (schedulerFiber !== undefined) await Effect.runPromise(Fiber.interrupt(schedulerFiber)); - await server.stop(true); - await runtime.dispose(); - }; + let stopping: Promise | undefined; + const stop = (): Promise => + (stopping ??= (async () => { + if (schedulerFiber !== undefined) await Effect.runPromise(Fiber.interrupt(schedulerFiber)); + await serviceOf(runtime, RunRegistry).shutdown( + options.shutdownTimeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS, + ); + await server.stop(true); + await runtime.dispose(); + })()); return { server, schedulerFiber, runtime, stop }; } diff --git a/src/server/http.ts b/src/server/http.ts index 27bb1db..c41829f 100644 --- a/src/server/http.ts +++ b/src/server/http.ts @@ -46,6 +46,7 @@ import { serviceOf, type DaemonRuntime } from "./daemon-runtime"; import { RunPubSub, type PubSub } from "./pubsub"; import { ConcurrencyLimitError, + DaemonShuttingDownError, RunRegistry, startTrackedRun, type DispatchEnv, @@ -419,6 +420,9 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise if (err instanceof ConcurrencyLimitError) { return json({ error: err.message }, { status: 409 }); } + if (err instanceof DaemonShuttingDownError) { + return json({ error: err.message }, { status: 503 }); + } // Issue #15: the key collision surfaces as a conflict that names both // the key and the run holding it, so a client can see exactly whom it // raced with. @@ -488,6 +492,9 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise if (err instanceof ConcurrencyLimitError) { return json({ error: err.message }, { status: 409 }); } + if (err instanceof DaemonShuttingDownError) { + return json({ error: err.message }, { status: 503 }); + } if (err instanceof DedupeKeyError) { return json( { error: err.message, dedupeKey: err.key, holderRunId: err.holderRunId }, @@ -588,6 +595,9 @@ export function createHandler(options: ServerOptions): (req: Request) => Promise if (err instanceof ConcurrencyLimitError) { return json({ error: err.message }, { status: 409 }); } + if (err instanceof DaemonShuttingDownError) { + return json({ error: err.message }, { status: 503 }); + } if (err instanceof DedupeKeyError) { return json( { error: err.message, dedupeKey: err.key, holderRunId: err.holderRunId }, diff --git a/src/server/runs.ts b/src/server/runs.ts index fc1018f..ba6c2d2 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -12,6 +12,16 @@ * `cancel` that arrives while a run is still reserving is deferred into run * start rather than dropped. * + * Shutdown cancels what the registry holds: `RunRegistryShape.shutdown` + * refuses further starts, cancels every running handle, marks every reserved + * slot cancelled (so the run is cancelled the moment it starts, exactly like + * a deferred API cancel), and waits — bounded — for all of them to settle. + * Settled means each run persisted its `RunCancelled` and its `ctx.exec` + * children were killed, rather than being orphaned by process exit and read + * back as "interrupted" after a restart. `DaemonHandle.stop` runs it while + * the daemon runtime is still live; the registry layer's finalizer runs it + * again as a safety net when the runtime is disposed without `stop`. + * * #38 (ADR 0009 §5): the registry is a `Context.Service` (`RunRegistry`) * whose layer builds a fresh map per daemon runtime, alongside the pubsub, * dedupe registry and refresh gates. Nothing here holds module-level state: @@ -22,7 +32,7 @@ * code over plain maps. */ -import { Context, Layer, Schema } from "effect"; +import { Context, Effect, Layer, Schema } from "effect"; import type { Database } from "bun:sqlite"; import { rm } from "node:fs/promises"; import { admitRun } from "./admission"; @@ -57,6 +67,30 @@ function isReserved(entry: RunHandle | ReservedSlot | undefined): boole return entry !== undefined && !("result" in entry) && "cancelled" in entry; } +/** + * A start refused because the daemon is shutting down — its registry stopped + * admitting runs. HTTP maps it to 503; a `ctx.dispatch` surfaces it to the + * parent, which is itself being cancelled. + */ +export class DaemonShuttingDownError extends Schema.TaggedError()( + "DaemonShuttingDownError", + { message: Schema.String }, +) { + // Same `.stack` header fix as `ConcurrencyLimitError` below. + static { + this.prototype.name = "DaemonShuttingDownError"; + } + + static of(): DaemonShuttingDownError { + return new DaemonShuttingDownError({ + message: "the daemon is shutting down and no longer starts runs", + }); + } +} + +/** How long shutdown waits for cancelled runs to settle before giving up on them. */ +export const DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000; + export class ConcurrencyLimitError extends Schema.TaggedError()( "ConcurrencyLimitError", { maxConcurrentRuns: Schema.Number, message: Schema.String }, @@ -96,6 +130,16 @@ export interface RunRegistryShape { | { readonly kind: "reserved" } | undefined; reserve(runId: string): void; + /** True once `shutdown` has begun: starts are refused from then on. */ + readonly closed: boolean; + /** + * Close admission, cancel every run held — running handles directly, + * reserved slots by marking them for cancel-on-start — and resolve once the + * registry is empty or `timeoutMs` has passed (runs still held then are + * logged and abandoned). Idempotent: every call returns the first call's + * promise. + */ + shutdown(timeoutMs: number): Promise; setHandle(runId: string, handle: RunHandle): void; delete(runId: string): void; get(runId: string): RunHandle | ReservedSlot | undefined; @@ -104,6 +148,12 @@ export interface RunRegistryShape { export function createRunRegistry(): RunRegistryShape { const active = new Map | ReservedSlot>(); + let closed = false; + let shutdownPromise: Promise | undefined; + const onEmpty: Array<() => void> = []; + const notifyIfEmpty = (): void => { + if (active.size === 0) for (const resolve of onEmpty.splice(0)) resolve(); + }; return { isActive: (runId) => active.has(runId), activeRunIds: () => [...active.keys()], @@ -129,6 +179,36 @@ export function createRunRegistry(): RunRegistryShape { }, delete(runId) { active.delete(runId); + notifyIfEmpty(); + }, + get closed() { + return closed; + }, + shutdown(timeoutMs) { + if (shutdownPromise !== undefined) return shutdownPromise; + closed = true; + for (const entry of active.values()) { + if (isReserved(entry)) (entry as ReservedSlot).cancelled = true; + else void (entry as RunHandle).cancel(); + } + const drained = new Promise<"drained">((resolve) => { + onEmpty.push(() => resolve("drained")); + notifyIfEmpty(); + }); + let timer: ReturnType | undefined; + const timedOut = new Promise<"timeout">((resolve) => { + timer = setTimeout(() => resolve("timeout"), timeoutMs); + }); + shutdownPromise = Promise.race([drained, timedOut]).then((outcome) => { + clearTimeout(timer); + if (outcome === "timeout") { + console.error( + `[daemon] shutdown: ${active.size} run(s) did not settle within ${timeoutMs}ms ` + + `and are abandoned: ${[...active.keys()].join(", ")}`, + ); + } + }); + return shutdownPromise; }, get(runId) { return active.get(runId); @@ -145,10 +225,19 @@ export function createRunRegistry(): RunRegistryShape { */ export class RunRegistry extends Context.Service()("RunRegistry") {} -/** A fresh, empty registry per layer build — i.e. per daemon runtime. */ -export const RunRegistryLayer: Layer.Layer = Layer.sync( +/** + * A fresh, empty registry per layer build — i.e. per daemon runtime. Its + * finalizer shuts the registry down, so disposing a daemon runtime never + * orphans a run. It is the safety net, not the main path: by the time it + * runs the runtime is already disposed, so a reserved run that starts during + * it cannot resolve its adapter (`DaemonHandle.stop` shuts the registry down + * first, while the runtime is live, and this is then a no-op). + */ +export const RunRegistryLayer: Layer.Layer = Layer.effect( RunRegistry, - createRunRegistry, + Effect.acquireRelease(Effect.sync(createRunRegistry), (registry) => + Effect.promise(() => registry.shutdown(DEFAULT_SHUTDOWN_TIMEOUT_MS)), + ), ); export interface WorkspaceSpec { @@ -283,6 +372,7 @@ async function dispatchChildRun( const registry = serviceOf(runtime, RunRegistry); const dedupeRegistry = serviceOf(runtime, DedupeRegistry); + if (registry.closed) throw DaemonShuttingDownError.of(); if ( env.maxConcurrentRuns !== undefined && !admitRun(env.maxConcurrentRuns, registry.activeRunIds().length) @@ -365,6 +455,10 @@ export async function startTrackedRun( const pubsub = serviceOf(runtime, RunPubSub); const refreshGates = serviceOf(runtime, RefreshGates); + // Shutdown closed admission: nothing new starts, so nothing escapes the + // cancel-and-wait. + if (registry.closed) throw DaemonShuttingDownError.of(); + const existing = registry.get(runId); if (existing !== undefined && !isReserved(existing)) { throw new Error(`run ${runId} is already active`); @@ -384,7 +478,9 @@ export async function startTrackedRun( if (options.dedupeKey !== undefined && options.dedupeKeyClaimed !== true) { dedupeRegistry.claim(options.dedupeKey, runId); } - if (limit !== undefined) registry.reserve(runId); + // Every run reserves — not only limited ones — so shutdown sees runs that + // are still allocating whether or not an admission limit applies. + if (existing === undefined) registry.reserve(runId); try { // Issue #13: a scratch workspace takes its kind from the workflow and diff --git a/src/server/shutdown.test.ts b/src/server/shutdown.test.ts new file mode 100644 index 0000000..6dab7ff --- /dev/null +++ b/src/server/shutdown.test.ts @@ -0,0 +1,219 @@ +/** + * Shutdown cancels active runs (#38): `DaemonHandle.stop` — the path + * `factory serve`'s SIGINT/SIGTERM handler takes — cancels every run the + * daemon holds and waits for it to settle before the runtime goes away, so a + * run ends as a recorded `RunCancelled` with its `ctx.exec` child killed, + * rather than as an orphaned process and an "interrupted" run after restart. + */ + +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "bun:test"; +import { Schema } from "effect"; +import { defineConfig } from "../config"; +import { RefreshGates } from "../lib/workspace"; +import { getRunEvents, openStore } from "../persistence/store"; +import { createSlowFakeAdapter } from "../replay/adapter"; +import { defineWorkflow } from "../workflow"; +import { startDaemon, type DaemonHandle } from "./daemon"; +import { makeDaemonRuntime, serviceOf } from "./daemon-runtime"; +import { RunRegistry, startTrackedRun } from "./runs"; + +/** Writes its shell's pid, then becomes a 300s sleep — a child that outlives any test. */ +const longExec = defineWorkflow("shutdown-long-exec", { + input: Schema.Struct({ pidFile: Schema.String }), + workspace: { kind: "scratch" }, + run: async (ctx, input) => { + await ctx.exec(["sh", "-c", `echo $$ > ${input.pidFile}; exec sleep 300`]); + return {}; + }, +}); + +/** The same, on a clone workspace, so its allocation can be held open. */ +const cloneLongExec = defineWorkflow("shutdown-clone-long-exec", { + input: Schema.Struct({}), + run: async (ctx) => { + await ctx.exec(["sleep", "300"]); + return {}; + }, +}); + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await Bun.sleep(20); + } + throw new Error("condition not met within timeout"); +} + +function tagsOf(dbPath: string, runId: string): Array { + const db = openStore(dbPath); + try { + return getRunEvents(db, runId).map((event) => event.payload._tag); + } finally { + db.close(); + } +} + +async function makeDaemon(root: string, sshUrl = join(root, "no-remote")): Promise { + return startDaemon({ + dbPath: join(root, "factory.db"), + port: 0, + config: defineConfig({ + agent: { adapter: createSlowFakeAdapter([], 1) }, + repo: { + sshUrl, + identity: { name: "Factory", email: "factory@factory.test" }, + baseBranch: "main", + slug: "acme/widgets", + }, + workflows: [longExec, cloneLongExec], + workspaceRoot: join(root, "workspaces"), + retainedWorkspaces: 10, + }), + }); +} + +async function post(handle: DaemonHandle, body: unknown): Promise { + return fetch(`http://localhost:${handle.server.port}/api/runs`, { + method: "POST", + body: JSON.stringify(body), + }); +} + +describe("daemon shutdown cancels active runs (#38)", () => { + test("stop() cancels a running exec: RunCancelled is recorded and the child is gone", async () => { + const root = mkdtempSync(join(tmpdir(), "factory-shutdown-exec-")); + const pidFile = join(root, "child.pid"); + const handle = await makeDaemon(root); + try { + const res = await post(handle, { workflowId: "shutdown-long-exec", input: { pidFile } }); + expect(res.status).toBe(201); + const { runId } = (await res.json()) as { runId: string }; + + await waitFor(() => existsSync(pidFile) && readFileSync(pidFile, "utf8").trim() !== ""); + const pid = Number(readFileSync(pidFile, "utf8").trim()); + expect(isAlive(pid)).toBe(true); + + await handle.stop(); + + expect(tagsOf(join(root, "factory.db"), runId).at(-1)).toBe("RunCancelled"); + expect(isAlive(pid)).toBe(false); + } finally { + await handle.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); + + test("stop() cancels a reserved, allocation-bound run cleanly and refuses new starts meanwhile", async () => { + const root = mkdtempSync(join(tmpdir(), "factory-shutdown-reserved-")); + const seed = join(root, "seed"); + await Bun.$`git init -b main -q ${seed}`.quiet(); + await Bun.$`git -C ${seed} -c user.name=seed -c user.email=seed@seed.local commit -q --allow-empty -m seed`.quiet(); + const handle = await makeDaemon(root, seed); + + // Hold allocation open: the mirror's refresh queues behind this gate, so + // the run is a reserved slot and nothing more until it is released. + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + serviceOf(handle.runtime, RefreshGates).set(join(root, "workspaces", ".mirror.git"), gate); + const registry = serviceOf(handle.runtime, RunRegistry); + + try { + // The POST itself is held by the allocation, so it is not awaited here. + const starting = post(handle, { workflowId: "shutdown-clone-long-exec", input: {} }); + await waitFor(() => registry.activeRunIds().length === 1); + const runId = registry.activeRunIds()[0]!; + expect(registry.getActiveHandle(runId)).toBeUndefined(); + + let stopped = false; + const stopping = handle.stop().then(() => { + stopped = true; + }); + + // Shutdown is waiting on the reserved run, and admits nothing new. + await Bun.sleep(50); + expect(stopped).toBe(false); + const refused = await post(handle, { workflowId: "shutdown-clone-long-exec", input: {} }); + expect(refused.status).toBe(503); + + release(); + await stopping; + expect((await starting).status).toBe(201); + + expect(registry.activeRunIds()).toEqual([]); + const tags = tagsOf(join(root, "factory.db"), runId); + expect(tags[0]).toBe("RunStarted"); + expect(tags.at(-1)).toBe("RunCancelled"); + expect(tags).not.toContain("ExecStarted"); + } finally { + release(); + await handle.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); + + test("stop() is idempotent", async () => { + const root = mkdtempSync(join(tmpdir(), "factory-shutdown-idem-")); + const pidFile = join(root, "child.pid"); + const handle = await makeDaemon(root); + try { + const res = await post(handle, { workflowId: "shutdown-long-exec", input: { pidFile } }); + const { runId } = (await res.json()) as { runId: string }; + await waitFor(() => existsSync(pidFile) && readFileSync(pidFile, "utf8").trim() !== ""); + + const first = handle.stop(); + const second = handle.stop(); + expect(second).toBe(first); + await Promise.all([first, second]); + await handle.stop(); + + const tags = tagsOf(join(root, "factory.db"), runId); + expect(tags.filter((tag) => tag === "RunCancelled")).toHaveLength(1); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("disposing a daemon runtime without stop() still cancels its runs (layer finalizer)", async () => { + const root = mkdtempSync(join(tmpdir(), "factory-shutdown-dispose-")); + const pidFile = join(root, "child.pid"); + const dbPath = join(root, "factory.db"); + const db = openStore(dbPath); + const runtime = makeDaemonRuntime(createSlowFakeAdapter([], 1)); + try { + const runId = await startTrackedRun(runtime, db, longExec, { + workspace: { + workspaceRoot: join(root, "workspaces"), + sshUrl: join(root, "no-remote"), + identity: { name: "T", email: "t@t.test" }, + retainedWorkspaces: 10, + }, + input: { pidFile }, + }); + await waitFor(() => existsSync(pidFile) && readFileSync(pidFile, "utf8").trim() !== ""); + const pid = Number(readFileSync(pidFile, "utf8").trim()); + + await runtime.dispose(); + + expect(getRunEvents(db, runId).at(-1)?.payload._tag).toBe("RunCancelled"); + expect(isAlive(pid)).toBe(false); + } finally { + db.close(); + rmSync(root, { recursive: true, force: true }); + } + }); +}); From a7839f6fcdd15db305f5c0b93e75c77c63751540 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 10:59:25 +0000 Subject: [PATCH 22/26] fix(runtime): end an abandoned agent stream at once Stream.fromAsyncIterable closes the adapter iterable with a scope finalizer that awaits iter.return(), and an async generator's return() queues behind its pending next(). Interrupting a step that was waiting on the model therefore blocked until opencode's next chunk, and the onInterrupt abort that would have hurried it along only ran after that finalizer: a POST /cancel took ~20s to record RunCancelled, and shutdown blew its budget. abortableIterable wraps the adapter stream: next() races the abort signal, and return() aborts, starts the adapter's own return() without awaiting it, and resolves immediately. The adapter teardown (killing the opencode process) still runs; AgentStepHandle.teardown exposes it, RunHandle.settled waits for every abandoned step's teardown after the result, and registry shutdown now waits for settled rather than only for the registry to empty, so stop() does not exit under a live agent. Refs #38 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/runtime/agent-step.test.ts | 45 +++++++++++++- src/runtime/agent-step.ts | 98 ++++++++++++++++++++++++++++-- src/runtime/run.ts | 10 ++++ src/server/daemon.ts | 5 +- src/server/runs.ts | 52 ++++++++++------ src/server/shutdown.test.ts | 105 +++++++++++++++++++++++++++++++++ 6 files changed, 288 insertions(+), 27 deletions(-) diff --git a/src/runtime/agent-step.test.ts b/src/runtime/agent-step.test.ts index 9afe22e..eac7f0c 100644 --- a/src/runtime/agent-step.test.ts +++ b/src/runtime/agent-step.test.ts @@ -2,7 +2,7 @@ import { Effect, ManagedRuntime } from "effect"; import { describe, expect, test } from "bun:test"; import type { AgentAdapterYield, AgentSignal } from "./agent-adapter"; import { AgentRuntimeLayer } from "./agent-runtime"; -import { buildAgentStepEffect } from "./agent-step"; +import { abortableIterable, buildAgentStepEffect } from "./agent-step"; function makeYield(chunk: unknown, signal?: AgentSignal): AgentAdapterYield { return signal !== undefined ? { chunk, signal } : { chunk }; @@ -169,3 +169,46 @@ describe("buildAgentStepEffect signal extraction", () => { expect(outcome.chunkCount).toBe(1); }); }); + +describe("abortableIterable (#38: abandoning a step is immediate)", () => { + test("a stream that ends on its own is passed through and never aborted", async () => { + const controller = new AbortController(); + async function* source() { + yield 1; + yield 2; + } + const { iterable, teardown } = abortableIterable(source(), controller); + const seen: Array = []; + for await (const n of iterable) seen.push(n); + await teardown(); + expect(seen).toEqual([1, 2]); + expect(controller.signal.aborted).toBe(false); + }); + + test("return() during a pending next() resolves at once, aborts, and tears down in the background", async () => { + const controller = new AbortController(); + let finalized = false; + async function* source() { + try { + yield 1; + // Waiting on the model: only the abort lets this generator move on. + await new Promise((resolve) => + controller.signal.addEventListener("abort", () => setTimeout(resolve, 50)), + ); + } finally { + finalized = true; + } + } + const { iterable, teardown } = abortableIterable(source(), controller); + const iterator = iterable[Symbol.asyncIterator](); + expect(await iterator.next()).toEqual({ done: false, value: 1 }); + const pending = iterator.next(); + + expect(await iterator.return!()).toEqual({ done: true, value: undefined }); + expect(controller.signal.aborted).toBe(true); + expect(await pending).toEqual({ done: true, value: undefined }); + expect(finalized).toBe(false); + await teardown(); + expect(finalized).toBe(true); + }); +}); diff --git a/src/runtime/agent-step.ts b/src/runtime/agent-step.ts index 5c9dfb1..149124b 100644 --- a/src/runtime/agent-step.ts +++ b/src/runtime/agent-step.ts @@ -19,6 +19,19 @@ * wiring even though `Stream.fromAsyncIterable`'s implicit `.return()` on * scope closure was sufficient in the tested case — cheap, never harmful, * covers the untested non-cooperative-abort case. + * + * WHY THE ADAPTER STREAM IS WRAPPED (`abortableIterable`, #38 shutdown): the + * implicit `.return()` above is a scope finalizer, and an async generator's + * `return()` queues behind its pending `next()`. Interrupting a step that was + * waiting on the model therefore blocked until opencode's *next* chunk — and + * the `onInterrupt` abort, which runs after that finalizer, never got the + * chance to hurry it along. A cancel took tens of seconds, and a shutdown + * outlived its budget. The wrapper makes the stream end the moment the step + * is abandoned: `next()` races the abort signal, and `return()` aborts first + * (an early `return()` *is* the consumer giving up), fires the adapter's own + * `return()` without awaiting it, and resolves at once. The adapter's + * teardown — killing the opencode process — still runs, in the background, + * and `AgentStepHandle.teardown` lets the caller wait for it. */ import { Effect, Schema, Stream } from "effect"; @@ -96,14 +109,84 @@ export interface AgentStepHandle { readonly effect: Effect.Effect; readonly abortController: AbortController; readonly partial: AgentStepPartial; + /** + * Settles once the adapter has torn its stream down after the step was + * abandoned (e.g. the opencode process is gone) — at once when it never + * was. The step itself does not wait for this; shutdown does. + */ + readonly teardown: () => Promise; +} + +/** + * Wrap an adapter's stream so abandoning it is immediate (see the module + * comment): `next()` resolves `done` as soon as `abortController` aborts, and + * `return()` aborts (unless the stream already finished), starts the inner + * `return()` without waiting for it, and resolves at once. `teardown()` + * settles when that inner `return()` does — i.e. when the adapter has + * actually cleaned up — or at once if the stream was never abandoned. + */ +export function abortableIterable( + iterable: AsyncIterable, + abortController: AbortController, +): { readonly iterable: AsyncIterable; readonly teardown: () => Promise } { + let teardown: Promise = Promise.resolve(); + const done: IteratorReturnResult = { done: true, value: undefined }; + const wrapped: AsyncIterable = { + [Symbol.asyncIterator](): AsyncIterator { + const inner = iterable[Symbol.asyncIterator](); + const signal = abortController.signal; + const aborted = new Promise>((resolve) => { + if (signal.aborted) resolve(done); + else signal.addEventListener("abort", () => resolve(done), { once: true }); + }); + // `innerDone`: the adapter's own stream ended — nothing to tear down. + // `released`: we abandoned it, and its `return()` is already started. + let innerDone = false; + let released = false; + const release = (): void => { + if (innerDone || released) return; + released = true; + abortController.abort(); + if (inner.return !== undefined) { + teardown = Promise.resolve(inner.return()).then( + () => undefined, + () => undefined, + ); + } + }; + return { + next: async () => { + if (innerDone || released || signal.aborted) { + release(); + return done; + } + const result = await Promise.race([ + inner.next().then((r) => { + if (r.done === true) innerDone = true; + return r; + }), + aborted, + ]); + if (result.done === true) release(); + return result; + }, + return: async () => { + release(); + return done; + }, + }; + }, + }; + return { iterable: wrapped, teardown: () => teardown }; } export function buildAgentStepEffect(options: AgentStepEffectOptions): AgentStepHandle { const abortController = new AbortController(); + let teardown: () => Promise = () => Promise.resolve(); const rawStream = Stream.unwrap( - Effect.map(Effect.service(AgentRuntime), ({ adapter }) => - Stream.fromAsyncIterable( + Effect.map(Effect.service(AgentRuntime), ({ adapter }) => { + const wrapped = abortableIterable( adapter.stream({ threadId: options.threadId, dir: options.dir, @@ -112,9 +195,14 @@ export function buildAgentStepEffect(options: AgentStepEffectOptions): AgentStep outputSchema: options.outputSchema, abortController, }), + abortController, + ); + teardown = wrapped.teardown; + return Stream.fromAsyncIterable( + wrapped.iterable, (cause) => new AgentStepChunkError({ cause }), - ), - ), + ); + }), ); const partial: AgentStepPartial = { @@ -198,5 +286,5 @@ export function buildAgentStepEffect(options: AgentStepEffectOptions): AgentStep durationMs: Date.now() - startedAt, })); - return { effect, abortController, partial }; + return { effect, abortController, partial, teardown: () => teardown() }; } diff --git a/src/runtime/run.ts b/src/runtime/run.ts index d6d05fd..a403000 100644 --- a/src/runtime/run.ts +++ b/src/runtime/run.ts @@ -111,6 +111,13 @@ export type RunOutcome = export interface RunHandle { readonly result: Promise>; + /** + * Resolves after `result`, once every agent step the run abandoned has + * finished tearing its adapter down (e.g. the opencode process is gone). + * A cancelled run records `RunCancelled` without waiting for that; daemon + * shutdown waits for this so it does not exit under a live agent process. + */ + readonly settled: Promise; cancel(): Promise; } @@ -173,6 +180,7 @@ export function startRun( const runController = new AbortController(); let cancelled = false; let activeAgentFiber: Fiber.Fiber | null = null; + const agentTeardowns: Array> = []; runController.signal.addEventListener("abort", () => { cancelled = true; @@ -257,6 +265,7 @@ export function startRun( activeAgentFiber = fiber; const exit = await Effect.runPromise(Fiber.await(fiber)); activeAgentFiber = null; + agentTeardowns.push(handle.teardown()); const durationMs = Date.now() - stepStartedAt; if (Exit.isFailure(exit)) { @@ -538,6 +547,7 @@ export function startRun( return { result: resultPromise, + settled: resultPromise.then(() => Promise.all(agentTeardowns)).then(() => undefined), cancel: async () => { runController.abort(); await resultPromise; diff --git a/src/server/daemon.ts b/src/server/daemon.ts index 8d45111..190d3cc 100644 --- a/src/server/daemon.ts +++ b/src/server/daemon.ts @@ -137,9 +137,8 @@ export async function startDaemon(options: DaemonOptions): Promise const stop = (): Promise => (stopping ??= (async () => { if (schedulerFiber !== undefined) await Effect.runPromise(Fiber.interrupt(schedulerFiber)); - await serviceOf(runtime, RunRegistry).shutdown( - options.shutdownTimeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS, - ); + const registry = serviceOf(runtime, RunRegistry); + await registry.shutdown(options.shutdownTimeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS); await server.stop(true); await runtime.dispose(); })()); diff --git a/src/server/runs.ts b/src/server/runs.ts index ba6c2d2..da0f7b9 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -135,9 +135,9 @@ export interface RunRegistryShape { /** * Close admission, cancel every run held — running handles directly, * reserved slots by marking them for cancel-on-start — and resolve once the - * registry is empty or `timeoutMs` has passed (runs still held then are - * logged and abandoned). Idempotent: every call returns the first call's - * promise. + * registry is empty and every cancelled run has `settled` (its agent + * processes torn down), or `timeoutMs` has passed (runs still pending then + * are logged). Idempotent: every call returns the first call's promise. */ shutdown(timeoutMs: number): Promise; setHandle(runId: string, handle: RunHandle): void; @@ -151,6 +151,25 @@ export function createRunRegistry(): RunRegistryShape { let closed = false; let shutdownPromise: Promise | undefined; const onEmpty: Array<() => void> = []; + // The runs shutdown cancelled (or that started after it began), so it can + // wait for their `settled`, not merely for the registry to empty. + const cancelled: Array> = []; + const allSettled = (): Promise => + new Promise((resolve) => { + onEmpty.push(resolve); + notifyIfEmpty(); + }) + .then(() => Promise.allSettled(cancelled.map((handle) => handle.settled))) + .then(() => undefined); + const within = async (promise: Promise, timeoutMs: number): Promise => { + let timer: ReturnType | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + }); + const outcome = await Promise.race([promise.then(() => true as const), timedOut]); + clearTimeout(timer); + return outcome; + }; const notifyIfEmpty = (): void => { if (active.size === 0) for (const resolve of onEmpty.splice(0)) resolve(); }; @@ -176,6 +195,7 @@ export function createRunRegistry(): RunRegistryShape { }, setHandle(runId, handle) { active.set(runId, handle); + if (closed) cancelled.push(handle); }, delete(runId) { active.delete(runId); @@ -188,23 +208,19 @@ export function createRunRegistry(): RunRegistryShape { if (shutdownPromise !== undefined) return shutdownPromise; closed = true; for (const entry of active.values()) { - if (isReserved(entry)) (entry as ReservedSlot).cancelled = true; - else void (entry as RunHandle).cancel(); + if (isReserved(entry)) { + (entry as ReservedSlot).cancelled = true; + } else { + const handle = entry as RunHandle; + cancelled.push(handle); + void handle.cancel(); + } } - const drained = new Promise<"drained">((resolve) => { - onEmpty.push(() => resolve("drained")); - notifyIfEmpty(); - }); - let timer: ReturnType | undefined; - const timedOut = new Promise<"timeout">((resolve) => { - timer = setTimeout(() => resolve("timeout"), timeoutMs); - }); - shutdownPromise = Promise.race([drained, timedOut]).then((outcome) => { - clearTimeout(timer); - if (outcome === "timeout") { + shutdownPromise = within(allSettled(), timeoutMs).then((settled) => { + if (!settled) { console.error( - `[daemon] shutdown: ${active.size} run(s) did not settle within ${timeoutMs}ms ` + - `and are abandoned: ${[...active.keys()].join(", ")}`, + `[daemon] shutdown: run(s) did not settle within ${timeoutMs}ms: ` + + `${[...active.keys()].join(", ") || "(agent teardown pending)"}`, ); } }); diff --git a/src/server/shutdown.test.ts b/src/server/shutdown.test.ts index 6dab7ff..560b7e7 100644 --- a/src/server/shutdown.test.ts +++ b/src/server/shutdown.test.ts @@ -15,6 +15,10 @@ import { defineConfig } from "../config"; import { RefreshGates } from "../lib/workspace"; import { getRunEvents, openStore } from "../persistence/store"; import { createSlowFakeAdapter } from "../replay/adapter"; +import type { AgentAdapter, AgentAdapterYield } from "../runtime/agent-adapter"; +import { makeAgentRuntime } from "../runtime/agent-runtime"; +import { startRun } from "../runtime/run"; +import type { RunEvent } from "../events"; import { defineWorkflow } from "../workflow"; import { startDaemon, type DaemonHandle } from "./daemon"; import { makeDaemonRuntime, serviceOf } from "./daemon-runtime"; @@ -39,6 +43,51 @@ const cloneLongExec = defineWorkflow("shutdown-clone-long-exec", { }, }); +/** One agent step and nothing else: the run is mid-step until cancelled. */ +const agentStep = defineWorkflow("shutdown-agent-step", { + input: Schema.Struct({}), + workspace: { kind: "scratch" }, + run: async (ctx) => { + await ctx.agent("think", "waiting on the model"); + return {}; + }, +}); + +/** + * An adapter waiting on the model, like opencode between chunks: its + * generator's `next()` is pending until the step's abort reaches it, then + * takes `abortLatencyMs` to wind down (the HTTP `session.abort()` round trip) + * before its `finally` — the process kill — runs. Before #38's fix the + * stream's implicit `return()` queued behind that `next()`, and the abort that + * would release it only fired after `return()` finished: a deadlock until the + * next chunk. `cooperative: false` never ends at all, abort or not. + */ +function waitingAdapter(options: { + readonly cooperative: boolean; + readonly abortLatencyMs?: number; +}): AgentAdapter & { + readonly tornDown: () => boolean; +} { + let tornDown = false; + return { + tornDown: () => tornDown, + async prepareWorkspace(): Promise {}, + async *stream({ abortController }): AsyncIterable { + try { + yield { chunk: { type: "RUN_STARTED" } }; + await new Promise((resolve) => { + if (!options.cooperative) return; + abortController.signal.addEventListener("abort", () => + setTimeout(resolve, options.abortLatencyMs ?? 0), + ); + }); + } finally { + tornDown = true; + } + }, + }; +} + function isAlive(pid: number): boolean { try { process.kill(pid, 0); @@ -216,4 +265,60 @@ describe("daemon shutdown cancels active runs (#38)", () => { rmSync(root, { recursive: true, force: true }); } }); + + test("a cancel mid-agent-step records RunCancelled promptly even if the adapter never yields again", async () => { + const adapter = waitingAdapter({ cooperative: false }); + const events: Array = []; + const handle = startRun(agentStep, makeAgentRuntime(adapter), { + runId: "run-blocked-agent", + dir: tmpdir(), + input: {}, + onEvent: (event) => events.push(event), + }); + await waitFor(() => events.some((e) => e.payload._tag === "AgentChunk")); + + const started = Date.now(); + await handle.cancel(); + expect(Date.now() - started).toBeLessThan(500); + expect(events.at(-1)?.payload._tag).toBe("RunCancelled"); + expect( + events.find((e) => e.payload._tag === "AgentStepFinished")?.payload as { outcome?: string }, + ).toMatchObject({ outcome: "cancelled" }); + }); + + test("stop() mid-agent-step finishes well within budget and the adapter is torn down", async () => { + const root = mkdtempSync(join(tmpdir(), "factory-shutdown-agent-")); + const adapter = waitingAdapter({ cooperative: true, abortLatencyMs: 100 }); + const handle = await startDaemon({ + dbPath: join(root, "factory.db"), + port: 0, + config: defineConfig({ + agent: { adapter }, + repo: { + sshUrl: join(root, "no-remote"), + identity: { name: "Factory", email: "factory@factory.test" }, + baseBranch: "main", + slug: "acme/widgets", + }, + workflows: [agentStep], + workspaceRoot: join(root, "workspaces"), + retainedWorkspaces: 10, + }), + }); + try { + const res = await post(handle, { workflowId: "shutdown-agent-step", input: {} }); + const { runId } = (await res.json()) as { runId: string }; + await waitFor(() => tagsOf(join(root, "factory.db"), runId).includes("AgentChunk")); + + const started = Date.now(); + await handle.stop(); + expect(Date.now() - started).toBeLessThan(1_000); + expect(tagsOf(join(root, "factory.db"), runId).at(-1)).toBe("RunCancelled"); + // stop() waited for the adapter's own teardown, not just the run's end. + expect(adapter.tornDown()).toBe(true); + } finally { + await handle.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); }); From 1e725cb032e97e4a224888a77a8f70f0fb8e3223 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 10:59:34 +0000 Subject: [PATCH 23/26] fix(daemon): let cancelled runs record their end after a timed-out shutdown When the shutdown wait times out, stop() used to resolve as soon as the runtime was disposed, and factory serve's process.exit could land between AgentStepFinished{cancelled} and RunCancelled, so the run read back as interrupted. After disposal (which interrupts whatever still held the runs), stop() now waits a further bounded 2s for the cancelled runs to settle via RunRegistry.awaitCancelled. Refs #38 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/server/daemon.ts | 11 ++++++++++- src/server/runs.ts | 10 ++++++++++ src/server/shutdown.test.ts | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/server/daemon.ts b/src/server/daemon.ts index 190d3cc..1a7fa2d 100644 --- a/src/server/daemon.ts +++ b/src/server/daemon.ts @@ -23,7 +23,9 @@ * registry down — refuse new starts, cancel every active run and reserved * slot, and wait (bounded) for them to persist `RunCancelled` and kill their * `ctx.exec` children; only then stop the HTTP server (so live SSE tails see - * the cancellations) and dispose the runtime. + * the cancellations) and dispose the runtime; if the shutdown wait timed + * out, wait a further bounded moment for the cancelled runs to record their + * end, so the caller's `process.exit` does not cut off a `RunCancelled`. */ import { mkdir } from "node:fs/promises"; @@ -80,6 +82,9 @@ export interface DaemonHandle { readonly stop: () => Promise; } +/** After disposal, how long `stop()` still waits for cancelled runs to record their end. */ +const SETTLE_AFTER_DISPOSE_MS = 2_000; + /** Issue #16: how often the scheduler's due window check runs. */ export const DEFAULT_SCHEDULER_INTERVAL_MS = 30_000; @@ -141,6 +146,10 @@ export async function startDaemon(options: DaemonOptions): Promise await registry.shutdown(options.shutdownTimeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS); await server.stop(true); await runtime.dispose(); + // If shutdown timed out, disposing interrupted whatever still held the + // runs; give them a short, bounded moment to persist `RunCancelled` + // before the caller (`factory serve`) exits the process. + await registry.awaitCancelled(SETTLE_AFTER_DISPOSE_MS); })()); return { server, schedulerFiber, runtime, stop }; diff --git a/src/server/runs.ts b/src/server/runs.ts index da0f7b9..19ea8ec 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -140,6 +140,13 @@ export interface RunRegistryShape { * are logged). Idempotent: every call returns the first call's promise. */ shutdown(timeoutMs: number): Promise; + /** + * After a timed-out `shutdown`: wait up to `timeoutMs` more for the runs it + * cancelled to settle — e.g. once disposing the runtime has interrupted + * what was still holding them — so their final `RunCancelled` lands before + * the process exits. Resolves at once when nothing is pending. + */ + awaitCancelled(timeoutMs: number): Promise; setHandle(runId: string, handle: RunHandle): void; delete(runId: string): void; get(runId: string): RunHandle | ReservedSlot | undefined; @@ -226,6 +233,9 @@ export function createRunRegistry(): RunRegistryShape { }); return shutdownPromise; }, + async awaitCancelled(timeoutMs) { + await within(allSettled(), timeoutMs); + }, get(runId) { return active.get(runId); }, diff --git a/src/server/shutdown.test.ts b/src/server/shutdown.test.ts index 560b7e7..48da70c 100644 --- a/src/server/shutdown.test.ts +++ b/src/server/shutdown.test.ts @@ -321,4 +321,39 @@ describe("daemon shutdown cancels active runs (#38)", () => { rmSync(root, { recursive: true, force: true }); } }); + + test("stop() is bounded even when an adapter never tears down", async () => { + const root = mkdtempSync(join(tmpdir(), "factory-shutdown-stuck-")); + const handle = await startDaemon({ + dbPath: join(root, "factory.db"), + port: 0, + shutdownTimeoutMs: 200, + config: defineConfig({ + agent: { adapter: waitingAdapter({ cooperative: false }) }, + repo: { + sshUrl: join(root, "no-remote"), + identity: { name: "Factory", email: "factory@factory.test" }, + baseBranch: "main", + slug: "acme/widgets", + }, + workflows: [agentStep], + workspaceRoot: join(root, "workspaces"), + retainedWorkspaces: 10, + }), + }); + try { + const res = await post(handle, { workflowId: "shutdown-agent-step", input: {} }); + const { runId } = (await res.json()) as { runId: string }; + await waitFor(() => tagsOf(join(root, "factory.db"), runId).includes("AgentChunk")); + + const started = Date.now(); + await handle.stop(); + // 200ms shutdown budget + the bounded post-dispose settle wait. + expect(Date.now() - started).toBeLessThan(3_000); + expect(tagsOf(join(root, "factory.db"), runId).at(-1)).toBe("RunCancelled"); + } finally { + await handle.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); }); From c2cde064a7bafce966fcf2e6efd2a83b725042b7 Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 11:00:24 +0000 Subject: [PATCH 24/26] fix(runs): refuse a start whose run id is still reserving Since every start reserves its slot, a reserved entry for the same run id can only be a concurrent start with the same explicit runId, and that path skipped admission and started a duplicate run. Any existing entry, running or reserved, now throws "already active". Refs #38 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/server/runs.test.ts | 32 ++++++++++++++++++++++++++++++++ src/server/runs.ts | 10 ++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/server/runs.test.ts b/src/server/runs.test.ts index 3591f0b..a0b870f 100644 --- a/src/server/runs.test.ts +++ b/src/server/runs.test.ts @@ -195,6 +195,38 @@ describe("startTrackedRun admission (M1: the slot is reserved before any await)" finish(); }); + test("a second start with the same runId while the first is still reserving is refused", async () => { + const { root, finish } = tmpRoot(); + const db = openStore(join(root, "factory.db")); + const daemon = createTestDaemon(SLOW_ADAPTER); + const gate = makeGate(); + const workspace = await gatedWorkspace(root, daemon, gate); + + // No limit, so admission cannot be what refuses it: the reserved slot is. + const first = startTrackedRun(daemon.runtime, db, echoWorkflow, { + runId: "run-same-id", + workspace, + input: {}, + }); + const second = startTrackedRun(daemon.runtime, db, echoWorkflow, { + runId: "run-same-id", + workspace, + input: {}, + }); + + const err = await second.catch((e: unknown) => e); + expect(err).toBeInstanceOf(Error); + expect((err as Error).message).toContain("already active"); + expect(daemon.registry.activeRunIds()).toEqual(["run-same-id"]); + + gate.release(); + await first; + await waitFor(() => !daemon.registry.isActive("run-same-id")); + + db.close(); + finish(); + }); + test("a start refused at the limit does not keep its dedupe key", async () => { const { root, finish } = tmpRoot(); const db = openStore(join(root, "factory.db")); diff --git a/src/server/runs.ts b/src/server/runs.ts index 19ea8ec..935f1c6 100644 --- a/src/server/runs.ts +++ b/src/server/runs.ts @@ -485,15 +485,17 @@ export async function startTrackedRun( // cancel-and-wait. if (registry.closed) throw DaemonShuttingDownError.of(); - const existing = registry.get(runId); - if (existing !== undefined && !isReserved(existing)) { + // Running or still reserving alike: a run id is started once. (A reserved + // entry here can only be a concurrent start with the same explicit runId, + // which must not slip past admission and start a duplicate.) + if (registry.get(runId) !== undefined) { throw new Error(`run ${runId} is already active`); } // D29 admission is decided before the dedupe claim, so a refused start // throws having claimed nothing — otherwise its key would stay held by a // run that never started (and a `"skip"` schedule would skip forever). - const limit = existing === undefined ? options.maxConcurrentRuns : undefined; + const limit = options.maxConcurrentRuns; if (limit !== undefined && !admitRun(limit, registry.size)) { throw ConcurrencyLimitError.of({ maxConcurrentRuns: limit }); } @@ -506,7 +508,7 @@ export async function startTrackedRun( } // Every run reserves — not only limited ones — so shutdown sees runs that // are still allocating whether or not an admission limit applies. - if (existing === undefined) registry.reserve(runId); + registry.reserve(runId); try { // Issue #13: a scratch workspace takes its kind from the workflow and From e21d37ef0513ca98f3715cc5db0ce31633b00e9d Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 11:00:24 +0000 Subject: [PATCH 25/26] fix(daemon): build the daemon runtime's layers at startup serviceOf resolves with runSync, which only works over an already-built or synchronously buildable context. startDaemon now awaits runtime.context() before serving, so a failing (or future asynchronous) layer surfaces at startup rather than on the first request, and every later lookup reads the built context. createTestDaemon already builds eagerly by resolving its services at construction; documented. Refs #38 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/server/daemon-runtime.ts | 4 +++- src/server/daemon.ts | 3 +++ src/server/test-daemon.ts | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/server/daemon-runtime.ts b/src/server/daemon-runtime.ts index e12d26b..f1fde25 100644 --- a/src/server/daemon-runtime.ts +++ b/src/server/daemon-runtime.ts @@ -46,7 +46,9 @@ export const makeDaemonRuntime = (adapter?: AgentAdapter): DaemonRuntime => /** * Resolve one of the daemon's services from its runtime, synchronously — the - * plain-async boundary's way into the context. + * plain-async boundary's way into the context. Owners build the layers eagerly + * (`startDaemon` awaits `runtime.context()`; `createTestDaemon` resolves at + * construction), so this reads an already-built context. */ export function serviceOf( runtime: DaemonRuntime, diff --git a/src/server/daemon.ts b/src/server/daemon.ts index 1a7fa2d..31fb20b 100644 --- a/src/server/daemon.ts +++ b/src/server/daemon.ts @@ -93,6 +93,9 @@ export async function startDaemon(options: DaemonOptions): Promise const db = openStore(options.dbPath); const runtime = makeDaemonRuntime(options.config?.agent.adapter); + // Build the layers now: a failing (or, one day, asynchronous) layer + // surfaces at startup, and every later `serviceOf` reads the built context. + await runtime.context(); const server = serve({ db, diff --git a/src/server/test-daemon.ts b/src/server/test-daemon.ts index b93cec8..695d994 100644 --- a/src/server/test-daemon.ts +++ b/src/server/test-daemon.ts @@ -2,7 +2,9 @@ * Test support (#38): a fresh daemon runtime per test, with its per-daemon * services already resolved for assertions. A new runtime is new state — the * tests' isolation comes from building one each, not from resetting - * singletons. + * singletons. Resolving the services below builds the runtime's layers + * eagerly and synchronously, so a layer that cannot build that way fails here, + * at construction, not inside a test's first `serviceOf`. */ import type { DedupeRegistryShape } from "../lib/dedupe"; From 40478a8b165aac3fd254d32f54340043934752da Mon Sep 17 00:00:00 2001 From: FreshlyBrewedCode Date: Fri, 2 Oct 2026 11:05:59 +0000 Subject: [PATCH 26/26] fix(runtime): bound how long a run waits for adapter teardown Measured live against opencode: after an abort, @tanstack/ai's chat engine can sit on the aborted stream indefinitely, so the adapter's generator teardown never settles, even though the abort has already killed the opencode process (the local-process sandbox kills the process group on the spawn's abort signal). Waiting on that teardown made every shutdown with an agent run use its whole budget. RunHandle.settled now waits at most AGENT_TEARDOWN_GRACE_MS (1s) per abandoned step. Refs #38 Co-Authored-By: Claude Opus 5.5 (1M context) --- src/runtime/agent-step.ts | 6 ++++-- src/runtime/run.ts | 26 ++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/runtime/agent-step.ts b/src/runtime/agent-step.ts index 149124b..4df2d64 100644 --- a/src/runtime/agent-step.ts +++ b/src/runtime/agent-step.ts @@ -30,8 +30,10 @@ * is abandoned: `next()` races the abort signal, and `return()` aborts first * (an early `return()` *is* the consumer giving up), fires the adapter's own * `return()` without awaiting it, and resolves at once. The adapter's - * teardown — killing the opencode process — still runs, in the background, - * and `AgentStepHandle.teardown` lets the caller wait for it. + * teardown still runs, in the background, and `AgentStepHandle.teardown` + * lets the caller wait for it. (For opencode the process itself dies on the + * abort: the local-process sandbox kills its process group on the spawn's + * abort signal, independently of the generator's `finally`.) */ import { Effect, Schema, Stream } from "effect"; diff --git a/src/runtime/run.ts b/src/runtime/run.ts index a403000..ad5f5eb 100644 --- a/src/runtime/run.ts +++ b/src/runtime/run.ts @@ -113,7 +113,8 @@ export interface RunHandle { readonly result: Promise>; /** * Resolves after `result`, once every agent step the run abandoned has - * finished tearing its adapter down (e.g. the opencode process is gone). + * finished tearing its adapter down — each bounded by + * `AGENT_TEARDOWN_GRACE_MS`. * A cancelled run records `RunCancelled` without waiting for that; daemon * shutdown waits for this so it does not exit under a live agent process. */ @@ -121,6 +122,27 @@ export interface RunHandle { cancel(): Promise; } +/** + * How long a run's `settled` waits for one abandoned agent step's adapter + * teardown. Bounded because that teardown is third-party code that need not + * finish: measured live, `@tanstack/ai`'s chat engine can sit on an aborted + * opencode stream indefinitely, even though the abort itself already killed + * the opencode process (the local-process sandbox kills the process group on + * the spawn's abort signal). The grace covers a cooperative adapter's + * cleanup without letting an uncooperative one hold shutdown to its budget. + */ +export const AGENT_TEARDOWN_GRACE_MS = 1_000; + +function boundedTeardown(teardown: Promise): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, AGENT_TEARDOWN_GRACE_MS); + void teardown.then(() => { + clearTimeout(timer); + resolve(); + }); + }); +} + function makeIdCounter(prefix: string): () => string { let n = 0; return () => `${prefix}-${n++}`; @@ -265,7 +287,7 @@ export function startRun( activeAgentFiber = fiber; const exit = await Effect.runPromise(Fiber.await(fiber)); activeAgentFiber = null; - agentTeardowns.push(handle.teardown()); + agentTeardowns.push(boundedTeardown(handle.teardown())); const durationMs = Date.now() - stepStartedAt; if (Exit.isFailure(exit)) {