diff --git a/packages/core/src/budget.ts b/packages/core/src/budget.ts index 9769a01..8aa6059 100644 --- a/packages/core/src/budget.ts +++ b/packages/core/src/budget.ts @@ -143,7 +143,7 @@ export function nextChannel(current: ChannelId): ChannelId | null { /* ------------------------------------------------------------------ */ export interface LoopGuardOptions { - /** Identical action signatures in a row before we stop. */ + /** Turns in a row with identical actions and identical output before we stop. */ repeatLimit?: number; /** Turns with no tool output change before we stop. */ stallLimit?: number; @@ -182,13 +182,17 @@ export class LoopGuard { } observe(signature: string, output: string): LoopVerdict { - if (signature !== "" && signature === this.lastSignature) this.repeats++; + // A repeat needs the same result as well as the same call. A resumable job + // or a poll is driven by repeating one command whose output moves on; that + // is progress, and stopping it ended long sessions that still had work to do. + const sameOutput = output === this.lastOutput; + if (signature !== "" && signature === this.lastSignature && sameOutput) this.repeats++; else { this.lastSignature = signature; this.repeats = 1; } - if (output === this.lastOutput && output !== "") this.stalls++; + if (sameOutput && output !== "") this.stalls++; else { this.lastOutput = output; this.stalls = 0; diff --git a/packages/core/test/loop.test.ts b/packages/core/test/loop.test.ts index 3b9a6f8..51eaf47 100644 --- a/packages/core/test/loop.test.ts +++ b/packages/core/test/loop.test.ts @@ -13,6 +13,7 @@ import { CORE_TOOLS } from "@motifcode/tools"; import { FaultTransport, ScriptedTransport, doneBody, toolCallBody } from "@motifcode/replay"; import { EmptyTaskError, + LoopGuard, TransportError, runLoop, resetIds, @@ -595,6 +596,39 @@ describe("loop detection", () => { expect(r.reason).toBe("loop_detected"); expect(kinds(events, "loop_detected")).toHaveLength(1); }); + + it("keeps going when the same action makes progress", async () => { + // A resumable fit is driven by one command, run again until it says done. + // Two long benchmark sessions were stopped at its third call, though each + // call had run new evaluations. + const { events, emit } = collect(); + const step = toolCallBody("bash", { command: "python3 fit.py step spec.json --budget 600" }); + const transport = new ScriptedTransport([ + step, step, step, step, step, + doneBody("fit converged"), + doneBody("fit converged", { confirm: true }), + ]); + let calls = 0; + const executor: Executor = { + run: async () => ({ ok: true, output: `e00${++calls} chi2 ${(60 / calls).toFixed(1)}` }), + }; + const r = await runLoop({ ...base, transport, executor, emit }); + expect(r.reason).toBe("done"); + expect(calls).toBe(5); + expect(kinds(events, "loop_detected")).toHaveLength(0); + }); + + it("counts a repeat only when the output repeats too", () => { + const sig = (command: string) => LoopGuard.signature([{ name: "bash", arguments: { command } }]); + const stuck = new LoopGuard(); + expect([1, 2, 3].map(() => stuck.observe(sig("make"), "error: x").tripped)).toEqual([false, false, true]); + const moving = new LoopGuard(); + expect([1, 2, 3, 4, 5, 6].some((i) => moving.observe(sig("make"), `step ${i}`).tripped)).toBe(false); + // Different calls that keep producing the same output are still a stall. + const stalled = new LoopGuard(); + expect([1, 2, 3, 4, 5].map((i) => stalled.observe(sig(`try ${i}`), "no such file").tripped)) + .toEqual([false, false, false, false, true]); + }); }); describe("server death", () => {