Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions packages/core/src/budget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ export function nextChannel(current: ChannelId): ChannelId | null {
/* ------------------------------------------------------------------ */

export interface LoopGuardOptions {
/** Identical action signatures in a row before we stop. */
/** Turns in a row with identical actions and identical output before we stop. */
repeatLimit?: number;
/** Turns with no tool output change before we stop. */
stallLimit?: number;
Expand Down Expand Up @@ -182,13 +182,17 @@ export class LoopGuard {
}

observe(signature: string, output: string): LoopVerdict {
if (signature !== "" && signature === this.lastSignature) this.repeats++;
// A repeat needs the same result as well as the same call. A resumable job
// or a poll is driven by repeating one command whose output moves on; that
// is progress, and stopping it ended long sessions that still had work to do.
const sameOutput = output === this.lastOutput;
if (signature !== "" && signature === this.lastSignature && sameOutput) this.repeats++;
else {
this.lastSignature = signature;
this.repeats = 1;
}

if (output === this.lastOutput && output !== "") this.stalls++;
if (sameOutput && output !== "") this.stalls++;
else {
this.lastOutput = output;
this.stalls = 0;
Expand Down
34 changes: 34 additions & 0 deletions packages/core/test/loop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { CORE_TOOLS } from "@motifcode/tools";
import { FaultTransport, ScriptedTransport, doneBody, toolCallBody } from "@motifcode/replay";
import {
EmptyTaskError,
LoopGuard,
TransportError,
runLoop,
resetIds,
Expand Down Expand Up @@ -595,6 +596,39 @@ describe("loop detection", () => {
expect(r.reason).toBe("loop_detected");
expect(kinds(events, "loop_detected")).toHaveLength(1);
});

it("keeps going when the same action makes progress", async () => {
// A resumable fit is driven by one command, run again until it says done.
// Two long benchmark sessions were stopped at its third call, though each
// call had run new evaluations.
const { events, emit } = collect();
const step = toolCallBody("bash", { command: "python3 fit.py step spec.json --budget 600" });
const transport = new ScriptedTransport([
step, step, step, step, step,
doneBody("fit converged"),
doneBody("fit converged", { confirm: true }),
]);
let calls = 0;
const executor: Executor = {
run: async () => ({ ok: true, output: `e00${++calls} chi2 ${(60 / calls).toFixed(1)}` }),
};
const r = await runLoop({ ...base, transport, executor, emit });
expect(r.reason).toBe("done");
expect(calls).toBe(5);
expect(kinds(events, "loop_detected")).toHaveLength(0);
});

it("counts a repeat only when the output repeats too", () => {
const sig = (command: string) => LoopGuard.signature([{ name: "bash", arguments: { command } }]);
const stuck = new LoopGuard();
expect([1, 2, 3].map(() => stuck.observe(sig("make"), "error: x").tripped)).toEqual([false, false, true]);
const moving = new LoopGuard();
expect([1, 2, 3, 4, 5, 6].some((i) => moving.observe(sig("make"), `step ${i}`).tripped)).toBe(false);
// Different calls that keep producing the same output are still a stall.
const stalled = new LoopGuard();
expect([1, 2, 3, 4, 5].map((i) => stalled.observe(sig(`try ${i}`), "no such file").tripped))
.toEqual([false, false, false, false, true]);
});
});

describe("server death", () => {
Expand Down
Loading