Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,9 @@ playwright-report/
*.ndjson

# ...except the recorded corpora promoted to committed test fixtures. D3's
# event type is designed against these (ADR 0003) and phase 1's corpus-replay
# adapter reads them, so they have to survive a fresh checkout. 128K total.
# event type is designed against the opencode ones (ADR 0003), and the
# corpus-replay adapter reads the ACP ones (`scripts/record-corpus.ts`), so
# they have to survive a fresh checkout.
!test/corpus/*.ndjson

# playwright-cli session snapshots
Expand All @@ -58,6 +59,4 @@ playwright-report/
# per-machine Tailscale cert for the Vite dev server (`tailscale cert`)
.certs/

# tanstack sandbox projected state
.tanstack-projected-*
spikes/out/
8 changes: 0 additions & 8 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

192 changes: 104 additions & 88 deletions e2e/implement-issue.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
/**
* Phase 1 exit criterion, second half: the same workflow that ran live
* against opencode (see `docs/findings/`, PR #4 on `factory-spike`) also
* runs green under the corpus-replay adapter in `bun test`.
* Phase 1 exit criterion, second half: the same workflow that ran live also
* runs green under the corpus-replay adapter in `bun test` — once per agent,
* replaying the run `scripts/record-corpus.ts` recorded through the ACP
* adapter.
*
* The replay adapter only replays recorded *chunks* (ADR — `src/replay/
* adapter.ts`'s docstring); it never actually invokes a tool, so it cannot
Expand Down Expand Up @@ -30,7 +31,19 @@ import { makeAgentRuntime } from "../src/runtime/agent-runtime";
import { startRun } from "../src/runtime/run";
import implementIssue from "./implement-issue";

const FULL_ROUND_TRIP_CORPUS = `${import.meta.dir}/../test/corpus/run-1789308170212.ndjson`;
/** Each agent's recorded run, and the branch its pr-metadata step named. */
const CORPORA: Array<{ agent: string; path: string; branch: string }> = [
{
agent: "claude",
path: `${import.meta.dir}/../test/corpus/acp-claude-implement-issue.ndjson`,
branch: "factory/issue-1-add-slugify",
},
{
agent: "opencode",
path: `${import.meta.dir}/../test/corpus/acp-opencode-implement-issue.ndjson`,
branch: "factory/issue-1-add-slugify-helper",
},
];

const GREET_ONLY_INDEX = `/** Returns a friendly greeting for the given name. */
export function greet(name: string): string {
Expand Down Expand Up @@ -84,88 +97,91 @@ async function git(dir: string, args: ReadonlyArray<string>): Promise<void> {
}
}

describe("implement-issue workflow, replayed against the recorded round-trip corpus", () => {
test("agent steps replay, exec/assert/write-back run for real against a local git fixture", async () => {
const root = mkdtempSync(join(tmpdir(), "factory-implement-issue-test-"));
const remoteDir = join(root, "remote.git");
const workDir = join(root, "work");
const binDir = join(root, "bin");
const originalPath = process.env.PATH;

try {
await hostExec(["git", "init", "--bare", remoteDir]);
await hostExec(["git", "init", workDir]);
await git(workDir, ["symbolic-ref", "HEAD", "refs/heads/main"]);
await git(workDir, ["config", "user.name", "Factory Test"]);
await git(workDir, ["config", "user.email", "factory-test@example.com"]);

mkdirSync(join(workDir, "src"), { recursive: true });
writeFileSync(join(workDir, "src/index.ts"), GREET_ONLY_INDEX);
writeFileSync(join(workDir, "src/index.test.ts"), GREET_ONLY_TEST);
await git(workDir, ["add", "-A"]);
await git(workDir, ["commit", "-m", "seed: greet only"]);
await git(workDir, ["remote", "add", "origin", remoteDir]);
await git(workDir, ["push", "-u", "origin", "main"]);

// Simulate "the implement/fix steps already edited the tree" — the
// corpus replay itself cannot, since it only replays chunks.
writeFileSync(join(workDir, "src/index.ts"), FINAL_INDEX);
writeFileSync(join(workDir, "src/index.test.ts"), FINAL_TEST);

mkdirSync(binDir, { recursive: true });
const fakeGhPath = join(binDir, "gh");
writeFileSync(fakeGhPath, FAKE_GH_SCRIPT);
chmodSync(fakeGhPath, 0o755);
process.env.PATH = `${binDir}:${originalPath}`;

const events: Array<unknown> = [];
const handle = startRun(
implementIssue,
makeAgentRuntime(fakeAgents(createCorpusReplayAdapter(FULL_ROUND_TRIP_CORPUS))),
{
runId: "test-run-corpus-replay",
dir: workDir,
input: {
issueNumber: 1,
describe.each(CORPORA)(
"implement-issue workflow, replayed from $agent's recorded run",
(corpus) => {
test("agent steps replay, exec/assert/write-back run for real against a local git fixture", async () => {
const root = mkdtempSync(join(tmpdir(), "factory-implement-issue-test-"));
const remoteDir = join(root, "remote.git");
const workDir = join(root, "work");
const binDir = join(root, "bin");
const originalPath = process.env.PATH;

try {
await hostExec(["git", "init", "--bare", remoteDir]);
await hostExec(["git", "init", workDir]);
await git(workDir, ["symbolic-ref", "HEAD", "refs/heads/main"]);
await git(workDir, ["config", "user.name", "Factory Test"]);
await git(workDir, ["config", "user.email", "factory-test@example.com"]);

mkdirSync(join(workDir, "src"), { recursive: true });
writeFileSync(join(workDir, "src/index.ts"), GREET_ONLY_INDEX);
writeFileSync(join(workDir, "src/index.test.ts"), GREET_ONLY_TEST);
await git(workDir, ["add", "-A"]);
await git(workDir, ["commit", "-m", "seed: greet only"]);
await git(workDir, ["remote", "add", "origin", remoteDir]);
await git(workDir, ["push", "-u", "origin", "main"]);

// Simulate "the implement/fix steps already edited the tree" — the
// corpus replay itself cannot, since it only replays chunks.
writeFileSync(join(workDir, "src/index.ts"), FINAL_INDEX);
writeFileSync(join(workDir, "src/index.test.ts"), FINAL_TEST);

mkdirSync(binDir, { recursive: true });
const fakeGhPath = join(binDir, "gh");
writeFileSync(fakeGhPath, FAKE_GH_SCRIPT);
chmodSync(fakeGhPath, 0o755);
process.env.PATH = `${binDir}:${originalPath}`;

const events: Array<unknown> = [];
const handle = startRun(
implementIssue,
makeAgentRuntime(fakeAgents(createCorpusReplayAdapter(corpus.path))),
{
runId: "test-run-corpus-replay",
dir: workDir,
input: {
issueNumber: 1,
},
repo: { slug: "local/fixture", baseBranch: "main" },
onEvent: (event) => events.push(event),
},
repo: { slug: "local/fixture", baseBranch: "main" },
onEvent: (event) => events.push(event),
},
);

const outcome = await handle.result;

expect(outcome.outcome).toBe("completed");
if (outcome.outcome !== "completed") return;
expect(outcome.output.testAfterImplementExitCode).toBe(0);
expect(outcome.output.testAfterFixExitCode).toBe(0);
expect(outcome.output.hostSideStabilityIntact).toBe(true);
expect(outcome.output.fixStepSurvivalIntact).toBe(true);
expect(outcome.output.prMetadataMechanism).toBe("extracted");
expect(outcome.output.prUrl).toBe("https://github.com/local/fixture/pull/1");
expect(outcome.output.prBranch).toBe("factory/issue-1");

// The commit event carries the message the template produced: exactly
// one "Closes" line, no escaped-newline litter, blank-line-separated.
const commit = events.find(
(event) =>
typeof event === "object" &&
event !== null &&
(event as { payload?: { _tag?: string } }).payload?._tag === "ExecStarted" &&
JSON.stringify(
(event as { payload?: { command?: ReadonlyArray<string> } }).payload?.command,
).startsWith('["git","commit","-m"'),
) as { payload: { command: ReadonlyArray<string> } } | undefined;
expect(commit).toBeDefined();
const message = commit!.payload.command[3]!;
expect(message).not.toContain("\\n");
expect(message.split("Closes #1.").length - 1).toBe(1);
expect(message).toBe(
`Implement issue #1\n\nCloses #1.\n\nAutomated by the factory implement-issue workflow.`,
);
} finally {
process.env.PATH = originalPath;
await rm(root, { recursive: true, force: true });
}
});
});
);

const outcome = await handle.result;

expect(outcome.outcome).toBe("completed");
if (outcome.outcome !== "completed") return;
expect(outcome.output.testAfterImplementExitCode).toBe(0);
expect(outcome.output.testAfterFixExitCode).toBe(0);
expect(outcome.output.hostSideStabilityIntact).toBe(true);
expect(outcome.output.fixStepSurvivalIntact).toBe(true);
expect(outcome.output.prMetadataMechanism).toBe("extracted");
expect(outcome.output.prUrl).toBe("https://github.com/local/fixture/pull/1");
expect(outcome.output.prBranch).toBe(corpus.branch);

// The commit event carries the message the template produced: exactly
// one "Closes" line, no escaped-newline litter, blank-line-separated.
const commit = events.find(
(event) =>
typeof event === "object" &&
event !== null &&
(event as { payload?: { _tag?: string } }).payload?._tag === "ExecStarted" &&
JSON.stringify(
(event as { payload?: { command?: ReadonlyArray<string> } }).payload?.command,
).startsWith('["git","commit","-m"'),
) as { payload: { command: ReadonlyArray<string> } } | undefined;
expect(commit).toBeDefined();
const message = commit!.payload.command[3]!;
expect(message).not.toContain("\\n");
expect(message.split("Closes #1.").length - 1).toBe(1);
expect(message).toBe(
`Implement issue #1\n\nCloses #1.\n\nAutomated by the factory implement-issue workflow.`,
);
} finally {
process.env.PATH = originalPath;
await rm(root, { recursive: true, force: true });
}
});
},
);
39 changes: 35 additions & 4 deletions e2e/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@
*
* - **Static corpus.** `createCorpusReplayAdapter` turns a committed
* `test/corpus/*.ndjson` into a real event log with no AI in the loop. The
* rich one is the implement-issue round trip, run against a local bare
* rich ones are the implement-issue round trip, run against a local bare
* `origin` and a fake `gh` (the same fixture `e2e/implement-issue.test.ts`
* uses); a second, small run adds ordering/status variety.
* uses): `run-static-acp-replay` from the ACP adapter's recording, and
* `run-static-corpus` from the opencode adapter's, for how the SPA reads
* logs written before ADR 0013. A second, small run adds ordering/status
* variety.
* - **Live.** The daemon serves with `createSlowFakeAdapter`, so a
* `POST /api/runs` from a test yields a run whose steps arrive over SSE on
* a controllable clock.
Expand Down Expand Up @@ -48,11 +51,15 @@ const nestedInputWorkflow = defineWorkflow("nested-input-test", {
},
});

/** Recorded by the opencode adapter, before ADR 0013. */
const CORPUS_ROUND_TRIP = join(import.meta.dir, "../test/corpus/run-1789308170212.ndjson");
const CORPUS_ONE_STEP = join(
/** Recorded through the ACP adapter by `scripts/record-corpus.ts`; opencode thinks aloud, so it has reasoning. */
const CORPUS_ACP_ROUND_TRIP = join(
import.meta.dir,
"../test/corpus/effect-boundary-control-3-1789309633183.ndjson",
"../test/corpus/acp-opencode-implement-issue.ndjson",
);
/** Its first step is all the one-step echo run replays. */
const CORPUS_ONE_STEP = join(import.meta.dir, "../test/corpus/acp-claude-implement-issue.ndjson");

const GREET_ONLY_INDEX = `/** Returns a friendly greeting for the given name. */
export function greet(name: string): string {
Expand Down Expand Up @@ -157,6 +164,30 @@ async function seedCorpusRuns(root: string, db: ReturnType<typeof openStore>): P

await Bun.sleep(20);

// The same round trip as the ACP adapter records it, in its own clone: the
// run above already wrote back from `workDir`.
const acpWorkDir = join(root, "work-acp");
await hostExec(["git", "clone", "-q", "-b", "main", remoteDir, acpWorkDir]);
await git(acpWorkDir, ["config", "user.name", "Factory E2E"]);
await git(acpWorkDir, ["config", "user.email", "factory-e2e@example.com"]);
writeFileSync(join(acpWorkDir, "src/index.ts"), FINAL_INDEX);
writeFileSync(join(acpWorkDir, "src/index.test.ts"), FINAL_TEST);
await awaitRun(
startRun(
implementIssue,
makeAgentRuntime(fakeAgents(createCorpusReplayAdapter(CORPUS_ACP_ROUND_TRIP))),
{
runId: "run-static-acp-replay",
dir: acpWorkDir,
repo: { slug: "local/fixture", baseBranch: "main" },
input: { issueNumber: 1 },
onEvent: (event) => appendEvent(db, event),
},
),
);

await Bun.sleep(20);

// A second, cheap run so list ordering has more than one data point.
await awaitRun(
startRun(
Expand Down
Loading
Loading