${esc(entry.path)}
`
+ (entry.error ? `
${esc(entry.error)}` : '')
+ (chips.length ? `
${chips.join('')}
` : '')
diff --git a/src/reportMarkdown.js b/src/reportMarkdown.js
index eda3a7a..dfd6779 100644
--- a/src/reportMarkdown.js
+++ b/src/reportMarkdown.js
@@ -48,10 +48,17 @@ function table(report) {
return [`| ${cols.join(' | ')} |`, `|${cols.map(() => '---').join('|')}|`, `| ${cells.join(' | ')} |`];
}
+function resultNote(entry) {
+ if (typeof entry.runs === 'number') {
+ const count = `${entry.passes}/${entry.runs} passed`;
+ return ` _(${entry.flaky ? `flaky, ${count}` : count})_`;
+ }
+ return entry.attempts > 1 ? ` _(${entry.attempts} attempts)_` : '';
+}
+
function item(entry) {
if (entry.kind === 'test') {
- const retried = entry.attempts > 1 ? ` _(${entry.attempts} attempts)_` : '';
- const lines = [`- ❌ **${escapeMd(pretty(entry.path))}**${retried}`];
+ const lines = [`- ❌ **${escapeMd(pretty(entry.path))}**${resultNote(entry)}`];
if (entry.error) lines.push(` > ${code(firstLine(entry.error))}`);
if (entry.snapshot) lines.push(` > Layout snapshot ${code(entry.snapshot.name)} differs, diff in the report`);
return lines;
@@ -83,6 +90,8 @@ export function renderMarkdown(report) {
lines.push(...table(report), '');
+ if (report.run?.repeat > 1) lines.push(`Repeated ${report.run.repeat}×.`, '');
+
const items = needsAttention(report);
if (items.length) {
lines.push('#### Needs attention');
diff --git a/src/runReport.js b/src/runReport.js
index 48e81a3..2d7967e 100644
--- a/src/runReport.js
+++ b/src/runReport.js
@@ -69,6 +69,7 @@ export function buildRunReport({
snapshots = [],
recordingFailed = false,
coverage = null,
+ repeat = null,
}) {
// Resolved here, in the shard that ran the tests, because this is the only
// place the handler map is valid: ids are per-page-load random, so shard 2's
@@ -80,7 +81,7 @@ export function buildRunReport({
return finalizeReport({
schemaVersion: REPORT_SCHEMA_VERSION,
- run: { twdCliVersion: version, url },
+ run: { twdCliVersion: version, url, ...(repeat > 1 ? { repeat } : {}) },
error,
snapshots,
coverage,
@@ -116,12 +117,15 @@ export function buildRunReport({
...test,
// For display. May be null if the handler somehow went missing, so every
// renderer has to tolerate that.
- path: buildTestPath(test.id, handlers),
+ // A repeat reloads the page, so an id from an earlier round is absent from
+ // the last handler map. The tally stored the path and index while that
+ // round's map was still valid; keep them when the lookup misses.
+ path: buildTestPath(test.id, handlers) ?? test.path ?? null,
// Identity. Position in the shard-independent ordered list is stable
// across shards because registration order is deterministic, unlike the
// random id. The path cannot serve as the key: two tests may share one
// (duplicate names) and can legally land in different shards.
- index: positions.has(test.id) ? positions.get(test.id) : null,
+ index: positions.has(test.id) ? positions.get(test.id) : (test.index ?? null),
attempts: attemptsFor(test, retryCount),
})),
contracts: contracts ?? {
diff --git a/src/testSummary.js b/src/testSummary.js
index 92c021f..12635c8 100644
--- a/src/testSummary.js
+++ b/src/testSummary.js
@@ -1,5 +1,6 @@
import { buildTestPath } from './buildTestPath.js';
import { formatFailureDiagnostics } from './failureDiagnostics.js';
+import { isFlaky } from './repeat.js';
/**
* Display name for one test result.
@@ -16,6 +17,28 @@ function resolvePath(entry, handlers) {
return entry.path ?? buildTestPath(entry.id, handlers) ?? entry.id;
}
+// Count tests, not rounds, so the console matches run.json and the reports.
+// A test that failed any round has status "fail" and counts once; the
+// per-test (passes/runs passed) suffix carries the round detail.
+function outcomeCounts(testStatus) {
+ return {
+ passed: testStatus.filter((t) => t.status === 'pass').length,
+ failed: testStatus.filter((t) => t.status === 'fail').length,
+ skipped: testStatus.filter((t) => t.status === 'skip').length,
+ };
+}
+
+function pushResultLines(lines, test, handlers, marker, suffix) {
+ const testPath = resolvePath(test, handlers);
+ lines.push(` ${marker} ${testPath}${suffix}`);
+ for (const row of formatFailureDiagnostics(test.diagnostics)) {
+ lines.push(` ${row}`);
+ }
+ if (test.error) {
+ lines.push(` ${String(test.error).replace(/\n/g, '\n ')}`);
+ }
+}
+
export function formatRunComplete({
testStatus,
handlers,
@@ -27,10 +50,9 @@ export function formatRunComplete({
computeMs = null,
reportPath = null,
cpuThrottle = 1,
+ repeat = null,
}) {
- const passed = testStatus.filter((t) => t.status === 'pass').length;
- const failed = testStatus.filter((t) => t.status === 'fail').length;
- const skipped = testStatus.filter((t) => t.status === 'skip').length;
+ const { passed, failed, skipped } = outcomeCounts(testStatus);
const duration = (durationMs / 1000).toFixed(1);
const lines = [
@@ -54,21 +76,26 @@ export function formatRunComplete({
// Right under the duration it inflates. Only when active, so an unthrottled
// block stays exactly what it has always been.
if (cpuThrottle > 1) lines.push(` CPU throttle: ${cpuThrottle}x`);
+ if (repeat > 1) lines.push(` Repeat: ${repeat}x`);
- const failures = testStatus.filter((t) => t.status === 'fail');
+ const repeating = repeat > 1;
+ const flaky = repeating ? testStatus.filter((t) => isFlaky(t)) : [];
+ const failures = testStatus.filter((t) => t.status === 'fail' && !(repeating && isFlaky(t)));
if (failures.length > 0) {
lines.push('', ` Failed tests (${failures.length}):`);
for (const failure of failures) {
- const testPath = resolvePath(failure, handlers);
- lines.push(` × ${testPath}`);
- // Above the error, not below: a twd-js failure message can carry a full
- // accessible-roles dump, which would bury the block underneath it.
- for (const row of formatFailureDiagnostics(failure.diagnostics)) {
- lines.push(` ${row}`);
- }
- if (failure.error) {
- lines.push(` ${String(failure.error).replace(/\n/g, '\n ')}`);
- }
+ const suffix = repeating ? ` (${failure.passes ?? 0}/${failure.runs} passed)` : '';
+ // The marker line, then diagnostics, then the error. A twd-js failure
+ // message can carry a full accessible-roles dump, which would bury the
+ // block if the error came first.
+ pushResultLines(lines, failure, handlers, '×', suffix);
+ }
+ }
+
+ if (flaky.length > 0) {
+ lines.push('', ` Flaky (${flaky.length}):`);
+ for (const test of flaky) {
+ pushResultLines(lines, test, handlers, '~', ` (${test.passes}/${test.runs} passed)`);
}
}
diff --git a/src/usage.js b/src/usage.js
index c859643..fb3a1f4 100644
--- a/src/usage.js
+++ b/src/usage.js
@@ -63,6 +63,18 @@ CPU throttling:
shows. Overrides cpuThrottle in twd.config.json
(default 1, full speed). Below 1 is refused.
+Repeating:
+ --repeat
Run the selected tests times in this browser,
+ reloading the page between rounds, and list each test
+ that failed a round with how often it passed. is
+ a positive integer. Retries are off for the run.
+ maxFailures counts tests that failed, not each failed
+ round. The console counts tests, not rounds. Cannot
+ be combined with --record, --shard,
+ --update-snapshots or --ci. Skips coverage,
+ contracts and layout snapshots.
+ Exits 1 if any round of any test failed.
+
Sharding (beta):
--shard / Run slice i of n. Each shard discovers the whole
suite and takes every nth test, so the count never
@@ -94,6 +106,7 @@ Examples:
npx twd-cli run --record --changed-since origin/main
npx twd-cli run --shard 2/4
npx twd-cli run --cpu-throttle 6 --test "checkout flow"
+ npx twd-cli run --repeat 10 --cpu-throttle 6 --test "checkout flow"
Create a twd.config.json file in your project root to customize settings.
`;
diff --git a/tests/cli.test.js b/tests/cli.test.js
index 274fb0c..84c05d3 100644
--- a/tests/cli.test.js
+++ b/tests/cli.test.js
@@ -83,6 +83,20 @@ describe("twd-cli help", () => {
expect(stderr).toMatch(/Did you mean --test\?/);
});
+ it("run with a bad --repeat refuses to run: stderr names it, exit 1", async () => {
+ const { code, stdout, stderr } = await cli("run", "--repeat", "0");
+ expect(code).toBe(1);
+ expect(stdout).toBe("");
+ expect(stderr).toMatch(/Invalid --repeat: expected a positive integer, got "0"/);
+ });
+
+ it("run with --repeat and --ci refuses to run: stderr names it, exit 1, nothing on stdout", async () => {
+ const { code, stdout, stderr } = await cli("run", "--repeat", "2", "--ci");
+ expect(code).toBe(1);
+ expect(stdout).toBe("");
+ expect(stderr).toMatch(/--repeat cannot be combined with --ci/);
+ });
+
it("run with a --cpu-throttle below 1 refuses to run: stderr names it, exit 1", async () => {
const { code, stdout, stderr } = await cli("run", "--cpu-throttle", "0");
expect(code).toBe(1);
diff --git a/tests/needsAttention.test.js b/tests/needsAttention.test.js
index 7c0a15a..66be726 100644
--- a/tests/needsAttention.test.js
+++ b/tests/needsAttention.test.js
@@ -7,6 +7,22 @@ describe('needsAttention', () => {
expect(needsAttention(report())).toEqual([]);
});
+ it('marks a test that both passed and failed as flaky and keeps its counts', () => {
+ const [item] = needsAttention(report({
+ tests: [{ id: 't1', status: 'fail', error: 'boom', runs: 10, passes: 8, skips: 0 }],
+ }));
+ expect(item).toMatchObject({ kind: 'test', flaky: true, runs: 10, passes: 8, error: 'boom' });
+ });
+
+ it('does not mark a test that failed every round as flaky', () => {
+ const [item] = needsAttention(report({
+ tests: [{ id: 't1', status: 'fail', error: 'boom', runs: 10, passes: 0, skips: 0 }],
+ }));
+ expect(item.flaky).toBe(false);
+ expect(item.runs).toBe(10);
+ expect(item.passes).toBe(0);
+ });
+
it('lists failed tests in run order with their error and attempts', () => {
const items = needsAttention(report({
tests: [
diff --git a/tests/parseArgs.test.js b/tests/parseArgs.test.js
index 13a994d..9e2336b 100644
--- a/tests/parseArgs.test.js
+++ b/tests/parseArgs.test.js
@@ -3,7 +3,7 @@ import { parseRunArgs, parseMergeArgs, parseReportArgs } from "../src/parseArgs.
describe("parseRunArgs", () => {
it("returns empty filters when no args", () => {
- expect(parseRunArgs([])).toEqual({ testFilters: [], changedSince: null, record: {}, shard: null, reportDir: null, noReport: false, updateSnapshots: false, ci: false, cpuThrottle: null });
+ expect(parseRunArgs([])).toEqual({ testFilters: [], changedSince: null, record: {}, shard: null, reportDir: null, noReport: false, updateSnapshots: false, ci: false, cpuThrottle: null, repeat: null });
});
it("parses a single --test ", () => {
@@ -17,6 +17,7 @@ describe("parseRunArgs", () => {
updateSnapshots: false,
ci: false,
cpuThrottle: null,
+ repeat: null,
});
});
@@ -31,6 +32,7 @@ describe("parseRunArgs", () => {
updateSnapshots: false,
ci: false,
cpuThrottle: null,
+ repeat: null,
});
});
@@ -45,11 +47,12 @@ describe("parseRunArgs", () => {
updateSnapshots: false,
ci: false,
cpuThrottle: null,
+ repeat: null,
});
});
it("ignores a trailing --test with no value", () => {
- expect(parseRunArgs(['--test'])).toEqual({ testFilters: [], changedSince: null, record: {}, shard: null, reportDir: null, noReport: false, updateSnapshots: false, ci: false, cpuThrottle: null });
+ expect(parseRunArgs(['--test'])).toEqual({ testFilters: [], changedSince: null, record: {}, shard: null, reportDir: null, noReport: false, updateSnapshots: false, ci: false, cpuThrottle: null, repeat: null });
});
it("ignores positional tokens", () => {
@@ -100,6 +103,7 @@ describe("parseRunArgs", () => {
updateSnapshots: false,
ci: false,
cpuThrottle: null,
+ repeat: null,
});
});
@@ -137,6 +141,7 @@ describe("parseRunArgs", () => {
updateSnapshots: false,
ci: false,
cpuThrottle: null,
+ repeat: null,
});
});
@@ -177,6 +182,7 @@ describe('parseRunArgs shard and report flags', () => {
updateSnapshots: false,
ci: false,
cpuThrottle: null,
+ repeat: null,
});
});
@@ -304,6 +310,61 @@ describe('parseRunArgs --cpu-throttle', () => {
});
});
+describe('parseRunArgs --repeat', () => {
+ it('parses --repeat in both forms', () => {
+ expect(parseRunArgs(['--repeat', '5']).repeat).toBe(5);
+ expect(parseRunArgs(['--repeat=10']).repeat).toBe(10);
+ });
+
+ it('is null when the flag is absent', () => {
+ expect(parseRunArgs([]).repeat).toBeNull();
+ });
+
+ it('accepts 1, a single run', () => {
+ expect(parseRunArgs(['--repeat', '1']).repeat).toBe(1);
+ });
+
+ it('throws on zero, fractions, and non-numbers', () => {
+ expect(() => parseRunArgs(['--repeat', '0'])).toThrow(/Invalid --repeat/);
+ expect(() => parseRunArgs(['--repeat=1.5'])).toThrow(/Invalid --repeat/);
+ expect(() => parseRunArgs(['--repeat', 'nope'])).toThrow(/got "nope"/);
+ expect(() => parseRunArgs(['--repeat', '-3'])).toThrow(/got "-3"/);
+ });
+
+ it('throws on a trailing --repeat with no value', () => {
+ expect(() => parseRunArgs(['--repeat'])).toThrow(/got nothing/);
+ });
+
+ it('throws rather than swallowing the flag that follows a valueless --repeat', () => {
+ expect(() => parseRunArgs(['--repeat', '--test', 'Login'])).toThrow(/got nothing/);
+ });
+
+ it('combines with --test and --cpu-throttle', () => {
+ const result = parseRunArgs(['--repeat', '10', '--cpu-throttle', '6', '--test', 'checkout']);
+ expect(result.repeat).toBe(10);
+ expect(result.cpuThrottle).toBe(6);
+ expect(result.testFilters).toEqual(['checkout']);
+ });
+
+ it.each([
+ [['--repeat', '2', '--shard', '1/2'], /--repeat cannot be combined with --shard\./],
+ [['--repeat', '2', '--record'], /--repeat cannot be combined with --record\./],
+ [['--repeat', '2', '--update-snapshots'], /--repeat cannot be combined with --update-snapshots\./],
+ [['--repeat', '2', '--ci'], /--repeat cannot be combined with --ci\./],
+ [['--repeat', '2', '--shard', '1/2', '--ci'], /--repeat cannot be combined with --shard and --ci\./],
+ ])('refuses %j in the parser', (argv, message) => {
+ expect(() => parseRunArgs(argv)).toThrow(message);
+ });
+
+ it('allows --repeat 1 with those flags', () => {
+ expect(parseRunArgs(['--repeat', '1', '--ci']).repeat).toBe(1);
+ });
+
+ it('suggests --repeat for a near miss', () => {
+ expect(() => parseRunArgs(['--repeats', '2'])).toThrow(/Did you mean --repeat\?/);
+ });
+});
+
describe('parseRunArgs unknown options', () => {
it('refuses an unknown --flag instead of dropping it', () => {
// The whole reason `run --help` ran the suite: a token no branch claimed
diff --git a/tests/repeat.test.js b/tests/repeat.test.js
new file mode 100644
index 0000000..42c0793
--- /dev/null
+++ b/tests/repeat.test.js
@@ -0,0 +1,95 @@
+import { describe, it, expect } from 'vitest';
+import { parseRepeat, recordRound, failedTestCount, tallyToStatus, isFlaky } from '../src/repeat.js';
+
+describe('parseRepeat', () => {
+ it('accepts a positive integer string', () => {
+ expect(parseRepeat('5')).toBe(5);
+ expect(parseRepeat('1')).toBe(1);
+ });
+
+ it('rejects zero, fractions, and junk', () => {
+ expect(() => parseRepeat('0')).toThrow(/Invalid --repeat: expected a positive integer, got "0"/);
+ expect(() => parseRepeat('1.5')).toThrow(/got "1.5"/);
+ expect(() => parseRepeat('-2')).toThrow(/got "-2"/);
+ expect(() => parseRepeat('nope')).toThrow(/got "nope"/);
+ expect(() => parseRepeat(undefined)).toThrow(/got nothing/);
+ });
+});
+
+describe('recordRound', () => {
+ const handlersFor = (id, suiteId = 's') => [
+ { id: suiteId, name: 'Checkout', type: 'suite' },
+ { id, name: 'checkout flow', parent: suiteId, type: 'test' },
+ ];
+
+ it('counts passes and failures of the same test across reloads', () => {
+ const tally = new Map();
+ recordRound(tally, ['a'], [{ id: 'a', status: 'pass' }], handlersFor('a', 's1'));
+ recordRound(
+ tally,
+ ['b'],
+ [{ id: 'b', status: 'fail', error: 'total mismatch', diagnostics: { mock: true } }],
+ handlersFor('b', 's2'),
+ );
+
+ const [row] = tallyToStatus(tally);
+ expect(row).toMatchObject({
+ id: 'b',
+ status: 'fail',
+ runs: 2,
+ passes: 1,
+ skips: 0,
+ error: 'total mismatch',
+ path: 'Checkout > checkout flow',
+ index: 0,
+ });
+ expect(row.diagnostics).toEqual({ mock: true });
+ expect(isFlaky(row)).toBe(true);
+ expect(failedTestCount(tally)).toBe(1);
+ });
+
+ it('does not count a second result for the same id in one round', () => {
+ const tally = new Map();
+ recordRound(tally, ['a'], [
+ { id: 'a', status: 'pass' },
+ { id: 'a', status: 'fail', error: 'dup' },
+ ], handlersFor('a'));
+ expect(tallyToStatus(tally)[0]).toMatchObject({ runs: 1, passes: 1, status: 'pass' });
+ });
+
+ it('keeps a test that failed every round as a failure with no passes', () => {
+ const tally = new Map();
+ const handlers = [{ id: 'a', name: 't', type: 'test' }];
+ recordRound(tally, ['a'], [{ id: 'a', status: 'fail', error: 'x' }], handlers);
+ recordRound(tally, ['a'], [{ id: 'a', status: 'fail', error: 'y' }], handlers);
+ const [row] = tallyToStatus(tally);
+ expect(row).toMatchObject({ runs: 2, passes: 0, status: 'fail', error: 'y' });
+ expect(isFlaky(row)).toBe(false);
+ });
+
+ it('counts a skip separately from a failure', () => {
+ const tally = new Map();
+ const handlers = [{ id: 'a', name: 't', type: 'test' }];
+ recordRound(tally, ['a'], [{ id: 'a', status: 'skip' }], handlers);
+ recordRound(tally, ['a'], [{ id: 'a', status: 'pass' }], handlers);
+ expect(tallyToStatus(tally)[0]).toMatchObject({ runs: 2, passes: 1, skips: 1, status: 'pass' });
+ expect(failedTestCount(tally)).toBe(0);
+ });
+
+ it('counts distinct failed tests, not failed rounds', () => {
+ const tally = new Map();
+ const handlers = [
+ { id: 'a', name: 'one', type: 'test' },
+ { id: 'b', name: 'two', type: 'test' },
+ ];
+ recordRound(tally, ['a', 'b'], [
+ { id: 'a', status: 'fail', error: 'x' },
+ { id: 'b', status: 'pass' },
+ ], handlers);
+ recordRound(tally, ['a', 'b'], [
+ { id: 'a', status: 'fail', error: 'x' },
+ { id: 'b', status: 'pass' },
+ ], handlers);
+ expect(failedTestCount(tally)).toBe(1);
+ });
+});
diff --git a/tests/reportHtml.test.js b/tests/reportHtml.test.js
index 874e9b4..3d2bbd8 100644
--- a/tests/reportHtml.test.js
+++ b/tests/reportHtml.test.js
@@ -26,6 +26,24 @@ describe('renderHtml', () => {
expect(html).not.toContain('Needs attention');
});
+ it('shows a flaky test under needs attention with its pass count', () => {
+ const html = renderHtml(report({
+ repeat: 10,
+ tests: [{ id: 't1', status: 'fail', error: 'boom', runs: 10, passes: 8, skips: 0 }],
+ }));
+ expect(html).toContain('Needs attention (1)');
+ expect(html).toContain('flaky · 8/10 passed');
+ expect(html).toContain('>10<');
+ });
+
+ it('shows a pass count for a test that failed every round without calling it flaky', () => {
+ const html = renderHtml(report({
+ tests: [{ id: 't1', status: 'fail', error: 'boom', runs: 10, passes: 0, skips: 0 }],
+ }));
+ expect(html).toContain('0/10 passed');
+ expect(html).not.toContain('flaky');
+ });
+
it('puts a contract error next to test failures', () => {
const html = renderHtml(report({ contracts: { results: [contractResult({ validation: invalid() })] } }));
expect(html).toContain('Needs attention (1)');
diff --git a/tests/reportMarkdown.test.js b/tests/reportMarkdown.test.js
index 641a6e3..334fec7 100644
--- a/tests/reportMarkdown.test.js
+++ b/tests/reportMarkdown.test.js
@@ -25,6 +25,25 @@ describe('renderMarkdown', () => {
expect(md).toContain('1 ✓ · 0 ✕ · 0 ⚠');
});
+ it('labels a flaky test under needs attention and notes the repeat', () => {
+ const md = renderMarkdown(report({
+ repeat: 10,
+ tests: [{ id: 't1', status: 'fail', error: 'Expected visible\nroles dump...', runs: 10, passes: 8, skips: 0 }],
+ }));
+ expect(md).toContain('- ❌ **Invoices › shows empty state** _(flaky, 8/10 passed)_');
+ expect(md).toContain('Repeated 10×');
+ expect(md).toContain(' > `Expected visible`');
+ });
+
+ it('shows a pass count for a test that failed every round without calling it flaky', () => {
+ const md = renderMarkdown(report({
+ repeat: 10,
+ tests: [{ id: 't1', status: 'fail', error: 'boom', runs: 10, passes: 0, skips: 0 }],
+ }));
+ expect(md).toContain('_(0/10 passed)_');
+ expect(md).not.toContain('flaky');
+ });
+
it('lists each failure with its first error line', () => {
const md = renderMarkdown(report({
tests: [{ id: 't1', status: 'fail', error: 'Expected visible\nroles dump...' }],
diff --git a/tests/runReport.test.js b/tests/runReport.test.js
index cd313ef..fa6a1ef 100644
--- a/tests/runReport.test.js
+++ b/tests/runReport.test.js
@@ -205,6 +205,29 @@ describe('schema v3', () => {
expect(build().schemaVersion).toBe(3);
});
+ it('carries repeat on the run and per-test counts without changing the outcome', () => {
+ const report = build({
+ repeat: 10,
+ tests: [{ id: 't1', status: 'fail', error: 'total', runs: 10, passes: 8, skips: 0 }],
+ });
+ expect(report.run.repeat).toBe(10);
+ expect(report.tests[0]).toMatchObject({ status: 'fail', runs: 10, passes: 8, skips: 0 });
+ expect(report.outcome).toBe('failed');
+ expect(report.schemaVersion).toBe(3);
+ });
+
+ it('omits repeat from the header when the run was not repeated', () => {
+ expect(build().run.repeat).toBeUndefined();
+ });
+
+ it('keeps a path recorded before the test id went stale across a reload', () => {
+ const report = build({
+ tests: [{ id: 'stale', status: 'fail', error: 'e', path: 'Checkout > checkout flow', index: 4, runs: 2, passes: 1 }],
+ });
+ expect(report.tests[0].path).toBe('Checkout > checkout flow');
+ expect(report.tests[0].index).toBe(4);
+ });
+
it('records the run header', () => {
const report = build({ version: '1.10.0', url: 'http://localhost:5173' });
expect(report.run).toEqual({
diff --git a/tests/runTests.test.js b/tests/runTests.test.js
index dc2501a..898f651 100644
--- a/tests/runTests.test.js
+++ b/tests/runTests.test.js
@@ -812,7 +812,7 @@ describe("runTests recording", () => {
await runTests();
const [inject, flags] = page.evaluateOnNewDocument.mock.calls[0];
- expect(flags).toEqual({ update: false, ci: false });
+ expect(flags).toEqual({ update: false, ci: false, snapshots: true });
// The injected function is serialised into the browser, so run it here
// against a stand-in global to see what it actually sets.
@@ -836,7 +836,7 @@ describe("runTests recording", () => {
await runTests({ updateSnapshots: true, ci: true });
- expect(page.evaluateOnNewDocument.mock.calls[0][1]).toEqual({ update: true, ci: true });
+ expect(page.evaluateOnNewDocument.mock.calls[0][1]).toEqual({ update: true, ci: true, snapshots: true });
});
it("does not touch any recording API when recording is disabled", async () => {
@@ -2693,3 +2693,249 @@ describe("runTests CPU throttling", () => {
expect(logs.some((l) => /CPU throttl/.test(l))).toBe(false);
});
});
+
+describe('runTests --repeat', () => {
+ let logSpy;
+
+ function writtenReport() {
+ const call = vi.mocked(fs.writeFileSync).mock.calls
+ .find(([file]) => String(file).endsWith('run.json'));
+ return call ? JSON.parse(call[1]) : null;
+ }
+
+ function completeBlock() {
+ const logs = logSpy.mock.calls.map((c) => String(c[0]));
+ return logs.find((l) => l.startsWith('--- Run complete ---'));
+ }
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig });
+ vi.mocked(fs.writeFileSync).mockReset();
+ // An earlier test installs an rmSync that throws EPERM. clearAllMocks does
+ // not drop that implementation, and a leaked throw becomes a clean warning
+ // on every later run.
+ vi.mocked(fs.rmSync).mockReset();
+ logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it('reloads between rounds, forces a single attempt, and fails the run when any round failed', async () => {
+ const rounds = [
+ {
+ handlers: [
+ { id: 's1', name: 'Checkout', type: 'suite' },
+ { id: 'a', name: 'checkout flow', parent: 's1', type: 'test' },
+ { id: 'b', name: 'receipt', parent: 's1', type: 'test' },
+ ],
+ status: [
+ { id: 'a', status: 'pass' },
+ { id: 'b', status: 'pass' },
+ ],
+ },
+ {
+ handlers: [
+ { id: 's2', name: 'Checkout', type: 'suite' },
+ { id: 'c', name: 'checkout flow', parent: 's2', type: 'test' },
+ { id: 'd', name: 'receipt', parent: 's2', type: 'test' },
+ ],
+ status: [
+ { id: 'c', status: 'fail', error: 'total mismatch' },
+ { id: 'd', status: 'pass' },
+ ],
+ },
+ ];
+ let enumerated = 0;
+ const page = createMockPage();
+ page.evaluate = vi.fn(async (_fn, retryCount, ids) => {
+ if (ids === undefined) {
+ const round = rounds[enumerated];
+ enumerated += 1;
+ return round.handlers;
+ }
+ expect(retryCount).toBe(1);
+ return rounds[enumerated - 1].status;
+ });
+ vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page));
+ vi.mocked(loadConfig).mockReturnValue({
+ ...defaultMockConfig, retryCount: 3, coverage: true,
+ });
+
+ const failed = await runTests({ repeat: 2 });
+
+ expect(failed).toBe(true);
+ expect(page.goto).toHaveBeenCalledTimes(2);
+ expect(page.waitForSelector).toHaveBeenCalledTimes(2);
+ expect(puppeteer.launch).toHaveBeenCalledTimes(1);
+ const block = completeBlock();
+ expect(block).toContain('Passed: 1 | Failed: 1 | Skipped: 0');
+ expect(block).toContain('Repeat: 2x');
+ expect(block).toContain('Flaky (1):');
+ expect(block).toContain('~ Checkout > checkout flow (1/2 passed)');
+ expect(block).toContain('total mismatch');
+ expect(block).not.toContain('Retried');
+ const logs = logSpy.mock.calls.map((c) => String(c[0]));
+ expect(logs.some((l) => l.includes('retries are off'))).toBe(true);
+ expect(logs.some((l) => l.includes('Skipping coverage collection (--repeat)'))).toBe(true);
+
+ const report = writtenReport();
+ expect(report.run.repeat).toBe(2);
+ expect(report.outcome).toBe('failed');
+ expect(report.tests.find((t) => t.passes === 1)).toMatchObject({
+ status: 'fail', runs: 2, passes: 1,
+ });
+ const files = vi.mocked(fs.writeFileSync).mock.calls.map(([f]) => String(f));
+ expect(files.some((f) => f.includes('.nyc_output'))).toBe(false);
+ });
+
+ it('exits 0 when every round passed', async () => {
+ const handlers = [{ id: 't', name: 'stable', type: 'test' }];
+ let enumerated = 0;
+ const page = createMockPage();
+ page.evaluate = vi.fn(async (_fn, _retryCount, ids) => {
+ if (ids === undefined) {
+ enumerated += 1;
+ return handlers;
+ }
+ return [{ id: 't', status: 'pass' }];
+ });
+ vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page));
+
+ const failed = await runTests({ repeat: 3 });
+
+ expect(failed).toBe(false);
+ expect(page.goto).toHaveBeenCalledTimes(3);
+ expect(enumerated).toBe(3);
+ expect(completeBlock()).toContain('Passed: 1 | Failed: 0 | Skipped: 0');
+ expect(completeBlock()).not.toContain('Flaky');
+ expect(writtenReport().outcome).toBe('passed');
+ expect(writtenReport().run.repeat).toBe(3);
+ });
+
+ it('does not bail when the same test fails every round under the default failure limit', async () => {
+ const handlers = [{ id: 't', name: 'flaky', type: 'test' }];
+ const page = createMockPage();
+ page.evaluate = vi.fn(async (_fn, _retryCount, ids) => {
+ if (ids === undefined) return handlers;
+ return [{ id: 't', status: 'fail', error: 'x' }];
+ });
+ vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page));
+ vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig, maxFailures: 2 });
+
+ const failed = await runTests({ repeat: 4 });
+
+ expect(failed).toBe(true);
+ expect(page.goto).toHaveBeenCalledTimes(4);
+ const chunks = page.evaluate.mock.calls.filter((call) => call.length === 3);
+ expect(chunks).toHaveLength(4);
+ expect(completeBlock()).toContain('(0/4 passed)');
+ expect(completeBlock()).not.toContain('Flaky');
+ expect(completeBlock()).not.toContain('Stopped early');
+ });
+
+ it('stops once enough distinct tests have failed, and does not start another round', async () => {
+ const handlers = [
+ { id: 't1', name: 'one', type: 'test' },
+ { id: 't2', name: 'two', type: 'test' },
+ { id: 't3', name: 'three', type: 'test' },
+ ];
+ const page = createMockPage();
+ page.evaluate = vi.fn(async (_fn, _retryCount, ids) => {
+ if (ids === undefined) return handlers;
+ return ids.map((id) => ({ id, status: 'fail', error: 'x' }));
+ });
+ vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page));
+ vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig, maxFailures: 1, chunkSize: 1 });
+
+ await runTests({ repeat: 3 });
+
+ expect(page.goto).toHaveBeenCalledTimes(1);
+ const chunks = page.evaluate.mock.calls.filter((call) => call.length === 3);
+ expect(chunks).toHaveLength(1);
+ expect(chunks[0][2]).toEqual(['t1']);
+ expect(completeBlock()).toContain('Stopped early');
+ expect(writtenReport().run.repeat).toBe(3);
+ expect(writtenReport().tests).toEqual([
+ expect.objectContaining({ runs: 1, passes: 0, status: 'fail' }),
+ ]);
+ });
+
+ it('refuses --repeat together with --shard before launching a browser', async () => {
+ await expect(runTests({ repeat: 2, shard: { index: 1, total: 2 } }))
+ .rejects.toThrow(/--repeat cannot be combined with --shard/);
+ expect(puppeteer.launch).not.toHaveBeenCalled();
+ });
+
+ it('refuses --repeat together with recording before launching a browser', async () => {
+ await expect(runTests({ repeat: 2, recordOverrides: { enabled: true } }))
+ .rejects.toThrow(/--repeat cannot be combined with --record/);
+ expect(puppeteer.launch).not.toHaveBeenCalled();
+ expect(assertFfmpegCapable).not.toHaveBeenCalled();
+ });
+
+ it('refuses --repeat when recording is enabled in the config', async () => {
+ vi.mocked(loadConfig).mockReturnValue({
+ ...defaultMockConfig, record: { enabled: true },
+ });
+ await expect(runTests({ repeat: 2 })).rejects.toThrow(
+ /--repeat cannot run with "record\.enabled": true in twd\.config\.json\./,
+ );
+ expect(puppeteer.launch).not.toHaveBeenCalled();
+ });
+
+ it('does not clean the report folder when the config recording refusal throws', async () => {
+ vi.mocked(loadConfig).mockReturnValue({
+ ...defaultMockConfig, record: { enabled: true },
+ });
+ // cleanReportDir would remove these if it ran; a refused run must not.
+ vi.mocked(fs.readdirSync).mockReturnValue(['run.json']);
+
+ await expect(runTests({ repeat: 2 })).rejects.toThrow(/record\.enabled/);
+
+ expect(fs.rmSync).not.toHaveBeenCalled();
+ expect(puppeteer.launch).not.toHaveBeenCalled();
+ });
+
+ it('disables layout snapshots while repeating', async () => {
+ const handlers = [{ id: 't', name: 'stable', type: 'test' }];
+ const page = createMockPage();
+ page.evaluate = vi.fn(async (_fn, _retryCount, ids) => {
+ if (ids === undefined) return handlers;
+ return [{ id: 't', status: 'pass' }];
+ });
+ vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page));
+
+ await runTests({ repeat: 2, updateSnapshots: true, ci: true });
+
+ const [inject, flags] = page.evaluateOnNewDocument.mock.calls[0];
+ expect(flags).toEqual({ update: false, ci: false, snapshots: false });
+ expect(runInjected(inject, flags)).toEqual({ __TWD_SNAPSHOTS__: false });
+ const logs = logSpy.mock.calls.map((c) => String(c[0]));
+ expect(logs.some((l) => l.includes('Skipping layout snapshots (--repeat)'))).toBe(true);
+ });
+
+ it('skips contract validation while repeating', async () => {
+ const handlers = [{ id: 't', name: 'stable', type: 'test' }];
+ const page = createMockPage();
+ page.evaluate = vi.fn(async (_fn, _retryCount, ids) => {
+ if (ids === undefined) return handlers;
+ return [{ id: 't', status: 'pass' }];
+ });
+ vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page));
+ vi.mocked(loadConfig).mockReturnValue({
+ ...defaultMockConfig, contracts: [{ source: 'api.json' }],
+ });
+ vi.mocked(loadContracts).mockResolvedValue([]);
+ vi.mocked(validateMocks).mockReturnValue({ results: [], skipped: [] });
+
+ await runTests({ repeat: 2 });
+
+ expect(loadContracts).not.toHaveBeenCalled();
+ expect(validateMocks).not.toHaveBeenCalled();
+ const logs = logSpy.mock.calls.map((c) => String(c[0]));
+ expect(logs.some((l) => l.includes('Skipping contract validation (--repeat)'))).toBe(true);
+ });
+});
diff --git a/tests/testSummary.test.js b/tests/testSummary.test.js
index 64058b5..9555ac5 100644
--- a/tests/testSummary.test.js
+++ b/tests/testSummary.test.js
@@ -1,5 +1,6 @@
import { describe, it, expect } from 'vitest';
import { formatRunComplete } from '../src/testSummary.js';
+import { buildRunReport } from '../src/runReport.js';
const handlers = [
{ id: 's1', name: 'Login', type: 'suite' },
@@ -250,6 +251,105 @@ describe('formatRunComplete', () => {
});
});
+describe('formatRunComplete with --repeat', () => {
+ const checkout = [
+ { id: 's', name: 'Checkout', type: 'suite' },
+ { id: 't1', name: 'pays', parent: 's', type: 'test' },
+ { id: 't2', name: 'checkout flow', parent: 's', type: 'test' },
+ ];
+
+ it('counts tests and lists a test that both passed and failed as flaky', () => {
+ const block = formatRunComplete({
+ testStatus: [
+ { id: 't1', status: 'pass', runs: 10, passes: 10, skips: 0 },
+ {
+ id: 't2',
+ status: 'fail',
+ runs: 10,
+ passes: 8,
+ skips: 0,
+ error: 'Expected element [data-testid="total"] to have text "42.00"',
+ },
+ ],
+ handlers: checkout,
+ durationMs: 41300,
+ repeat: 10,
+ });
+ expect(block).toBe(
+ '--- Run complete ---\n' +
+ ' Passed: 1 | Failed: 1 | Skipped: 0\n' +
+ ' Duration: 41.3s\n' +
+ ' Repeat: 10x\n' +
+ '\n' +
+ ' Flaky (1):\n' +
+ ' ~ Checkout > checkout flow (8/10 passed)\n' +
+ ' Expected element [data-testid="total"] to have text "42.00"'
+ );
+ });
+
+ it('lists a test that failed every round with its pass count, not as flaky', () => {
+ const block = formatRunComplete({
+ testStatus: [{ id: 't1', status: 'fail', error: 'boom', runs: 10, passes: 0, skips: 0 }],
+ handlers,
+ durationMs: 1000,
+ repeat: 10,
+ });
+ expect(block).toContain(' Passed: 0 | Failed: 1 | Skipped: 0');
+ expect(block).toContain(' Repeat: 10x');
+ expect(block).toContain(' × Login > shows error on wrong password (0/10 passed)');
+ expect(block).toContain(' boom');
+ expect(block).not.toContain('Flaky');
+ });
+
+ it('counts a partly skipped test once, as passed, not by rounds', () => {
+ const block = formatRunComplete({
+ testStatus: [{ id: 't1', status: 'pass', runs: 3, passes: 2, skips: 1 }],
+ handlers,
+ durationMs: 1000,
+ repeat: 3,
+ });
+ expect(block).toContain(' Passed: 1 | Failed: 0 | Skipped: 0');
+ expect(block).not.toContain('Flaky');
+ expect(block).not.toContain('Failed tests');
+ });
+
+ it('reports the same counts as buildRunReport for 3 tests x 10 rounds, one failing 2 rounds', () => {
+ const testStatus = [
+ { id: 't1', status: 'pass', runs: 10, passes: 10, skips: 0 },
+ { id: 't2', status: 'fail', runs: 10, passes: 8, skips: 0, error: 'boom' },
+ { id: 't3', status: 'pass', runs: 10, passes: 10, skips: 0 },
+ ];
+ const block = formatRunComplete({ testStatus, handlers, durationMs: 1000, repeat: 10 });
+ const report = buildRunReport({
+ shard: { index: 1, total: 1 },
+ startedAt: 1000,
+ endedAt: 2000,
+ allTestIds: ['t1', 't2', 't3'],
+ handlers,
+ tests: testStatus,
+ executed: 3,
+ notRun: 0,
+ stoppedEarly: false,
+ repeat: 10,
+ });
+ expect(report.summary).toMatchObject({ passed: 2, failed: 1, skipped: 0 });
+ expect(block).toContain(' Passed: 2 | Failed: 1 | Skipped: 0');
+ });
+
+ it('leaves a single run unchanged when repeat is absent', () => {
+ const block = formatRunComplete({
+ testStatus: [{ id: 't1', status: 'fail', error: 'boom' }],
+ handlers,
+ durationMs: 1000,
+ });
+ expect(block).not.toContain('Repeat:');
+ expect(block).not.toContain('Flaky');
+ expect(block).toContain(' Passed: 0 | Failed: 1 | Skipped: 0');
+ expect(block).toContain(' × Login > shows error on wrong password');
+ expect(block).not.toContain('(0/1 passed)');
+ });
+});
+
describe('formatRunComplete with shards', () => {
const handlers = [
{ id: 's1', name: 'Login', parent: null, type: 'suite' },
diff --git a/tests/usage.test.js b/tests/usage.test.js
index 38a89da..b85976a 100644
--- a/tests/usage.test.js
+++ b/tests/usage.test.js
@@ -29,6 +29,14 @@ describe("runUsage", () => {
expect(runUsage()).toMatch(/--flag=value/);
});
+ it("documents --repeat as a positive integer that cannot combine with record or shard", () => {
+ const text = runUsage();
+ expect(text).toContain('--repeat ');
+ expect(text).toMatch(/positive integer/);
+ expect(text).toMatch(/--record/);
+ expect(text).toMatch(/--shard/);
+ });
+
it("does not describe merge", () => {
expect(runUsage()).not.toMatch(/--out/);
});