diff --git a/CHANGELOG.md b/CHANGELOG.md index 4161dae5..463c0a42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,6 @@ on the evidence it recorded — passed only if every expected outcome was already met — while its generated code, screencast, and Testomatio steps are still saved. The session report notes that the time budget was reached. -### Changes - - Prima: every command now prints `### Artifacts` naming the files it just wrote — the ARIA tree and the html always, plus the screenshot and network log when they were captured. Until now those files were written on every command but only named when a `check` came back with a CONTRADICTION, so the @@ -37,6 +35,15 @@ finished. It also looks the hash up across every recorded site instead of assuming the most recent one, and lists everything else kept under the hash. A hash that really is missing now reports the directory it looked in. +- `explorbot test ` takes the site to run against from the plan itself — the URL of its + `### Prerequisite` section, or the `## Requirements` URL of its first test. A plan is therefore + enough to run it from any directory with a global installation (`~/.explorbot/config.js`) or with + the `EXPLORBOT_*` variables, both of which used to refuse to start with "No site to explore" + because the command named no URL of its own. + The plan is looked up the same way whichever name it is given — a path, or the bare file name of a + saved plan, which is searched for in the plans directory of every registered site. +- `explorbot test ` without an index runs every enabled test in the plan, as the help and + the docs already described. It used to run only the first pending one. - [Fisherman] A run that gives up no longer counts as prepared data. When the run ran out of iterations or stopped after repeated failures on one endpoint, a single successful write was enough to report success, so a half-built precondition looked ready to the test. Only a run the diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index 8827988c..71a0226f 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -260,14 +260,13 @@ addCommonOptions(program.command('plan ').description('Generate test plan addCommonOptions(program.command('plan:load [index]').description('Load a plan file and display its tests. Pass index to see test details.')).action(async (planfile: string, index: string | undefined) => { try { - const resolvedPath = path.resolve(planfile); - if (!fs.existsSync(resolvedPath)) { - console.error(`Plan file not found: ${resolvedPath}`); + const plan = Plan.loadFromFile(planfile); + if (!plan?.filePath) { + console.error(`Plan file not found: ${planfile}`); process.exit(1); } - const plan = Plan.fromMarkdown(resolvedPath); - const planFile = path.basename(resolvedPath); + const planFile = path.basename(plan.filePath); if (index) { const idx = Number.parseInt(index, 10); @@ -279,7 +278,7 @@ addCommonOptions(program.command('plan:load [index]').description('Lo const lines: string[] = []; lines.push(`## #${idx} ${test.scenario}\n`); lines.push(`**Priority:** ${test.priority}`); - const planUrl = plan.url || plan.tests[0]?.startUrl; + const planUrl = plan.startUrl; if (planUrl) lines.push(`**Plan URL:** ${planUrl}`); if (test.startUrl && test.startUrl !== planUrl) lines.push(`**Test URL:** ${test.startUrl}`); if (test.plannedSteps.length) { @@ -296,7 +295,7 @@ addCommonOptions(program.command('plan:load [index]').description('Lo return; } - const planUrl = plan.url || plan.tests[0]?.startUrl; + const planUrl = plan.startUrl; const lines: string[] = [`**${plan.title}** (${plan.tests.length} tests)\n`]; if (planUrl) { lines.push(`URL: ${planUrl}\n`); @@ -328,9 +327,6 @@ addCommonOptions(program.command('plan:load [index]').description('Lo addCommonOptions(program.command('test [index]').description('Execute tests from a plan file. Index: 1, 1,3, 1-5, *, all').option('--grep ', 'Run tests matching pattern').option('--from-plan ', 'Load plan file when the first argument is a test index')).action( async (planfile, index, options) => { try { - const explorBot = new ExplorBot(buildExplorBotOptions(undefined, options)); - await explorBot.start(); - let planfileArg = planfile; let indexArg = index; if (options.fromPlan) { @@ -338,11 +334,16 @@ addCommonOptions(program.command('test [index]').description('Execute indexArg = planfile; } + const planTarget = Plan.loadFromFile(planfileArg)?.startUrl; + + const explorBot = new ExplorBot(buildExplorBotOptions(planTarget, options)); + await explorBot.start(); + const plan = explorBot.loadPlan(planfileArg); const pending = plan.getPendingTests(); log(`Plan loaded: "${plan.title}" (${plan.tests.length} tests, ${pending.length} pending)`); - const startUrl = plan.url || pending[0]?.startUrl; + const startUrl = plan.startUrl; if (!startUrl) { throw new Error('No URL found in plan or tests. Cannot determine where to navigate.'); } @@ -350,7 +351,7 @@ addCommonOptions(program.command('test [index]').description('Execute log(`Navigating to ${startUrl}`); await explorBot.visit(startUrl); - let args = ''; + let args = '*'; if (indexArg) args = indexArg; else if (options.grep) args = options.grep; diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 23aeefeb..12b91bdb 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -434,6 +434,8 @@ npx explorbot test 3 --from-plan output/plans/login.md # index first, plan via | `--grep ` | Run only tests whose scenario matches the pattern | | `--from-plan ` | Load this plan file when the first argument is a test index | +The plan names the site it runs against: the URL of its `### Prerequisite` section, or the `## Requirements` URL of its first test. With a [global installation](configuration.md#running-from-anywhere-the-global-installation) that is enough to run a plan from any directory without a project config — `npx explorbot test ~/plans/checkout.md` registers the site and stores its output under `~/.explorbot/sites//`. Naming a saved plan is enough too: `npx explorbot test checkout` looks for `checkout.md` in the current directory, then in the plans directory of every registered site. + ### drill Drill all components on a page to learn interactions. diff --git a/src/ai/historian/codeceptjs.ts b/src/ai/historian/codeceptjs.ts index b48e3401..b1d8f064 100644 --- a/src/ai/historian/codeceptjs.ts +++ b/src/ai/historian/codeceptjs.ts @@ -64,7 +64,7 @@ export function WithCodeceptJS(Base: T) { lines.push(`Feature('${escapeString(plan.title)}')`); lines.push(''); - const startUrl = plan.url || plan.tests[0]?.startUrl; + const startUrl = plan.startUrl; if (startUrl) { lines.push('Before(({ I }) => {'); lines.push(` I.amOnPage('${escapeString(startUrl)}');`); diff --git a/src/ai/historian/playwright.ts b/src/ai/historian/playwright.ts index df8b7e7b..9cffc5b4 100644 --- a/src/ai/historian/playwright.ts +++ b/src/ai/historian/playwright.ts @@ -98,7 +98,7 @@ export function WithPlaywright(Base: T) { lines.push(''); lines.push(`test.describe('${escapeString(plan.title)}', () => {`); - const startUrl = plan.url || plan.tests[0]?.startUrl; + const startUrl = plan.startUrl; if (startUrl) { lines.push(' test.beforeEach(async ({ page }) => {'); lines.push(` await page.goto('${escapeString(startUrl)}');`); diff --git a/src/commands/plans-command.ts b/src/commands/plans-command.ts index e5847e97..953068f7 100644 --- a/src/commands/plans-command.ts +++ b/src/commands/plans-command.ts @@ -79,15 +79,15 @@ export class PlansCommand extends BaseCommand { return file; } - const resolved = this.explorBot.resolvePlanPath(target); - if (!existsSync(resolved)) { - throw new Error(`Plan file not found: ${resolved}`); + const plan = Plan.loadFromFile(target, this.explorBot.getPlansDir()); + if (!plan?.filePath) { + throw new Error(`Plan file not found: ${target}`); } return { - name: path.basename(resolved), - path: resolved, - modifiedAt: statSync(resolved).mtimeMs, + name: path.basename(plan.filePath), + path: plan.filePath, + modifiedAt: statSync(plan.filePath).mtimeMs, }; } } diff --git a/src/explorbot.ts b/src/explorbot.ts index c921ebc0..94b37a8b 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -457,41 +457,17 @@ export class ExplorBot { return urlPart + featurePart.slice(0, maxFeatureLen) + suffix; } - resolvePlanPath(filename: string): string { - let planPath = filename; - - if (path.isAbsolute(filename)) { - if (!existsSync(planPath) && !filename.endsWith('.md')) { - planPath = `${filename}.md`; - } - } else if (existsSync(filename) || existsSync(`${filename}.md`)) { - planPath = existsSync(filename) ? filename : `${filename}.md`; - } else { - const plansDir = this.getPlansDir(); - planPath = path.join(plansDir, filename); - if (!existsSync(planPath) && !filename.endsWith('.md')) { - planPath = path.join(plansDir, `${filename}.md`); - } - } - - return planPath; - } - loadPlan(filename: string): Plan { - const planPath = this.resolvePlanPath(filename); - if (!existsSync(planPath)) { - throw new Error(`Plan file not found: ${planPath}`); - } - this.setCurrentPlan(Plan.fromMarkdown(planPath)); - return this.currentPlan!; + const plan = Plan.loadFromFile(filename, this.getPlansDir()); + if (!plan) throw new Error(`Plan file not found: ${filename}`); + this.setCurrentPlan(plan); + return plan; } loadPlans(filename: string): Plan[] { - const planPath = this.resolvePlanPath(filename); - if (!existsSync(planPath)) { - throw new Error(`Plan file not found: ${planPath}`); - } - return parsePlansFromMarkdown(planPath); + const plan = Plan.loadFromFile(filename, this.getPlansDir()); + if (!plan?.filePath) throw new Error(`Plan file not found: ${filename}`); + return parsePlansFromMarkdown(plan.filePath); } setCurrentPlan(plan?: Plan): void { diff --git a/src/test-plan.ts b/src/test-plan.ts index 0d32f288..beeade70 100644 --- a/src/test-plan.ts +++ b/src/test-plan.ts @@ -1,6 +1,9 @@ import { createHash } from 'node:crypto'; +import { existsSync } from 'node:fs'; +import path from 'node:path'; import figures from 'figures'; import type { ActionResult } from './action-result.ts'; +import { listSites } from './global-config.ts'; import { WebPageState } from './state-manager.ts'; import { tag } from './utils/logger.ts'; import { parsePlanFromMarkdown, planToAiContext, savePlanToMarkdown, savePlansToMarkdown } from './utils/test-plan-markdown.ts'; @@ -388,12 +391,15 @@ export class Test extends Task { } } +const SITE_PLANS_DIR = ['output', 'plans']; + type PlanChangeListener = (tests: Test[]) => void; export class Plan { title: string; tests: Test[] = []; url?: string; + filePath?: string; iteration = 0; parentPlan?: Plan; private changeListeners: PlanChangeListener[] = []; @@ -455,6 +461,10 @@ export class Plan { return this.tests.filter((test) => test.status === 'pending' && test.enabled); } + get startUrl(): string | undefined { + return this.url || this.tests[0]?.startUrl; + } + get isComplete(): boolean { return this.tests.length > 0 && this.tests.every((test) => test.hasFinished); } @@ -469,6 +479,25 @@ export class Plan { updateStatus(): void {} + static loadFromFile(file: string, plansDir?: string): Plan | null { + const names = [file]; + if (!file.endsWith('.md')) names.push(`${file}.md`); + + const dirs = [process.cwd()]; + if (plansDir) dirs.push(plansDir); + if (!plansDir) dirs.push(...listSites().map((site) => path.join(site.dir, ...SITE_PLANS_DIR))); + + for (const dir of dirs) { + const filePath = names.map((name) => path.resolve(dir, name)).find(existsSync); + if (!filePath) continue; + const loaded = parsePlanFromMarkdown(filePath); + loaded.filePath = filePath; + return loaded; + } + + return null; + } + static fromMarkdown(filePath: string): Plan { return parsePlanFromMarkdown(filePath); } diff --git a/tests/unit/plans-command.test.ts b/tests/unit/plans-command.test.ts index 34f2f3e2..4c2221ff 100644 --- a/tests/unit/plans-command.test.ts +++ b/tests/unit/plans-command.test.ts @@ -77,7 +77,6 @@ describe('TestCommand', () => { function createMockExplorBot(overrides: Partial = {}): ExplorBot { return { getPlansDir: () => tmpPath, - resolvePlanPath: (filename: string) => path.join(tmpPath, filename), ...overrides, } as unknown as ExplorBot; } diff --git a/tests/unit/test-plan.test.ts b/tests/unit/test-plan.test.ts index 03b7b23c..7233d66a 100644 --- a/tests/unit/test-plan.test.ts +++ b/tests/unit/test-plan.test.ts @@ -1,6 +1,8 @@ -import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; -import { unlinkSync, writeFileSync } from 'node:fs'; +import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test'; +import { mkdirSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'; +import os, { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { registerSite } from '../../src/global-config.ts'; import { Plan, Test } from '../../src/test-plan.ts'; describe('Plan', () => { @@ -340,4 +342,134 @@ priority: low expect(context).toContain('- Enter text'); }); }); + + describe('startUrl', () => { + test('should take the prerequisite URL of the suite', () => { + const markdown = ` +# Test Suite + +### Prerequisite + +* URL: https://app.example.com/projects/demo/runs + + +# Test Scenario + +## Requirements +https://app.example.com/projects/demo/runs + +## Expected +* Page is rendered +`; + + writeFileSync(testFilePath, markdown, 'utf-8'); + const plan = Plan.fromMarkdown(testFilePath); + + expect(plan.startUrl).toBe('https://app.example.com/projects/demo/runs'); + }); + + test('should fall back to the first test URL when suite has no prerequisite', () => { + const markdown = ` +# Test Suite + + +# Test Scenario + +## Requirements +/login + +## Expected +* Login form is shown +`; + + writeFileSync(testFilePath, markdown, 'utf-8'); + const plan = Plan.fromMarkdown(testFilePath); + + expect(plan.url).toBeUndefined(); + expect(plan.startUrl).toBe('/login'); + }); + + test('should be undefined when neither suite nor tests carry a URL', () => { + expect(new Plan('Test Suite').startUrl).toBeUndefined(); + }); + }); + + describe('loadFromFile', () => { + let home: string; + let workDir: string; + let plansDir: string; + let originalCwd: string; + let homedirSpy: ReturnType; + + const writePlan = (dir: string, name: string): string => { + mkdirSync(dir, { recursive: true }); + const file = join(dir, name); + writeFileSync(file, '# Plan\n', 'utf-8'); + return file; + }; + + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'explorbot-home-')); + workDir = mkdtempSync(join(tmpdir(), 'explorbot-work-')); + plansDir = join(workDir, 'output', 'plans'); + homedirSpy = spyOn(os, 'homedir').mockReturnValue(home); + originalCwd = process.cwd(); + process.chdir(workDir); + }); + + afterEach(() => { + process.chdir(originalCwd); + homedirSpy.mockRestore(); + rmSync(home, { recursive: true, force: true }); + rmSync(workDir, { recursive: true, force: true }); + }); + + test('loads an absolute path', () => { + const file = writePlan(plansDir, 'saved.md'); + expect(Plan.loadFromFile(file)?.filePath).toBe(file); + }); + + test('appends .md to an absolute path', () => { + const file = writePlan(plansDir, 'saved.md'); + expect(Plan.loadFromFile(join(plansDir, 'saved'))?.filePath).toBe(file); + }); + + test('loads a plan named in the working directory', () => { + writePlan(workDir, 'saved.md'); + expect(Plan.loadFromFile('saved')?.filePath).toBe(join(workDir, 'saved.md')); + expect(Plan.loadFromFile('saved.md')?.filePath).toBe(join(workDir, 'saved.md')); + }); + + test('loads a plan named in the plans directory', () => { + const file = writePlan(plansDir, 'saved.md'); + expect(Plan.loadFromFile('saved', plansDir)?.filePath).toBe(file); + }); + + test('prefers the working directory over the plans directory', () => { + writePlan(plansDir, 'saved.md'); + writePlan(workDir, 'saved.md'); + expect(Plan.loadFromFile('saved', plansDir)?.filePath).toBe(join(workDir, 'saved.md')); + }); + + test('loads a plan saved for a registered site when no plans directory is known', () => { + const site = registerSite('https://app.example.com'); + const file = writePlan(join(site.dir, 'output', 'plans'), 'saved.md'); + expect(Plan.loadFromFile('saved')?.filePath).toBe(file); + }); + + test('ignores registered sites once a plans directory is known', () => { + const site = registerSite('https://app.example.com'); + writePlan(join(site.dir, 'output', 'plans'), 'saved.md'); + expect(Plan.loadFromFile('saved', plansDir)).toBeNull(); + }); + + test('returns null when the plan is nowhere to be found', () => { + expect(Plan.loadFromFile('missing', plansDir)).toBeNull(); + expect(Plan.loadFromFile('missing')).toBeNull(); + }); + }); });