diff --git a/.ci/test-count-floors.json b/.ci/test-count-floors.json new file mode 100644 index 00000000..06a714ff --- /dev/null +++ b/.ci/test-count-floors.json @@ -0,0 +1,17 @@ +{ + "floors": { + "core": 199, + "opencode": 1894, + "pi": 114 + }, + "measurement": { + "head": "1b27511a3b59f9ac8f9640d5936cb0ee146925a8", + "dirtyPaths": 0 + }, + "evidence": { + "_note": "How each floor was established. Two runs agreeing proves only that they shared a subject — a contaminated tree reproduces perfectly. A derivation is a second instrument with a different failure mode, so it is independent of the run but NOT of the framework's counting convention: this repo has test.each sites where declaration-counting and runner-counting disagree. Neither is an oracle; agreement between the two is the evidence.", + "core": "measured twice, plus an independent git derivation of the expected delta against another ref", + "opencode": "measured twice, plus a per-file decomposition summing to the observed delta", + "pi": "measured twice only" + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a3d3d7e..e3b72806 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6 + with: + fetch-depth: 0 - uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4 - run: bun install --frozen-lockfile - run: bun run types - run: bun run build + - run: bun test scripts/check-test-count-floors.test.ts + - run: bun scripts/check-test-count-floors.ts --base-ref "${{ github.event.pull_request.base.sha }}" - run: bun run --cwd packages/opencode smoke:tui env: TUI_SMOKE_SKIP_BUILD: "1" diff --git a/scripts/check-test-count-floors.test.ts b/scripts/check-test-count-floors.test.ts new file mode 100644 index 00000000..b66e77d2 --- /dev/null +++ b/scripts/check-test-count-floors.test.ts @@ -0,0 +1,281 @@ +import { afterEach, expect, test } from 'bun:test' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const script = resolve(import.meta.dir, 'check-test-count-floors.ts') +const workspaces: string[] = [] + +type Floors = Record<'core' | 'opencode' | 'pi', number> + +type LoweringMarker = { + reason: string + lowering: Partial< + Record<'core' | 'opencode' | 'pi', { from: number; to: number }> + > +} + +function runGit(cwd: string, args: string[]) { + const result = Bun.spawnSync(['git', ...args], { + cwd, + env: { + ...process.env, + GIT_AUTHOR_NAME: 'Test Runner', + GIT_AUTHOR_EMAIL: 'test@example.com', + GIT_COMMITTER_NAME: 'Test Runner', + GIT_COMMITTER_EMAIL: 'test@example.com', + }, + stdout: 'pipe', + stderr: 'pipe', + }) + if (result.exitCode !== 0) { + throw new Error(new TextDecoder().decode(result.stderr)) + } + return new TextDecoder().decode(result.stdout).trim() +} + +async function writeFloors( + cwd: string, + floors: Floors, + measurement: { head: string; dirtyPaths: number }, +) { + await mkdir(join(cwd, '.ci'), { recursive: true }) + await writeFile( + join(cwd, '.ci', 'test-count-floors.json'), + `${JSON.stringify({ floors, measurement }, null, 2)}\n`, + ) +} + +async function makeStaleBranch( + baseFloors: Floors, + staleFloors: Floors, + marker?: LoweringMarker, +) { + const cwd = await mkdtemp(join(tmpdir(), 'test-count-floor-')) + workspaces.push(cwd) + + runGit(cwd, ['init', '--initial-branch=main']) + runGit(cwd, ['commit', '--allow-empty', '-m', 'seed measurement subject']) + const staleMeasurementHead = runGit(cwd, ['rev-parse', 'HEAD']) + await writeFloors(cwd, staleFloors, { + head: staleMeasurementHead, + dirtyPaths: 0, + }) + runGit(cwd, ['add', '.ci/test-count-floors.json']) + runGit(cwd, ['commit', '-m', 'record stale floors']) + runGit(cwd, ['branch', 'stale']) + + if (JSON.stringify(baseFloors) !== JSON.stringify(staleFloors)) { + await writeFloors(cwd, baseFloors, { + head: runGit(cwd, ['rev-parse', 'HEAD']), + dirtyPaths: 0, + }) + runGit(cwd, ['add', '.ci/test-count-floors.json']) + runGit(cwd, ['commit', '-m', 'raise main floor']) + } + runGit(cwd, ['checkout', 'stale']) + + if (marker) { + await writeFile( + join(cwd, '.ci', 'allow-test-count-floor-lowering.json'), + `${JSON.stringify(marker, null, 2)}\n`, + ) + runGit(cwd, ['add', '.ci/allow-test-count-floor-lowering.json']) + runGit(cwd, ['commit', '-m', 'authorize floor lowering']) + } + + return cwd +} + +function runGate(cwd: string, counts: Floors, baseRef = 'main') { + const result = Bun.spawnSync( + [ + 'bun', + script, + '--floor-file', + '.ci/test-count-floors.json', + '--base-ref', + baseRef, + '--counts', + JSON.stringify(counts), + ], + { cwd, stdout: 'pipe', stderr: 'pipe' }, + ) + return { + exitCode: result.exitCode, + output: `${new TextDecoder().decode(result.stdout)}${new TextDecoder().decode(result.stderr)}`, + } +} + +function runGateWithCounts(cwd: string, counts: string, baseRef = 'main') { + const result = Bun.spawnSync( + [ + 'bun', + script, + '--floor-file', + '.ci/test-count-floors.json', + '--base-ref', + baseRef, + '--counts', + counts, + ], + { cwd, stdout: 'pipe', stderr: 'pipe' }, + ) + return { + exitCode: result.exitCode, + output: `${new TextDecoder().decode(result.stdout)}${new TextDecoder().decode(result.stderr)}`, + } +} + +afterEach(async () => { + await Promise.all( + workspaces.splice(0).map((cwd) => rm(cwd, { recursive: true })), + ) +}) + +test('passes when counts equal the branch floors', async () => { + const floors = { core: 10, opencode: 20, pi: 30 } + const cwd = await makeStaleBranch(floors, floors) + const result = runGate(cwd, floors) + + expect(result.exitCode).toBe(0) + expect(result.output).toContain( + 'VERDICT: PASS packages=core,opencode,pi (test counts and floor ratchet satisfied)', + ) +}) + +test('passes when counts exceed the branch floors', async () => { + const floors = { core: 10, opencode: 20, pi: 30 } + const cwd = await makeStaleBranch(floors, floors) + const result = runGate(cwd, { core: 11, opencode: 21, pi: 31 }) + + expect(result.exitCode).toBe(0) + expect(result.output).toContain('VERDICT: PASS') +}) + +test('fails when a measured count falls below its branch floor', async () => { + const floors = { core: 10, opencode: 20, pi: 30 } + const cwd = await makeStaleBranch(floors, floors) + const result = runGate(cwd, { core: 9, opencode: 20, pi: 30 }) + + expect(result.exitCode).toBe(1) + expect(result.output).toContain('core measured 9 < branch floor 10') + expect(result.output).toContain('VERDICT: FAIL packages=core,opencode,pi') +}) + +test('fails when the branch floor is lower than the merge target floor', async () => { + const cwd = await makeStaleBranch( + { core: 11, opencode: 20, pi: 30 }, + { core: 10, opencode: 20, pi: 30 }, + ) + const result = runGate(cwd, { core: 10, opencode: 20, pi: 30 }) + + expect(result.exitCode).toBe(1) + expect(result.output).toContain( + 'core branch floor 10 < merge target floor 11', + ) +}) + +test('passes an explicit lowering marker that names the target floor', async () => { + const cwd = await makeStaleBranch( + { core: 11, opencode: 20, pi: 30 }, + { core: 10, opencode: 20, pi: 30 }, + { + reason: 'The core suite intentionally removed obsolete coverage.', + lowering: { core: { from: 11, to: 10 } }, + }, + ) + const result = runGate(cwd, { core: 10, opencode: 20, pi: 30 }) + + expect(result.exitCode).toBe(0) + expect(result.output).toContain('deliberate lowering authorized') +}) + +test('reports an unchecked non-zero verdict when the merge target is unavailable', async () => { + const floors = { core: 10, opencode: 20, pi: 30 } + const cwd = await makeStaleBranch(floors, floors) + const result = runGate(cwd, floors, 'missing-target') + + expect(result.exitCode).toBe(2) + expect(result.output).toContain( + 'VERDICT: UNCHECKED packages=core,opencode,pi', + ) +}) + +test('reports an unchecked verdict when CI supplies an empty merge target', async () => { + const floors = { core: 10, opencode: 20, pi: 30 } + const cwd = await makeStaleBranch(floors, floors) + const result = runGate(cwd, floors, '') + + expect(result.exitCode).toBe(2) + expect(result.output).toContain( + 'VERDICT: UNCHECKED packages=core,opencode,pi', + ) +}) + +test('reports an unchecked verdict for a floor stamped by an unrelated commit', async () => { + const staleFloors = { core: 10, opencode: 20, pi: 30 } + const cwd = await makeStaleBranch( + { core: 11, opencode: 20, pi: 30 }, + staleFloors, + ) + await writeFloors(cwd, staleFloors, { + head: runGit(cwd, ['rev-parse', 'main']), + dirtyPaths: 0, + }) + const result = runGate(cwd, staleFloors) + + expect(result.exitCode).toBe(2) + expect(result.output).toContain('VERDICT: UNCHECKED') + expect(result.output).toContain('not an ancestor') +}) + +test('allows the gate to run from a dirty working tree', async () => { + const floors = { core: 10, opencode: 20, pi: 30 } + const cwd = await makeStaleBranch(floors, floors) + await writeFile(join(cwd, 'uncommitted-note'), 'local work is allowed\n') + const result = runGate(cwd, floors) + + expect(result.exitCode).toBe(0) + expect(result.output).toContain('VERDICT: PASS') +}) + +test('fails as a noncompliant source when the branch floor file is absent', async () => { + const floors = { core: 10, opencode: 20, pi: 30 } + const cwd = await makeStaleBranch(floors, floors) + await rm(join(cwd, '.ci', 'test-count-floors.json')) + const result = runGate(cwd, floors) + + expect(result.exitCode).toBe(1) + expect(result.output).toContain( + 'VERDICT: FAIL packages=none (NONCOMPLIANT SOURCE', + ) +}) + +test('does not pass when no packages are evaluated', async () => { + const floors = { core: 10, opencode: 20, pi: 30 } + const cwd = await makeStaleBranch(floors, floors) + const result = runGateWithCounts(cwd, '{}') + + expect(result.exitCode).toBe(1) + expect(result.output).toContain('VERDICT: FAIL packages=none') + expect(result.output).not.toContain('VERDICT: PASS') +}) + +test('replays a stale branch after main raises the floor and rejects the replay', async () => { + const cwd = await makeStaleBranch( + { core: 11, opencode: 20, pi: 30 }, + { core: 10, opencode: 20, pi: 30 }, + ) + const branch = Bun.spawnSync(['git', 'branch', '--show-current'], { + cwd, + stdout: 'pipe', + }) + const result = runGate(cwd, { core: 10, opencode: 20, pi: 30 }) + + expect(new TextDecoder().decode(branch.stdout).trim()).toBe('stale') + expect(result.exitCode).toBe(1) + expect(result.output).toContain( + 'core branch floor 10 < merge target floor 11', + ) +}) diff --git a/scripts/check-test-count-floors.ts b/scripts/check-test-count-floors.ts new file mode 100644 index 00000000..2695c44a --- /dev/null +++ b/scripts/check-test-count-floors.ts @@ -0,0 +1,426 @@ +import { existsSync } from 'node:fs' +import { readFile } from 'node:fs/promises' +import { isAbsolute, join } from 'node:path' + +const packageNames = ['core', 'opencode', 'pi'] as const +type PackageName = (typeof packageNames)[number] +type Floors = Record + +type LoweringMarker = { + reason: string + lowering: Partial> +} + +type FloorDocument = { + floors: Floors + measurement: { + head: string + dirtyPaths: number + } +} + +type Options = { + baseRef?: string + counts?: Floors + floorFile: string +} + +const decoder = new TextDecoder() + +function packageScope(value: object | undefined): string { + const packages = value ? Object.keys(value).sort() : [] + return packages.length > 0 ? packages.join(',') : 'none' +} + +function verdict(status: string, scope: string, detail: string): string { + return `VERDICT: ${status} packages=${scope} (${detail})` +} + +function fail(message: string): never { + throw new Error(message) +} + +function parseFloors(raw: string, source: string): Floors { + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch { + fail(`${source} is not valid JSON`) + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + fail(`${source} must be an object`) + } + + const record = parsed as Record + const keys = Object.keys(record).sort() + if (keys.join(',') !== [...packageNames].sort().join(',')) { + fail(`${source} must contain exactly: ${packageNames.join(', ')}`) + } + + const floors = {} as Floors + for (const packageName of packageNames) { + const value = record[packageName] + if ( + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value <= 0 + ) { + fail(`${source}.${packageName} must be a positive integer`) + } + floors[packageName] = value + } + return floors +} + +function parseFloorDocument(raw: string, source: string): FloorDocument { + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch { + fail(`${source} is not valid JSON`) + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + fail(`${source} must be an object`) + } + + const document = parsed as { floors?: unknown; measurement?: unknown } + if (!document.floors || typeof document.floors !== 'object') { + fail(`${source}.floors must be an object`) + } + if (!document.measurement || typeof document.measurement !== 'object') { + fail(`${source}.measurement must be an object`) + } + const measurement = document.measurement as { + head?: unknown + dirtyPaths?: unknown + } + if ( + typeof measurement.head !== 'string' || + !/^[0-9a-f]{40}$/.test(measurement.head) + ) { + fail(`${source}.measurement.head must be a 40-hex SHA`) + } + const dirtyPaths = measurement.dirtyPaths + if ( + typeof dirtyPaths !== 'number' || + !Number.isSafeInteger(dirtyPaths) || + dirtyPaths < 0 + ) { + fail(`${source}.measurement.dirtyPaths must be a non-negative integer`) + } + return { + floors: parseFloors(JSON.stringify(document.floors), `${source}.floors`), + measurement: { + head: measurement.head, + dirtyPaths, + }, + } +} + +function parseOptions(args: string[]): Options { + const options: Options = { floorFile: '.ci/test-count-floors.json' } + for (let index = 0; index < args.length; index += 1) { + const flag = args[index] + const value = args[index + 1] + if (value === undefined || value.startsWith('--')) { + fail(`missing value for ${flag}`) + } + + if (flag === '--base-ref') options.baseRef = value + else if (flag === '--counts') + options.counts = parseFloors(value, '--counts') + else if (flag === '--floor-file') options.floorFile = value + else fail(`unknown argument: ${flag}`) + index += 1 + } + if (isAbsolute(options.floorFile)) + fail('--floor-file must be relative to the repository root') + return options +} + +async function readFloorDocument(floorFile: string): Promise { + return parseFloorDocument(await readFile(floorFile, 'utf8'), floorFile) +} + +function readMergeTargetFloors( + baseRef: string, + floorFile: string, +): FloorDocument | undefined { + const result = Bun.spawnSync(['git', 'show', `${baseRef}:${floorFile}`], { + stdout: 'pipe', + stderr: 'pipe', + }) + if (result.exitCode !== 0) return undefined + return parseFloorDocument( + decoder.decode(result.stdout), + `${baseRef}:${floorFile}`, + ) +} + +function stampError( + document: FloorDocument, + subject: string, + label: string, +): string | undefined { + if (document.measurement.dirtyPaths !== 0) { + return `${label} floor was measured with ${document.measurement.dirtyPaths} dirty paths` + } + const result = Bun.spawnSync( + ['git', 'merge-base', '--is-ancestor', document.measurement.head, subject], + { stdout: 'pipe', stderr: 'pipe' }, + ) + if (result.exitCode === 0) return undefined + if (result.exitCode === 1) { + return `${label} measurement head ${document.measurement.head} is not an ancestor of ${subject}` + } + return `could not validate ${label} measurement head ${document.measurement.head} against ${subject}` +} + +async function readLoweringMarker(): Promise { + const markerFile = '.ci/allow-test-count-floor-lowering.json' + if (!existsSync(markerFile)) return undefined + + let parsed: unknown + try { + parsed = JSON.parse(await readFile(markerFile, 'utf8')) + } catch { + fail(`${markerFile} is not valid JSON`) + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + fail(`${markerFile} must be an object`) + } + + const marker = parsed as { + reason?: unknown + lowering?: unknown + } + if (typeof marker.reason !== 'string' || marker.reason.trim().length === 0) { + fail(`${markerFile}.reason must be a non-empty string`) + } + if ( + !marker.lowering || + typeof marker.lowering !== 'object' || + Array.isArray(marker.lowering) + ) { + fail(`${markerFile}.lowering must be an object`) + } + + const lowering = marker.lowering as Record + const parsedLowering: LoweringMarker['lowering'] = {} + for (const [packageName, values] of Object.entries(lowering)) { + if (!packageNames.includes(packageName as PackageName)) { + fail(`${markerFile}.lowering has an unknown package: ${packageName}`) + } + if (!values || typeof values !== 'object' || Array.isArray(values)) { + fail(`${markerFile}.lowering.${packageName} must be an object`) + } + const { from, to } = values as { from?: unknown; to?: unknown } + if ( + typeof from !== 'number' || + typeof to !== 'number' || + !Number.isSafeInteger(from) || + !Number.isSafeInteger(to) + ) { + fail( + `${markerFile}.lowering.${packageName} requires integer from and to values`, + ) + } + parsedLowering[packageName as PackageName] = { from, to } + } + return { reason: marker.reason, lowering: parsedLowering } +} + +function markerError( + marker: LoweringMarker | undefined, + lowered: PackageName[], + branchFloors: Floors, + targetFloors: Floors, +): string | undefined { + if (!marker) return 'no deliberate-lowering marker' + + const marked = Object.keys(marker.lowering).sort() + if (marked.join(',') !== [...lowered].sort().join(',')) { + return `marker must name exactly: ${lowered.join(', ')}` + } + for (const packageName of lowered) { + const entry = marker.lowering[packageName] + if ( + !entry || + entry.from !== targetFloors[packageName] || + entry.to !== branchFloors[packageName] + ) { + return `${packageName} marker must declare from ${targetFloors[packageName]} to ${branchFloors[packageName]}` + } + } + return undefined +} + +function measureTests(): Floors { + const commands: Record = { + core: ['bun', 'test', 'src/tests'], + opencode: ['bun', 'test', 'src/tests'], + pi: ['bun', 'test', 'src/tests'], + } + const floors = {} as Floors + + for (const packageName of packageNames) { + console.log(`MEASURE: ${packageName}`) + const result = Bun.spawnSync(commands[packageName], { + cwd: join('packages', packageName), + stdout: 'pipe', + stderr: 'pipe', + }) + const output = `${decoder.decode(result.stdout)}${decoder.decode(result.stderr)}` + if (result.exitCode !== 0) { + process.stdout.write(output) + fail(`${packageName} test command exited ${result.exitCode}`) + } + const match = output.match(/^\s*(\d+) pass\s*$/m) + if (!match) fail(`could not parse ${packageName} test count`) + floors[packageName] = Number(match[1]) + console.log(`MEASURED: ${packageName} ${floors[packageName]}`) + } + return floors +} + +async function main() { + const options = parseOptions(process.argv.slice(2)) + let branchDocument: FloorDocument + try { + branchDocument = await readFloorDocument(options.floorFile) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + console.error( + verdict( + 'FAIL', + 'none', + `NONCOMPLIANT SOURCE: branch floor file is missing: ${options.floorFile}`, + ), + ) + process.exitCode = 1 + return + } + throw error + } + if (!options.baseRef) { + console.error( + verdict( + 'UNCHECKED', + packageScope(branchDocument.floors), + 'merge target ref was not provided', + ), + ) + process.exitCode = 2 + return + } + + const targetDocument = readMergeTargetFloors( + options.baseRef, + options.floorFile, + ) + if (!targetDocument) { + console.error( + verdict( + 'UNCHECKED', + packageScope(branchDocument.floors), + `could not resolve merge target floor ${options.baseRef}:${options.floorFile}`, + ), + ) + process.exitCode = 2 + return + } + + const branchStampError = stampError(branchDocument, 'HEAD', 'branch') + if (branchStampError) { + console.error( + verdict( + 'UNCHECKED', + packageScope(branchDocument.floors), + branchStampError, + ), + ) + process.exitCode = 2 + return + } + const targetStampError = stampError( + targetDocument, + options.baseRef, + 'merge target', + ) + if (targetStampError) { + console.error( + verdict( + 'UNCHECKED', + packageScope(branchDocument.floors), + targetStampError, + ), + ) + process.exitCode = 2 + return + } + + const counts = options.counts ?? measureTests() + const scope = packageScope(counts) + if (scope === 'none') { + console.error(verdict('UNCHECKED', scope, 'no packages were evaluated')) + process.exitCode = 2 + return + } + const branchFloors = branchDocument.floors + const targetFloors = targetDocument.floors + const failures: string[] = [] + for (const packageName of packageNames) { + if (counts[packageName] < branchFloors[packageName]) { + failures.push( + `COUNT: ${packageName} measured ${counts[packageName]} < branch floor ${branchFloors[packageName]}`, + ) + } else { + console.log( + `COUNT: ${packageName} measured ${counts[packageName]} >= branch floor ${branchFloors[packageName]}`, + ) + } + } + + const lowered = packageNames.filter( + (packageName) => branchFloors[packageName] < targetFloors[packageName], + ) + const marker = await readLoweringMarker() + const invalidMarker = markerError(marker, lowered, branchFloors, targetFloors) + for (const packageName of packageNames) { + if (lowered.includes(packageName)) continue + console.log( + `RATCHET: ${packageName} branch floor ${branchFloors[packageName]} >= merge target floor ${targetFloors[packageName]}`, + ) + } + if (lowered.length > 0 && invalidMarker) { + for (const packageName of lowered) { + failures.push( + `RATCHET: ${packageName} branch floor ${branchFloors[packageName]} < merge target floor ${targetFloors[packageName]} (${invalidMarker})`, + ) + } + } else if (lowered.length > 0 && marker) { + for (const packageName of lowered) { + console.log( + `RATCHET: ${packageName} branch floor ${branchFloors[packageName]} < merge target floor ${targetFloors[packageName]}; deliberate lowering authorized: ${marker.reason}`, + ) + } + } + + if (failures.length > 0) { + for (const failure of failures) console.error(failure) + console.error(verdict('FAIL', scope, 'test-count floor check failed')) + process.exitCode = 1 + return + } + console.log(verdict('PASS', scope, 'test counts and floor ratchet satisfied')) +} + +try { + await main() +} catch (error) { + console.error(error instanceof Error ? error.message : error) + console.error( + verdict('FAIL', 'none', 'test-count floor check rejected invalid input'), + ) + process.exitCode = 1 +}