diff --git a/.craft.yml b/.craft.yml index db78443..cb32d6e 100644 --- a/.craft.yml +++ b/.craft.yml @@ -1,11 +1,13 @@ minVersion: "2.21.0" changelog: policy: auto -# Nothing reads a version from a file. The git tag is the version. -preReleaseCommand: "" -artifactProvider: - name: none +# Without a preReleaseCommand, Craft writes the version to package.json +# for the npm target. targets: + # The client and the vitest integration. CI uploads the tarball of each + # commit as the artifact ``. + - name: npm + access: public # The service image. The Image workflow pushes `:` for the release # branch, and Craft copies it to `:`. - name: docker diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12d1435..02f1658 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,3 +48,25 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm test:coverage + + package: + name: package + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm pack + # Craft publishes the tarball of the release commit to npm. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ github.sha }} + path: "*.tgz" + if-no-files-found: error diff --git a/README.md b/README.md index 0dca2b2..ad50139 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,44 @@ Roach started in It uses Node built-ins and the `openssl` command. Only the service uses the Sentry SDK (`@sentry/node`). +## Use with vitest + +Install `@sentry/roach`, add a `roach.json` (see [Config file](#config-file)), +and add two lines to the vitest config: + +```ts +export default defineConfig({ + test: { + globalSetup: ["@sentry/roach/vitest"], + setupFiles: ["@sentry/roach/vitest/setup"], + }, +}); +``` + +The global setup (`src/vitest.ts`) starts one run for the invocation and +gives its proxy variables to the test workers. The setup file +(`src/vitest-setup.ts`) opens one session for each test, so test files +record in parallel workers at the same time. A passed test writes its new +recordings. A failed test drops them. In `replay` mode, a test with a miss +fails. At the end, the global setup writes the stats of the run to stdout +and to the job summary. + +- `ROACH_TOKEN`: the write token. Only the global setup reads it. Without + it, the run can only replay, as in jobs of forks. +- `ROACH_URL`: the service. Default: `https://roach-proxy.getsentry.net`. +- `ROACH_MODE`: `auto`, `replay`, `record`, or `off`. Default: `auto` with + the token, `replay` without it. With `off`, the global setup starts no run, + so requests go direct and need no service and no token. + +A project that has its own global setup calls `startRoach(project)` from +`@sentry/roach/vitest` in it, and returns its teardown. + +- It needs Node 24.14 or later (`http.setGlobalProxyFromEnv`). +- The tests of one file share one process, so they run one at a time. Do + not use `test.concurrent`. +- The package has TypeScript sources. Vitest compiles them, but Node does + not strip types in `node_modules`, so plain Node scripts cannot import it. + ## Use the service ```ts @@ -43,7 +81,7 @@ const run = await startRemoteRun( spawn("vitest", { env: { ...process.env, ...run.env } }); // Once per test, in a worker that has the `controlUrl` and `token` of the -// run. +// run. Send the requests of the test through `session.url`. const session = await connectRoach({ controlUrl, token }).startSession( testName, ); @@ -62,7 +100,8 @@ A client that does not use `client.ts` can do the same with plain HTTP. See ## Use the GitHub Action -A repository adds one step and a `roach.json` file: +A repository adds one step and a `roach.json` file (see +[Config file](#config-file)): ```yaml - uses: getsentry/roach@v0 @@ -71,38 +110,56 @@ A repository adds one step and a `roach.json` file: run: pnpm test ``` -`roach.json` is at the root of the repository. It has the `allow` origins -and the `rules` of the run (`RunConfig` in `src/service.ts`): +The action (`src/action.ts`) starts a run with the token in `auto` mode, or +without it in `replay` mode. It runs `run` in bash with the proxy variables +of the run, then ends the run and writes its stats to the job summary. + +- Only `run` uses the proxy. Later steps, such as cache and artifact + uploads, do not. `run` does not get the write token. +- `run` is one session. When it fails, the run drops its new recordings. A + miss in `replay` mode fails the step. The summary names the job, not the + test, of each miss. For a session per test, use + [vitest](#use-with-vitest) in `run`. +- `NODE_EXTRA_CA_CERTS` works only for Node. Python and `curl` need a CA + bundle that also has the system certificates. + +## Config file + +The action and the vitest integration read `roach.json` in the working +directory or the closest parent directory (`src/config.ts`): ```json { - "allow": ["https://ai-gateway.vercel.sh"], + "allow": ["https://api.anthropic.com"], + "secretEnv": ["ANTHROPIC_API_KEY"], "rules": [ { "name": "model", - "match": { "method": "POST", "url": "https://ai-gateway.vercel.sh/" }, - "keyHeaders": ["ai-model-id"] + "match": { "method": "POST", "url": "https://api.anthropic.com/v1/" }, + "values": { + "uuid": { "builtin": "uuid" }, + "build": "build-[0-9]+" + } } ] } ``` -The action (`src/action.ts`) starts a run of the tenant -`GITHUB_REPOSITORY`, in `auto` mode with the token or `replay` mode without -it. It runs `run` in bash with the proxy variables of the run, then ends -the run and writes its stats to the job summary. +- `allow` and `rules` are those of the run. See [Config](#config). +- A value is a regular expression source, or `{ "builtin": "" }` + for a pattern of `VALUE_PATTERNS` in `values.ts`. +- `secretEnv`: environment variables that hold credentials. Their values + are the `secrets` of the run. +- `tenant`: default `GITHUB_REPOSITORY`. Set it, so local runs use the + recordings of CI. +- `noProxy`: hosts that do not use the proxy. Default: loopback. -- Only `run` uses the proxy. Later steps, such as cache and artifact - uploads, do not. `run` does not get the write token. -- `run` is one session. When it fails, the run drops its new recordings. A - miss in `replay` mode fails the step. The summary names the job, not the - test, of each miss. -- `NODE_EXTRA_CA_CERTS` works only for Node. Python and `curl` need a CA - bundle that also has the system certificates. +The `name` of the run is `-` in GitHub +Actions. ## Config -A run takes `RunConfig` (`src/service.ts`): `tenant`, `mode`, `rules`, +A run takes `RunConfig` (`src/types.ts`): `tenant`, `mode`, `rules`, `allow`, `name`, and `secrets`. - `mode`: `auto` replays recordings and records misses. `replay` replays @@ -141,8 +198,10 @@ headers, and the body. A JSON body has sorted keys. A recording keeps the response and the session that recorded it. It does not keep the request body. -- One session is open at a time. The proxy keeps the new recordings of a - session in memory. A passed session writes them, and a failed session +- Many sessions can be open at once. A request belongs to the session of + its proxy URL (`url` of the session). A request through the proxy URL of + the run belongs to the open session when only one is open. The proxy + keeps the new recordings of a session in memory. A passed session writes them, and a failed session drops them, so a bad sample is never replayed. - A request belongs to the session that was open when it started. If it ends after its session ended, it follows the result of that session. @@ -410,7 +469,10 @@ It starts the service and prints its URL as one JSON line. The calls of a run are under `/__roach/runs/`. They need `Authorization: Bearer `. `client.ts` calls them. -- `POST /session` with `{"name": "..."}`: open a session. +- `POST /session` with `{"name": "..."}`: open a session. Returns + `{"url": "..."}`, the proxy URL of the session. Its user is + `-`. An open session with the same name fails + first. A run can have 64 open sessions. Another one gets HTTP 429. - `POST /session/end` with `{"name": "...", "passed": true}`: end it. Returns `{"missed": 0}`, the misses that `replay` mode failed. Returns HTTP 409 when the session is not open. @@ -434,9 +496,12 @@ manager accepts it, Craft: - copies the image `ghcr.io/getsentry/roach:` to `:`. - makes the GitHub release `v`, with the changelog, and moves the `v` tag. Repositories use the action as `getsentry/roach@v`. +- publishes `@sentry/roach` to npm: the tarball that the `package` job of + `CI` uploads for the release commit. Craft writes the version to + `package.json` on the release branch. -The action runs `src/action.ts` from the tag, so a release has no build -step. +The action runs `src/action.ts` from the tag, and the package has the +sources, so a release has no build step. The `Release` workflow uses the token of Sentry Release Bot. A push with `GITHUB_TOKEN` starts no workflows, so the release branch gets no checks and @@ -454,6 +519,8 @@ pnpm check # format, lint, types, unused code, and tests, as CI runs them - `tests/recording.test.ts` tests modes, sessions, changing values, and misses. - `tests/service.test.ts` tests the service in the test process. +- `tests/vitest.test.ts` runs vitest with the integration on a small + project, against a service in the test process. - `tests/deployed.test.ts` runs the service as it runs in production: from the command line, behind a TLS server that does what the load balancer does, with a local GCS server, metadata server, and Sentry server. Each @@ -465,7 +532,7 @@ pnpm check # format, lint, types, unused code, and tests, as CI runs them The proxy is in `src/`. Tests are in `tests/`. The production setup is in `deploy/gcp/` and `Dockerfile`. -- `types.ts`: rules, modes, and the results of a run. +- `types.ts`: rules, modes, runs, sessions, and the results of a run. - `server.ts`: sockets, HTTPS interception, the allow list, and the control API. - `addresses.ts`: the public addresses that the proxy can send requests to. @@ -481,6 +548,9 @@ The proxy is in `src/`. Tests are in `tests/`. The production setup is in - `secrets.ts`: redaction of credentials. - `certificates.ts`: the certificate authority for HTTPS. - `client.ts`: starts a run and calls the control API. +- `config.ts`: `roach.json`, for the action and vitest. +- `vitest.ts`: the vitest global setup: one run for each invocation. +- `vitest-setup.ts`: the vitest setup file: one session for each test. - `report.ts`: text reports of a run. - `action.ts`: the GitHub Action (`action.yml` at the root). - `cli.ts`: the command line. diff --git a/deploy/gcp/roach.tfvars b/deploy/gcp/roach.tfvars index 3d41aea..a061e44 100644 --- a/deploy/gcp/roach.tfvars +++ b/deploy/gcp/roach.tfvars @@ -11,7 +11,7 @@ domain = "roach-proxy.getsentry.net" deploy_repository = "getsentry/roach" deploy_repository_id = "1412275249" -# Runs send their own origins and value patterns (RunConfig in src/service.ts), +# Runs send their own origins and value patterns (RunConfig in src/types.ts), # so a tenant needs no change here. # The service sends metrics and spans to the sentry/roach project. diff --git a/knip.json b/knip.json index 3a2363c..b7cba30 100644 --- a/knip.json +++ b/knip.json @@ -2,5 +2,5 @@ "$schema": "https://unpkg.com/knip@6/schema.json", "project": ["src/**/*.ts", "tests/**/*.ts"], "ignoreBinaries": ["openssl"], - "entry": ["src/action.ts"] + "entry": ["src/action.ts", "src/cli.ts"] } diff --git a/package.json b/package.json index 99b941f..2894800 100644 --- a/package.json +++ b/package.json @@ -1,22 +1,27 @@ { "name": "@sentry/roach", - "private": true, + "version": "0.1.0", "description": "A recording HTTPS proxy for tests, as a shared service.", "license": "Apache-2.0", "repository": { "type": "git", "url": "git+https://github.com/getsentry/roach.git" }, - "bin": { - "roach": "./src/cli.ts" - }, + "files": [ + "src" + ], "type": "module", "exports": { "./client": "./src/client.ts", "./report": "./src/report.ts", "./service": "./src/service.ts", "./types": "./src/types.ts", - "./values": "./src/values.ts" + "./values": "./src/values.ts", + "./vitest": "./src/vitest.ts", + "./vitest/setup": "./src/vitest-setup.ts" + }, + "publishConfig": { + "access": "public" }, "scripts": { "check": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm knip && pnpm test", @@ -48,6 +53,14 @@ "undici": "^8.11.2", "vitest": "^5.0.3" }, + "peerDependencies": { + "vitest": ">=3" + }, + "peerDependenciesMeta": { + "vitest": { + "optional": true + } + }, "simple-git-hooks": { "pre-commit": "pnpm exec lint-staged", "pre-push": "pnpm typecheck" diff --git a/src/action.ts b/src/action.ts index f86a961..d940d9d 100644 --- a/src/action.ts +++ b/src/action.ts @@ -3,14 +3,14 @@ * `README.md`. * * It starts a run, runs the command of the step with the proxy variables of - * the run, and ends the run. The rules of the run come from `roach.json`. + * the run, and ends the run. The run config comes from `roach.json` + * (`config.ts`). * The runner gives each input as `INPUT_`. */ import { spawn } from "node:child_process"; -import { appendFile, readFile } from "node:fs/promises"; import { startRemoteRun } from "./client.ts"; -import { describeRecordingMisses, describeRecordingStats } from "./report.ts"; -import type { RunConfig } from "./service.ts"; +import { readEnv, readRoachFile, runConfigOf } from "./config.ts"; +import { writeRunReport } from "./report.ts"; /** * Variables that the command does not get: the inputs, which have the token, @@ -51,22 +51,13 @@ function runCommand(command: string, env: NodeJS.ProcessEnv): Promise { const command = process.env.INPUT_RUN; if (!command) throw new Error("The run input is required"); // The runner gives an empty string for an input that the step does not set. -const token = - process.env.INPUT_TOKEN === "" ? undefined : process.env.INPUT_TOKEN; -// Without the token, the service only allows `replay`, as for forks. -const mode = token ? "auto" : "replay"; -const config = JSON.parse(await readFile("roach.json", "utf8")) as Pick< - RunConfig, - "allow" | "rules" ->; +const token = readEnv("INPUT_TOKEN"); +const file = await readRoachFile(); +const config = runConfigOf(file, { canWrite: token !== undefined }); const run = await startRemoteRun( { url: process.env.INPUT_URL!, token }, - { - ...config, - tenant: process.env.GITHUB_REPOSITORY!, - mode, - name: `${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT}`, - }, + config, + file.noProxy ? { noProxy: file.noProxy } : {}, ); // The proxy URL has the run token. Keep it out of the log. process.stdout.write(`::add-mask::${run.token}\n`); @@ -85,14 +76,6 @@ try { const { missed } = await session.end(exitCode === 0); if (missed > 0 && exitCode === 0) exitCode = 1; } finally { - const stats = await run.close(); - const report = [ - `Roach (${mode}): ${describeRecordingStats(stats)}`, - ...describeRecordingMisses(stats.misses).map((miss) => `- ${miss}`), - ].join("\n"); - process.stdout.write(`${report}\n`); - if (process.env.GITHUB_STEP_SUMMARY) { - await appendFile(process.env.GITHUB_STEP_SUMMARY, `${report}\n`); - } + await writeRunReport(`Roach (${config.mode})`, await run.close()); } process.exitCode = exitCode; diff --git a/src/client.ts b/src/client.ts index cc921f9..6b4565e 100644 --- a/src/client.ts +++ b/src/client.ts @@ -11,12 +11,21 @@ import http from "node:http"; import https from "node:https"; import { tmpdir } from "node:os"; import path from "node:path"; -import { CONTROL_PATH } from "./server.ts"; -import type { RemoteRun, RunConfig } from "./service.ts"; -import type { RecordingStats } from "./types.ts"; +import { + CONTROL_PATH, + type OpenedSession, + type RecordingStats, + type RemoteRun, + type RunConfig, +} from "./types.ts"; /** The open session of one test. */ export interface RecordingSession { + /** + * The proxy URL of the session. Send the requests of the test through + * it when other sessions of the run can be open at the same time. + */ + url: string; /** * End the session. A passed session writes its new recordings, and a * failed one drops them. `missed` counts the requests of the session that @@ -27,7 +36,11 @@ export interface RecordingSession { /** The control API of a run. */ export interface RoachControl { - /** Open the session of one test. Only one session is open at a time. */ + /** + * Open the session of one test. Many sessions can be open at once. A + * request through `url` of the run belongs to the open session only when + * one is open, so concurrent tests send through `url` of their session. + */ startSession(name: string): Promise; /** The totals of the run. */ stats(): Promise; @@ -90,8 +103,15 @@ export function connectRoach({ }: Pick): RoachControl { return { async startSession(name) { - await callControl(controlUrl, token, "POST", "session", { name }); + const { url } = await callControl( + controlUrl, + token, + "POST", + "session", + { name }, + ); return { + url, end: (passed) => callControl(controlUrl, token, "POST", "session/end", { name, diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..6c02a1e --- /dev/null +++ b/src/config.ts @@ -0,0 +1,141 @@ +/** + * `roach.json`: the run config of a repository. The GitHub Action + * (`action.ts`) and the vitest integration (`vitest.ts`) both read it, so a + * repository writes its origins and rules once, as data. See "Config file" + * in `README.md`. + */ +import { access, readFile } from "node:fs/promises"; +import path from "node:path"; +import type { RecordingMode, RecordingRule, RunConfig } from "./types.ts"; +import { VALUE_PATTERNS } from "./values.ts"; + +/** The Roach service of Sentry. See "Deploy" in `README.md`. */ +export const DEFAULT_SERVICE_URL = "https://roach-proxy.getsentry.net"; + +/** The name of the config file. */ +const FILE_NAME = "roach.json"; + +/** + * A changing value in `roach.json`: a regular expression source, or the + * name of a pattern in `VALUE_PATTERNS`, such as `{ "builtin": "uuid" }`. + */ +type FileValue = string | { builtin: keyof typeof VALUE_PATTERNS }; + +/** A rule in `roach.json`. */ +interface FileRule extends Omit { + values?: Record; +} + +/** The content of `roach.json`. */ +export interface RoachFile { + allow: string[]; + rules: FileRule[]; + /** Default: `GITHUB_REPOSITORY`. */ + tenant?: string; + /** + * Environment variables that hold credentials, such as + * `ANTHROPIC_API_KEY`. The run never writes their values. + */ + secretEnv?: string[]; + /** Hosts that do not use the proxy. Default: loopback. */ + noProxy?: string; +} + +const MODES = new Set(["auto", "off", "record", "replay"]); + +/** A variable of `env`, or `undefined` when it is not set or is blank. */ +export function readEnv( + name: string, + env: NodeJS.ProcessEnv = process.env, +): string | undefined { + const value = env[name]?.trim(); + return value === "" ? undefined : value; +} + +/** + * Find `roach.json` in `from` or the closest parent directory, and read + * it. Throws when no directory has one. + */ +export async function readRoachFile( + from = process.cwd(), +): Promise { + for (let directory = path.resolve(from); ;) { + const file = path.join(directory, FILE_NAME); + const found = await access(file).then( + () => true, + () => false, + ); + if (found) { + return { + ...(JSON.parse(await readFile(file, "utf8")) as RoachFile), + path: file, + }; + } + const parent = path.dirname(directory); + if (parent === directory) { + throw new Error(`Roach: no ${FILE_NAME} in ${from} or its parents`); + } + directory = parent; + } +} + +function resolveValue(rule: string, name: string, value: FileValue): string { + if (typeof value === "string") return value; + const pattern = VALUE_PATTERNS[value.builtin]; + if (!pattern) { + throw new Error( + `Roach: rule ${rule} value ${name} names no built-in pattern: ${value.builtin}`, + ); + } + return pattern; +} + +/** + * The run config of `file` in the environment `env`. Without the write + * token (`canWrite`), the mode is `replay`, as for forks. `ROACH_MODE` + * sets another mode. + */ +export function runConfigOf( + file: RoachFile, + { + canWrite, + env = process.env, + }: { canWrite: boolean; env?: NodeJS.ProcessEnv }, +): RunConfig { + const mode = readEnv("ROACH_MODE", env) ?? (canWrite ? "auto" : "replay"); + if (!MODES.has(mode)) { + throw new Error( + `Roach: ROACH_MODE must be auto, replay, record, or off, got ${mode}`, + ); + } + const tenant = file.tenant ?? env.GITHUB_REPOSITORY; + if (!tenant) { + throw new Error( + `Roach: set tenant in ${FILE_NAME}, or run with GITHUB_REPOSITORY`, + ); + } + const config: RunConfig = { + tenant, + mode: mode as RecordingMode, + allow: file.allow, + rules: file.rules.map(({ values, ...rule }) => + values + ? { + ...rule, + values: Object.fromEntries( + Object.entries(values).map(([name, value]) => [ + name, + resolveValue(rule.name, name, value), + ]), + ), + } + : rule, + ), + secrets: (file.secretEnv ?? []).flatMap((name) => readEnv(name, env) ?? []), + }; + const runId = readEnv("GITHUB_RUN_ID", env); + if (runId) { + config.name = `${runId}-${readEnv("GITHUB_RUN_ATTEMPT", env) ?? "1"}`; + } + return config; +} diff --git a/src/recorder.ts b/src/recorder.ts index ba70522..441d335 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -7,16 +7,19 @@ * every read and write of the recordings. A store keeps them: the file * store (`recordings.ts`) or a GCS bucket (`gcs.ts`). * - * A session groups the requests of one test. One session is open at a - * time. The recorder keeps the new recordings of a session in memory until - * the session ends. A passed session writes them, and a failed session - * drops them, so a bad sample is never replayed. A request that ends after - * its session ended follows the result of that session. A request outside - * a session is written at once. + * A session groups the requests of one test. Many sessions can be open at + * once, such as one for each test worker. A request names its session in + * its proxy URL (`service.ts`). A request that names no session belongs to + * the open session when only one is open. The recorder keeps the new + * recordings of a session in memory until the session ends. A passed + * session writes them, and a failed session drops them, so a bad sample is + * never replayed. A request that ends after its session ended follows the + * result of that session. A request outside a session is written at once. * * The recorder never writes a known credential (`secrets.ts`). It reports * spans and metrics through the `Telemetry` hooks of `service.ts`. */ +import { randomBytes } from "node:crypto"; import type { IncomingHttpHeaders } from "node:http"; import { keyRequest, type KeyedRequest } from "./request-key.ts"; import { createSecrets } from "./secrets.ts"; @@ -82,6 +85,8 @@ export interface Telemetry { } interface Session { + /** The id in the proxy URL of the session. */ + id: string; name: string; /** New recordings, by key. */ recorded: Map; @@ -97,6 +102,13 @@ interface Session { } const MAX_MISSES = 500; +/** + * Ended sessions that a run keeps, so that late requests follow their + * result. The oldest ones go first, so memory stays bounded. + */ +const MAX_ENDED_SESSIONS = 1000; +/** Open sessions of a run, such as one for each test worker. */ +const MAX_OPEN_SESSIONS = 64; /** Response headers that a recording keeps. Other headers change each run. */ const RECORDED_HEADERS = ["content-type", "location"]; const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]); @@ -177,7 +189,10 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { throw new Error(`Roach rule name is not valid: ${rule.name}`); } } - let session: Session | undefined; + /** Sessions by id. An ended session stays, so its late requests follow it. */ + const sessions = new Map(); + const open = () => + [...sessions.values()].filter((s) => s.passed === undefined); const { telemetry } = config; const secrets = createSecrets(config.secrets); const stats: RecordingStats = { @@ -240,17 +255,27 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { process.stderr.write(`[roach] No ${rule.name} recording${where}: ${key}\n`); }; - /** End the open session. Returns its requests that `replay` mode failed. */ - const finish = async (passed: boolean): Promise => { - const ended = session; - if (!ended) return 0; - session = undefined; + /** End a session. Returns its requests that `replay` mode failed. */ + const finish = async (ended: Session, passed: boolean): Promise => { ended.passed = passed; - if (passed) await write([...ended.recorded]); - else drop([...ended.recorded.keys()]); + const recorded = [...ended.recorded]; + ended.recorded.clear(); + ended.known.clear(); + if (passed) await write(recorded); + else drop(recorded.map(([key]) => key)); return ended.missed; }; + /** + * The session of a request. `id` comes from its proxy URL. Without it, + * the request belongs to the open session when only one is open. + */ + const sessionOf = (id: string | undefined): Session | undefined => { + if (id !== undefined) return sessions.get(id); + const current = open(); + return current.length === 1 ? current[0] : undefined; + }; + return { /** The rule that records a request, or `undefined` when none does. */ ruleFor(request: ProxyRequest): RecordingRule | undefined { @@ -275,11 +300,12 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { rule: RecordingRule, request: ProxyRequest, sendLive: () => Promise, + sessionId?: string, ): Promise { const started = performance.now(); const origin = request.url.origin; // A request belongs to the session that was open when it started. - const owner = session; + const owner = sessionOf(sessionId); const counts = stats.counts[rule.name]!; secrets.learn(request.headers); telemetry.annotate({ rule: rule.name }); @@ -379,40 +405,57 @@ export function createRecorder(config: RecorderConfig, store: RecordingStore) { return { ...response, source: "live" }; }, - /** Open the session of one test. An open session fails first. */ - async startSession(name: string): Promise { - if (session) { + /** + * Open the session of one test, and return its id. An open session with + * the same name fails first. Throws when the run has too many open + * sessions. + */ + async startSession(name: string): Promise { + const same = open().find((other) => other.name === name); + if (same) { process.stderr.write( - `[roach] Session "${session.name}" did not end, so its new recordings were dropped\n`, + `[roach] Session "${name}" did not end, so its new recordings were dropped\n`, ); - await finish(false); + await finish(same, false); + } + if (open().length >= MAX_OPEN_SESSIONS) { + throw new Error(`A run can have ${MAX_OPEN_SESSIONS} open sessions`); + } + const ended = [...sessions.values()].filter( + (other) => other.passed !== undefined, + ); + const extra = ended.length - MAX_ENDED_SESSIONS + 1; + for (const old of ended.slice(0, Math.max(0, extra))) { + sessions.delete(old.id); } - session = { + const id = randomBytes(8).toString("hex"); + sessions.set(id, { + id, name, recorded: new Map(), missed: 0, known: new Map(), - }; + }); + return id; }, /** - * End the session `name`. Returns its requests that `replay` mode - * failed. Throws when `name` is not the open session. + * End the open session `name`. Returns its requests that `replay` mode + * failed. Throws when no open session has that name. */ async endSession(name: string, passed: boolean): Promise { - if (session?.name !== name) { - throw new Error(`Session "${name}" is not open`); - } - return finish(passed); + const ended = open().find((other) => other.name === name); + if (!ended) throw new Error(`Session "${name}" is not open`); + return finish(ended, passed); }, stats(): RecordingStats { return stats; }, - /** Fail the open session. */ + /** Fail the open sessions. */ async close(): Promise { - await finish(false); + for (const session of open()) await finish(session, false); }, }; } diff --git a/src/report.ts b/src/report.ts index 1fd288b..4e44434 100644 --- a/src/report.ts +++ b/src/report.ts @@ -1,6 +1,7 @@ /** * Text reports of a Roach run, for logs and CI summaries. */ +import { appendFile } from "node:fs/promises"; import type { RecordingMiss, RecordingStats } from "./types.ts"; /** Describe the totals of a run in one line. */ @@ -34,3 +35,21 @@ export function describeRecordingMisses(misses: RecordingMiss[]): string[] { }) .map((miss) => `${miss.session ?? "outside a test"}: ${miss.file}`); } + +/** + * Write the report of a run to stdout, and in GitHub Actions also to the + * job summary. `title` names the run, such as `Roach (auto)`. + */ +export async function writeRunReport( + title: string, + stats: RecordingStats, +): Promise { + const report = [ + `${title}: ${describeRecordingStats(stats)}`, + ...describeRecordingMisses(stats.misses).map((miss) => `- ${miss}`), + ].join("\n"); + process.stdout.write(`${report}\n`); + if (process.env.GITHUB_STEP_SUMMARY) { + await appendFile(process.env.GITHUB_STEP_SUMMARY, `${report}\n`); + } +} diff --git a/src/server.ts b/src/server.ts index 3671611..3ae001e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -24,9 +24,7 @@ import tls from "node:tls"; import { isPublicAddress, notPublicError, publicLookup } from "./addresses.ts"; import type { CertificateAuthority } from "./certificates.ts"; import type { ProxyRequest, Recorder, Telemetry } from "./recorder.ts"; - -/** The path prefix of the control API. */ -export const CONTROL_PATH = "/__roach"; +import type { OpenedSession } from "./types.ts"; /** Headers for one connection, which the proxy must not forward. */ const HOP_HEADERS = new Set([ @@ -176,6 +174,7 @@ function sendUpstream( /** Proxy one request to an allowed origin, through the recorder. */ async function proxyRequest( { recorder, origins, replayOnly, privateNetworks = false }: ProxyTarget, + session: string | undefined, incoming: http.IncomingMessage, outgoing: http.ServerResponse, tunnelOrigin: string | undefined, @@ -218,17 +217,22 @@ async function proxyRequest( return; } - const response = await recorder.respond(rule, request, async () => { - // Ask for a plain body, so the recording is readable. - const upstream = await sendUpstream(origin, request, privateNetworks, { - "accept-encoding": "identity", - }); - return { - status: upstream.statusCode ?? 502, - headers: upstream.headers, - body: await readBody(upstream), - }; - }); + const response = await recorder.respond( + rule, + request, + async () => { + // Ask for a plain body, so the recording is readable. + const upstream = await sendUpstream(origin, request, privateNetworks, { + "accept-encoding": "identity", + }); + return { + status: upstream.statusCode ?? 502, + headers: upstream.headers, + body: await readBody(upstream), + }; + }, + session, + ); outgoing.writeHead(response.status, { ...response.headers, "content-length": String(response.body.length), @@ -250,13 +254,15 @@ export function sendJson( /** * Answer a session or stats call of one recorder. `route` is the method * and the path after the prefix of the run, such as `POST /session`. - * Returns `false` when the route is not one of them. + * `sessionUrl` gives the proxy URL of a session id. Returns `false` when + * the route is not one of them. */ export async function controlRecorder( recorder: Recorder, route: string, incoming: http.IncomingMessage, outgoing: http.ServerResponse, + sessionUrl: (id: string) => string, ): Promise { switch (route) { case "GET /stats": @@ -268,8 +274,14 @@ export async function controlRecorder( sendJson(outgoing, 400, { error: "name must be a string" }); return true; } - await recorder.startSession(name); - outgoing.writeHead(204).end(); + try { + const id = await recorder.startSession(name); + sendJson(outgoing, 200, { + url: sessionUrl(id), + } satisfies OpenedSession); + } catch (error) { + sendJson(outgoing, 429, { error: (error as Error).message }); + } return true; } case "POST /session/end": { @@ -310,8 +322,13 @@ export interface ProxyTarget { telemetry: Telemetry; } -/** A target, or the HTTP status that refuses the request. */ -export type TargetResult = ProxyTarget | { status: 403 | 407 }; +/** + * The target of a request and the session id of its proxy URL, or the HTTP + * status that refuses the request. + */ +export type TargetResult = + | { target: ProxyTarget; session: string | undefined } + | { status: 403 | 407 }; /** The options of `listenProxy()`. */ export interface ListenOptions { @@ -354,7 +371,7 @@ function refuse(status: 403 | 407, outgoing: http.ServerResponse): void { /** Proxy one request through its target, and answer errors with 502. */ function serveProxied( - target: ProxyTarget, + { target, session }: { target: ProxyTarget; session: string | undefined }, incoming: http.IncomingMessage, outgoing: http.ServerResponse, tunnelOrigin: string | undefined, @@ -375,7 +392,7 @@ function serveProxied( attributes: { origin: String(origin), "http.request.method": method }, }, async () => { - await proxyRequest(target, incoming, outgoing, tunnelOrigin); + await proxyRequest(target, session, incoming, outgoing, tunnelOrigin); target.telemetry.annotate({ "http.response.status_code": outgoing.statusCode, }); @@ -401,10 +418,11 @@ export async function listenProxy( options: ListenOptions, ): Promise { const { authority } = options; - // The origin and target of each tunnel. Absolute-form requests have none. + // The origin, target, and session of each tunnel. Absolute-form requests + // have none. const tunnels = new WeakMap< Socket, - { origin: string; target: ProxyTarget } + { origin: string; target: ProxyTarget; session: string | undefined } >(); const sockets = new Set(); @@ -415,7 +433,7 @@ export async function listenProxy( outgoing.writeHead(400).end(); return; } - serveProxied(tunnel.target, incoming, outgoing, tunnel.origin); + serveProxied(tunnel, incoming, outgoing, tunnel.origin); }); const server = http.createServer((incoming, outgoing) => { // A request to the proxy itself has a path. A proxied plain HTTP @@ -463,7 +481,7 @@ export async function listenProxy( return; } const [, host, port] = match as unknown as [string, string, string]; - const allowed = [...target.origins.values()].some( + const allowed = [...target.target.origins.values()].some( (origin) => authorityOf(origin) === `${host.toLowerCase()}:${port}`, ); if (!allowed) { @@ -477,8 +495,8 @@ export async function listenProxy( // A TLS handshake starts with byte 0x16. Anything else is plain HTTP. if (first[0] !== 0x16) { tunnels.set(socket, { + ...target, origin: `http://${host}${port === "80" ? "" : `:${port}`}`, - target, }); tunnelServer.emit("connection", socket); socket.resume(); @@ -491,8 +509,8 @@ export async function listenProxy( }); secure.on("error", () => secure.destroy()); tunnels.set(secure, { + ...target, origin: `https://${host}${port === "443" ? "" : `:${port}`}`, - target, }); tunnelServer.emit("connection", secure); }; diff --git a/src/service.ts b/src/service.ts index 6afa132..73ddd06 100644 --- a/src/service.ts +++ b/src/service.ts @@ -42,7 +42,6 @@ import { createGcsStore } from "./gcs.ts"; import { createRecorder, type Telemetry } from "./recorder.ts"; import { createFileStore } from "./recordings.ts"; import { - CONTROL_PATH, controlRecorder, listenProxy, parseOrigins, @@ -53,7 +52,13 @@ import { type TargetResult, } from "./server.ts"; import { RULE_NAME, type RecordingStore } from "./store.ts"; -import type { RecordingMode, RecordingRule, RecordingStats } from "./types.ts"; +import { + CONTROL_PATH, + type RecordingMode, + type RecordingStats, + type RemoteRun, + type RunConfig, +} from "./types.ts"; import { invalidValuePatterns } from "./values.ts"; /** The configuration of the service. The operator owns it. */ @@ -94,48 +99,6 @@ export interface RoachServiceConfig { sentryDsn?: string; } -/** The configuration of one run. The client sends it. */ -export interface RunConfig { - /** - * The tenant of the run, as `owner/repo`, such as - * `process.env.GITHUB_REPOSITORY`. - */ - tenant: string; - mode: RecordingMode; - rules: RecordingRule[]; - /** - * The only origins that the run can reach, such as - * `https://ai-gateway.vercel.sh`. The service sends requests only to - * their public addresses. - */ - allow: string[]; - /** - * The name of the run in Sentry, such as - * `${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}`. Default: the run id. - */ - name?: string; - /** - * Credentials that this run must never write, such as the API keys of - * its CI job. The service redacts them with the `secrets` of the service - * config, keeps them only in memory, and forgets them when the run ends. - */ - secrets?: string[]; -} - -/** A run that the service created. */ -export interface RemoteRun { - /** The id of the run. */ - id: string; - /** The proxy URL, with the run id and token as its user and password. */ - url: string; - /** The run token. Session and stats calls use it. */ - token: string; - /** The base URL of the control API of the run. */ - controlUrl: string; - /** The PEM certificate of the authority that signs intercepted hosts. */ - caCert: string; -} - /** A service that runs in this process. */ export interface RoachService { /** The URL of the service. */ @@ -192,16 +155,35 @@ function isWriteToken( return token !== undefined && timingSafeEqual(expected, sha256(token)); } -/** The run id and token in a `Proxy-Authorization` header. */ +/** + * The run id, session id, and token in a `Proxy-Authorization` header. The + * user is `` or, for a session, `-`. + */ function proxyCredentials( header: string | undefined, -): { id: string; token: string } | undefined { +): { id: string; session: string | undefined; token: string } | undefined { const match = /^Basic (.+)$/i.exec(header ?? ""); if (!match) return undefined; const decoded = Buffer.from(match[1]!, "base64").toString("utf8"); const colon = decoded.indexOf(":"); if (colon <= 0) return undefined; - return { id: decoded.slice(0, colon), token: decoded.slice(colon + 1) }; + const [id, session] = decoded.slice(0, colon).split("-", 2) as [ + string, + string | undefined, + ]; + return { id, session, token: decoded.slice(colon + 1) }; +} + +/** The proxy URL of a run, or of one session of the run. */ +function proxyUrl( + publicUrl: string, + run: { id: string; token: string }, + session?: string, +): string { + const url = new URL(publicUrl); + url.username = session ? `${run.id}-${session}` : run.id; + url.password = run.token; + return url.href.replace(/\/$/, ""); } /** Check the run config of a client. Returns the error, or `undefined`. */ @@ -441,12 +423,9 @@ export async function startRoachService( process.stderr.write( `[roach] Run ${id} of ${runConfig.tenant} started in ${runConfig.mode} mode\n`, ); - const base = new URL(publicUrl!); - base.username = id; - base.password = token; return { id, - url: base.href.replace(/\/$/, ""), + url: proxyUrl(publicUrl!, run), token, controlUrl: `${publicUrl}${CONTROL_PATH}/runs/${id}`, caCert: authority.caCert, @@ -461,7 +440,7 @@ export async function startRoachService( if (!run || !sameToken(credentials.token, run.token)) { return { status: 407 }; } - return run; + return { target: run, session: credentials.session }; }; const control = async ( @@ -548,6 +527,7 @@ export async function startRoachService( `${incoming.method} ${rest}`, incoming, outgoing, + (session) => proxyUrl(publicUrl!, run, session), ); if (!handled) outgoing.writeHead(404).end(); }; diff --git a/src/types.ts b/src/types.ts index d46cd66..c53008f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,9 +1,13 @@ /** - * The public types of Roach: rules, modes, and the results of a run. See - * `README.md`. + * The public contract of Roach: rules, modes, runs, sessions, and the + * results of a run. The client, the service, and the vitest integration + * share it. See `README.md`. */ import type { ValuePatterns } from "./values.ts"; +/** The path prefix of the control API. */ +export const CONTROL_PATH = "/__roach"; + /** * - `auto`: replay a request that has a recording. Send other requests live * and record them. @@ -70,3 +74,57 @@ export interface RecordingStats { */ passthrough: Record; } + +/** The configuration of one run. The client sends it. */ +export interface RunConfig { + /** + * The tenant of the run, as `owner/repo`, such as + * `process.env.GITHUB_REPOSITORY`. + */ + tenant: string; + mode: RecordingMode; + rules: RecordingRule[]; + /** + * The only origins that the run can reach, such as + * `https://ai-gateway.vercel.sh`. The service sends requests only to + * their public addresses. + */ + allow: string[]; + /** + * The name of the run in Sentry, such as + * `${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}`. Default: the run id. + */ + name?: string; + /** + * Credentials that this run must never write, such as the API keys of + * its CI job. The service redacts them with the `secrets` of the service + * config, keeps them only in memory, and forgets them when the run ends. + */ + secrets?: string[]; +} + +/** A run that the service created. */ +export interface RemoteRun { + /** The id of the run. */ + id: string; + /** + * The proxy URL, with the run id and token as its user and password. A + * request through it belongs to the open session when only one is open. + */ + url: string; + /** The run token. Session and stats calls use it. */ + token: string; + /** The base URL of the control API of the run. */ + controlUrl: string; + /** The PEM certificate of the authority that signs intercepted hosts. */ + caCert: string; +} + +/** A session that the service opened. */ +export interface OpenedSession { + /** + * The proxy URL of the session. Each request through it belongs to the + * session, also when other sessions are open. + */ + url: string; +} diff --git a/src/vitest-setup.ts b/src/vitest-setup.ts new file mode 100644 index 0000000..26493b7 --- /dev/null +++ b/src/vitest-setup.ts @@ -0,0 +1,66 @@ +/** + * The vitest setup file of Roach (`@sentry/roach/vitest/setup`). See "Use + * with vitest" in `README.md`. + * + * It opens one session for each test of the run that `vitest.ts` started. + * Requests of the test go through the proxy URL of its session, so tests + * in other workers record at the same time. Its child processes get that + * URL too. A passed test writes its new recordings, and a failed test + * drops them. A test with a request that `replay` mode failed fails. + * + * The tests of one file share one process, so they must run one at a + * time. Do not use `test.concurrent` with Roach. + */ +import http from "node:http"; +import { + beforeEach, + inject, + onTestFinished, + type ProvidedContext, +} from "vitest"; +import { connectRoach } from "./client.ts"; + +const provided = inject("roach"); +if (provided === undefined) { + throw new Error( + "Roach: add @sentry/roach/vitest to globalSetup to use the setup file", + ); +} +// `null` is `off` mode: no run, so requests go direct. +if (provided) useRun(provided); + +/** Open a session for each test of the run. */ +function useRun(run: NonNullable): void { + const control = connectRoach(run); + beforeEach(async ({ task }) => { + // The full name starts with the file, so tests of two files never share + // a session. + const session = await control.startSession(task.fullName); + const previous = { + HTTP_PROXY: process.env.HTTP_PROXY, + HTTPS_PROXY: process.env.HTTPS_PROXY, + }; + process.env.HTTP_PROXY = session.url; + process.env.HTTPS_PROXY = session.url; + const restoreProxy = http.setGlobalProxyFromEnv({ + HTTP_PROXY: session.url, + HTTPS_PROXY: session.url, + NO_PROXY: run.noProxy, + }); + onTestFinished(async ({ task: finished }) => { + restoreProxy(); + for (const [name, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + const { missed } = await session.end(finished.result?.state === "pass"); + // `replay` mode failed these requests, so nothing went live. The test + // must fail, also when it recovered from the error. + if (missed > 0) { + throw new Error( + `Roach: ${missed} request(s) had no recording in replay mode. Run with ROACH_TOKEN to record them.`, + ); + } + }); + }); +} diff --git a/src/vitest.ts b/src/vitest.ts new file mode 100644 index 0000000..f12a49c --- /dev/null +++ b/src/vitest.ts @@ -0,0 +1,108 @@ +/** + * The vitest global setup of Roach (`@sentry/roach/vitest`). See "Use with + * vitest" in `README.md`. + * + * It starts one run with the config of `roach.json` (`config.ts`), and + * gives the proxy variables of the run to the test workers, which start + * after it. `vitest-setup.ts` opens one session for each test, so the + * tests of many workers record at the same time. At the end it closes the + * run and reports it. + * + * - `ROACH_TOKEN` is the write token. Only this file reads it, and the + * workers do not get it. Without it, the run can only replay. + * - `ROACH_URL` is the service. Default: the Roach service of Sentry. + * - `ROACH_MODE` sets the mode. See `runConfigOf()`. With `off`, it starts + * no run and does not use the service, so requests go direct. + */ +import http from "node:http"; +import tls from "node:tls"; +import type { TestProject } from "vitest/node"; +import { startRemoteRun } from "./client.ts"; +import { + DEFAULT_SERVICE_URL, + readEnv, + readRoachFile, + runConfigOf, +} from "./config.ts"; +import { writeRunReport } from "./report.ts"; +import type { RemoteRun } from "./types.ts"; + +declare module "vitest" { + export interface ProvidedContext { + /** The run of `@sentry/roach/vitest`, for the test workers. */ + roach: + | (Pick & { noProxy: string }) + | null; + } +} + +/** + * Start a Roach run for this vitest invocation. Returns the teardown, + * which ends the run and reports it. Call it from a global setup of your + * own when you have one, or use this module as a global setup. + */ +export async function startRoach( + project: Pick, +): Promise<() => Promise> { + if (typeof http.setGlobalProxyFromEnv !== "function") { + throw new Error("Roach needs Node 24.14 or later for vitest"); + } + const token = readEnv("ROACH_TOKEN"); + delete process.env.ROACH_TOKEN; + const file = await readRoachFile(); + const config = runConfigOf(file, { canWrite: token !== undefined }); + if (config.mode === "off") { + project.provide("roach", null); + return async () => {}; + } + const noProxy = file.noProxy ?? "localhost,127.0.0.1,::1"; + const run = await startRemoteRun( + { url: readEnv("ROACH_URL") ?? DEFAULT_SERVICE_URL, token }, + config, + { noProxy }, + ); + // The proxy URL has the run token. Keep it out of the CI log. + if (process.env.GITHUB_ACTIONS) { + process.stdout.write(`::add-mask::${run.token}\n`); + } + const previousEnv = Object.keys(run.env).map( + (name) => [name, process.env[name]] as const, + ); + // Test workers and their child processes start later and inherit these. + Object.assign(process.env, run.env); + // This process started before them, so it reads neither the proxy nor + // the CA file. Set both here. + const previousCa = tls.getCACertificates("default"); + tls.setDefaultCACertificates([...previousCa, run.caCert]); + const restoreProxy = http.setGlobalProxyFromEnv(run.env); + project.provide("roach", { + controlUrl: run.controlUrl, + token: run.token, + noProxy, + }); + + return async () => { + try { + const stats = await run.close(); + await writeRunReport(`Roach (${config.mode})`, stats); + // A miss in a test fails that test. A miss outside a test, such as in + // a global setup, must fail the invocation too. + const missed = Object.values(stats.counts).some( + (count) => count.missed > 0, + ); + if (missed) process.exitCode = 1; + } finally { + restoreProxy(); + tls.setDefaultCACertificates(previousCa); + for (const [name, value] of previousEnv) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + } + }; +} + +/** The vitest global setup. */ +export default function setup(project: TestProject) { + return startRoach(project); +} diff --git a/tests/recording.test.ts b/tests/recording.test.ts index 8e6c1c6..211f6b7 100644 --- a/tests/recording.test.ts +++ b/tests/recording.test.ts @@ -85,6 +85,21 @@ async function send(bodies: unknown[], target = `${origin}/v1/messages`) { return responses; } +/** Send one request through the proxy URL `url`, such as of a session. */ +async function sendThrough(url: string, body: unknown): Promise { + const dispatcher = new ProxyAgent({ uri: url, proxyTunnel: true }); + try { + const response = await request(`${origin}/v1/messages`, { + body: JSON.stringify(body), + dispatcher, + method: "POST", + }); + await response.body.dump(); + } finally { + await dispatcher.close(); + } +} + /** Send the requests of one test session, then end it. */ async function session(running: RemoteRoach, bodies: unknown[], passed = true) { const opened = await running.startSession("test"); @@ -311,6 +326,37 @@ describe("recording", () => { expect(liveRequests).toBe(1); }); + it("keeps concurrent sessions of one run apart by their proxy URLs", async () => { + const running = await start("auto"); + const failing = await running.startSession("fails"); + const passing = await running.startSession("passes"); + await Promise.all([ + sendThrough(failing.url, { model: "dropped" }), + sendThrough(passing.url, { model: "kept" }), + ]); + // With two open sessions, the run URL names no session, so the request + // is outside both and is written at once. + await send([{ model: "outside" }]); + await failing.end(false); + await passing.end(true); + + const recorded = await Promise.all( + (await files()).map(async (file) => { + const text = await readFile( + path.join(directory, TENANT, "model", file), + "utf8", + ); + return (JSON.parse(text) as { session?: string }).session ?? null; + }), + ); + expect(recorded).toHaveLength(2); + expect(recorded).toEqual(expect.arrayContaining([null, "passes"])); + await expect(running.stats()).resolves.toMatchObject({ + written: 2, + discarded: 1, + }); + }); + it("records again in record mode", async () => { const running = await start("record"); await session(running, [{ model: "m" }]); diff --git a/tests/service.test.ts b/tests/service.test.ts index 581e6aa..47c414e 100644 --- a/tests/service.test.ts +++ b/tests/service.test.ts @@ -10,11 +10,8 @@ import { startRemoteRun, type RemoteRoach, } from "../src/client.ts"; -import { - startRoachService, - type RoachService, - type RunConfig, -} from "../src/service.ts"; +import { startRoachService, type RoachService } from "../src/service.ts"; +import type { RunConfig } from "../src/types.ts"; import { VALUE_PATTERNS } from "../src/values.ts"; let upstream: Server; diff --git a/tests/vitest.test.ts b/tests/vitest.test.ts new file mode 100644 index 0000000..7d10ab5 --- /dev/null +++ b/tests/vitest.test.ts @@ -0,0 +1,203 @@ +/** + * The vitest integration (`src/vitest.ts`, `src/vitest-setup.ts`). Each + * test runs vitest in a child process on a small project with a + * `roach.json`, against a service and an upstream in this process. + */ +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createServer, type Server } from "node:http"; +import { + mkdtemp, + readdir, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { startRoachService, type RoachService } from "../src/service.ts"; + +const TOKEN = "write-token"; +const TENANT = "acme/vitest"; +const ROOT = path.resolve(import.meta.dirname, ".."); + +let upstream: Server; +let origin: string; +let liveRequests: string[]; +let directory: string; +let project: string; +let service: RoachService; + +/** A test file of the project. `FAIL` makes its second test fail. */ +const testFile = (model: string) => ` +import { execFileSync } from "node:child_process"; +import { expect, it } from "vitest"; + +const send = async (model) => { + const response = await fetch(\`\${process.env.UPSTREAM}/v1/messages\`, { + method: "POST", + body: JSON.stringify({ model }), + }); + await response.text(); + return response.headers.get("x-roach"); +}; + +it("passes", async () => { + const via = await send("${model}"); + // In \`off\` mode the request goes direct, so no proxy marks it. + if (process.env.ROACH_MODE === "off") expect(via).toBeNull(); + else expect(via).toMatch(/^(live|replayed)$/); + // A child process sends through the session of its test too. + execFileSync(process.execPath, [ + "-e", + \`fetch("\${process.env.UPSTREAM}/v1/messages", { method: "POST", body: JSON.stringify({ model: "${model}-child" }) }).then((r) => r.text())\`, + ]); +}); + +it.skipIf(!process.env.FAIL)("fails", async () => { + await send("${model}-fail"); + throw new Error("expected"); +}); +`; + +/** Run vitest on the project. Returns its exit code and output. */ +function runVitest( + env: Record, +): Promise<{ code: number | null; output: string }> { + const child = spawn( + process.execPath, + [path.join(ROOT, "node_modules/vitest/vitest.mjs"), "run"], + { + cwd: project, + env: { + PATH: process.env.PATH, + UPSTREAM: origin, + ROACH_URL: service.url, + ...env, + }, + }, + ); + let output = ""; + child.stdout.on("data", (chunk: Buffer) => (output += chunk)); + child.stderr.on("data", (chunk: Buffer) => (output += chunk)); + return new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => resolve({ code, output })); + }); +} + +/** The sessions of the recordings of the tenant. */ +async function recordedSessions(): Promise { + const rule = path.join(directory, TENANT, "model"); + const files = await readdir(rule).catch(() => []); + const sessions = await Promise.all( + files.map(async (file) => { + const text = await readFile(path.join(rule, file), "utf8"); + return (JSON.parse(text) as { session?: string }).session ?? ""; + }), + ); + return sessions.toSorted((a, b) => a.localeCompare(b)); +} + +beforeEach(async () => { + liveRequests = []; + upstream = createServer((incoming, outgoing) => { + const chunks: Buffer[] = []; + incoming.on("data", (chunk: Buffer) => chunks.push(chunk)); + incoming.on("end", () => { + liveRequests.push(Buffer.concat(chunks).toString("utf8")); + outgoing.writeHead(200, { "content-type": "application/json" }); + outgoing.end(`{"n":${liveRequests.length}}`); + }); + }); + await new Promise((resolve) => + upstream.listen(0, "127.0.0.1", resolve), + ); + const address = upstream.address(); + if (!address || typeof address === "string") throw new Error("No port"); + origin = `http://127.0.0.1:${address.port}`; + directory = await mkdtemp(path.join(tmpdir(), "roach-")); + service = await startRoachService({ + directory, + privateNetworks: true, + writeTokenHash: createHash("sha256").update(TOKEN).digest("hex"), + }); + + project = await mkdtemp(path.join(tmpdir(), "roach-vitest-")); + await symlink( + path.join(ROOT, "node_modules"), + path.join(project, "node_modules"), + ); + await writeFile( + path.join(project, "roach.json"), + JSON.stringify({ + tenant: TENANT, + allow: [origin], + // The upstream is on loopback, which the default skips. + noProxy: "none.invalid", + rules: [{ name: "model", match: { method: "POST", url: `${origin}/` } }], + }), + ); + await writeFile( + path.join(project, "vitest.config.ts"), + `export default { + test: { + include: ["*.fixture.ts"], + globalSetup: [${JSON.stringify(path.join(ROOT, "src/vitest.ts"))}], + setupFiles: [${JSON.stringify(path.join(ROOT, "src/vitest-setup.ts"))}], + }, +}; +`, + ); + await writeFile(path.join(project, "a.fixture.ts"), testFile("a")); + await writeFile(path.join(project, "b.fixture.ts"), testFile("b")); +}); + +afterEach(async () => { + await service.close(); + await new Promise((resolve) => upstream.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); + await rm(project, { recursive: true, force: true }); +}); + +describe("vitest", () => { + it("records each test in its own session, then replays without the token", async () => { + const recorded = await runVitest({ ROACH_TOKEN: TOKEN, FAIL: "1" }); + expect(recorded).toMatchObject({ code: 1 }); + expect(recorded.output).toContain("Roach (auto):"); + // Each test file runs in its own worker, at the same time. The failed + // tests dropped their recordings. + await expect(recordedSessions()).resolves.toEqual([ + "a.fixture.ts > passes", + "a.fixture.ts > passes", + "b.fixture.ts > passes", + "b.fixture.ts > passes", + ]); + expect(liveRequests).toHaveLength(6); + + const replayed = await runVitest({}); + expect(replayed).toMatchObject({ code: 0 }); + expect(replayed.output).toContain("Roach (replay):"); + expect(replayed.output).toContain("model 4 replayed, 0 live"); + expect(liveRequests).toHaveLength(6); + }); + + it("sends requests direct in off mode, without the service", async () => { + const result = await runVitest({ + ROACH_MODE: "off", + ROACH_URL: "http://127.0.0.1:1", + }); + expect(result).toMatchObject({ code: 0 }); + expect(liveRequests).toHaveLength(4); + await expect(recordedSessions()).resolves.toEqual([]); + }); + + it("fails a test whose request has no recording in replay mode", async () => { + const result = await runVitest({}); + expect(result.code).toBe(1); + expect(result.output).toContain("had no recording in replay mode"); + expect(liveRequests).toHaveLength(0); + }); +});