diff --git a/AGENTS.md b/AGENTS.md index 5d6b6c7..45b2421 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,14 +7,16 @@ - `yarn install` — install dependencies (also runs `yarn build` via `prepare`) - `yarn build` — compile TypeScript - `yarn lint` — ESLint -- `yarn test` — vitest (unit + contract tests pass without a running API; e2e/integration tests need `HYPERBROWSER_API_KEY`) +- `yarn test` — unit + local integration tests; no API credentials or Docker daemon needed +- `yarn test:unit` / `yarn test:integration` — run either local test group +- `yarn test:e2e` — live API tests; requires `HYPERBROWSER_API_KEY` - `yarn format` — Prettier ### Gotchas - `yarn install` triggers the `prepare` script which runs `yarn build`. If the build fails on install, check for TypeScript errors in `src/`. -- Integration and e2e tests (`tests/sandbox/e2e/`, `tests/integration/`) require - a running Hyperbrowser API and a valid `HYPERBROWSER_API_KEY`. Without these, - only the contract/unit tests in the suite will pass; the e2e tests fail with - `ECONNREFUSED`. +- `vitest.config.ts` defines unit, integration, and e2e projects. The default + selection (including watch mode) runs only unit and integration tests. +- Only live tests in `tests/e2e/` load env files and require a running + Hyperbrowser API and a valid `HYPERBROWSER_API_KEY`. diff --git a/README.md b/README.md index ac190af..dba5978 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,70 @@ await sandbox.stop(); `connect()` refreshes runtime auth and throws if the sandbox is no longer running. +Run commands and stream complete output. `exec()` and `processes.start()` open a +single streamed request and collect stdout/stderr from process start, so output is +complete even beyond the receiver's replay window. `wait()` timeouts are local and +keep collecting; `disconnect()` stops collecting without killing the process. + +```typescript +const result = await sandbox.exec("npm test", { + cwd: "/workspace", + maxOutputBytes: 128 * 1024 * 1024, // default 64 MiB; exceeding it fails, never truncates +}); +console.log(result.exitCode, result.stdout); + +const proc = await sandbox.processes.start("tail -f /var/log/app.log"); +try { + await proc.wait({ timeoutSec: 5 }); +} catch (error) { + // Local wait timeout: the process is still running and output is still collected. +} +for await (const event of proc.stream()) { + if (event.type === "stdout") process.stdout.write(event.data); + if (event.type === "exit") console.log(event.result.exitCode); +} +proc.disconnect(); +``` + +Build a custom sandbox image from a Dockerfile or a local Docker image. `getOrBuildImage()` +derives a content-based image name, reuses a ready image with the same identity, joins a +matching in-progress build, or creates a new one. Dockerfile builds package the effective +build context (Dockerfile sources, `.dockerignore`) and build remotely; no local Docker is +required. `dockerImage` imports a local `linux/amd64` image via the Docker CLI. + +```typescript +const resolved = await client.sandboxes.getOrBuildImage({ + contextPath: "./services/api", + dockerfile: "Dockerfile", // relative to contextPath + imageInit: { env: { NODE_ENV: "production" }, workingDir: "/app" }, + builderCpus: 4, + builderMemoryMiB: 8192, + builderScratchMiB: 20480, +}); +console.log(resolved.outcome, resolved.imageName, resolved.imageId); // "reused" | "joined" | "created" + +const sandbox = await client.sandboxes.create({ imageName: resolved.imageName }); + +// Import a local Docker image instead (requires docker CLI, linux/amd64 image): +const imported = await client.sandboxes.getOrBuildImage({ dockerImage: "myorg/app:1.2.3" }); + +// Detached build: return immediately and poll later. +const pending = await client.sandboxes.getOrBuildImage({ contextPath: ".", wait: false }); +if (pending.build) { + const build = await client.sandboxes.waitForImageBuild(pending.build.id, { + pollInterval: 3, + timeout: 35 * 60, + }); + console.log(build.status, build.imageId); +} +``` + +Lower-level helpers are also available: `buildImageFromDockerfile()`, +`buildImageFromDockerImage()`, `findReadyImage()`, `reuseDockerImage()`, plus the raw +`createImageBuild()` / `completeImageBuild()` / `getImageBuild()` / `listImageBuilds()` / +`cancelImageBuild()` APIs. Control-plane `GET` requests retry transient failures +(429/502/503/504 and network errors) up to three times with jittered backoff. + Create a sandbox with pre-exposed ports: ```typescript @@ -339,3 +403,129 @@ const terminal = await sandbox.terminal.create({ const connection = await terminal.attach(10); ``` + +### Streaming file transfers and watches + +`read({ format: "stream" })` retains its buffered behavior. Use `uploadStream()` +and `downloadStream()` for large transfers with backpressure and bounded memory: + +```typescript +import { createReadStream, createWriteStream } from "node:fs"; +import { pipeline } from "node:stream/promises"; + +await sandbox.files.withRunAs("root").uploadStream( + "/tmp/archive.tar", createReadStream("./archive.tar"), +); +await pipeline(sandbox.files.downloadStream("/tmp/archive.tar"), createWriteStream("./copy.tar")); +``` + +Uploads accept a Node readable or iterable of string/byte chunks and optional +`{ contentLength, signal }`. Downloads return an async generator of buffers and +accept `{ signal }`. Breaking download iteration closes the request. After an +authentication failure, a consumed upload is rejected with `stream_not_replayable`; +open a fresh source to retry it. `files.stat`, `mkdir`, `move`, and `delete` are +aliases; `rename(oldPath, newPath, { overwrite: false })` forwards overwrite policy. + +```typescript +const watch = await sandbox.files.watch("/workspace", { recursive: true }); +try { + for await (const message of watch.events({ cursor: 0, route: "ws" })) { + if (message.type === "done") break; + console.log(message.event.seq, message.event.path, message.event.op); + break; // Close this connection; the remote watch remains active. + } + // Persist watch.id and the last sequence to resume an active watch: + const resumed = await sandbox.files.getWatch(watch.id, true); + await resumed.refresh(true); + console.log(resumed.current, resumed.toJSON()); +} finally { + await watch.stop(); +} +``` + +Both watch routes (`ws` and `stream`) use WebSocket transport. Watch events include +an explicit `done` envelope. `watchDir()` remains the callback convenience API. +Stopping a watcher stops the remote watch; breaking event iteration only closes +that connection. Watch timestamps remain milliseconds; existing file metadata +continues to expose `modifiedTime` as a `Date`. + +### Image identities, snapshots, and runtime sessions + +Offline identity helpers are available from the package root and +`@hyperbrowser/sdk/image-builds`: + +```typescript +import { + dockerBuildContextFingerprint, imageBuildName, DockerBuildContextChangedError, +} from "@hyperbrowser/sdk/image-builds"; + +const fingerprint = await dockerBuildContextFingerprint("./app"); +const imageName = imageBuildName({ source: "dockerfile", fingerprint }); +try { + await client.sandboxes.buildImageFromDockerfile({ + contextPath: "./app", imageName, expectedContextFingerprint: fingerprint, + builderCpus: 4, builderMemoryMiB: 8192, builderScratchMiB: 20480, + }); +} catch (error) { + if (error instanceof DockerBuildContextChangedError) { + // Recompute identity after a local edit, then retry explicitly. + } else throw error; +} + +const restored = await client.sandboxes.startFromSnapshot({ snapshotName: "saved" }); +const session = await restored.createRuntimeSession({ forceRefresh: true }); +// Also available without keeping a handle: +await client.sandboxes.getRuntimeSession(restored.id); +``` + +Remote Dockerfile builds need no local Docker. Local builds/imports require Docker; +platform-specific digest lookup requires Docker API 1.49+ (Docker 28.1+). Imports +preserve image `PATH`, entrypoint/CMD, working directory, and supported environment +variables; explicit image initialization overrides take precedence. Names include +context/digest, platform, and initialization overrides. Mutable base tags and +network downloads are not resolved by fingerprinting; use `forceBuild` when needed. + +Only `linux/amd64` image builds are supported. Sandbox creation preserves +`runtimeClass: "firecracker" | "gvisor-cpu"`; use response capabilities when choosing +snapshot, volume, or exposure operations. Snapshot launches cannot override image +resources. Invalid or conflicting launch sources fail before a network request. + +### Timeouts, cancellation, and compatibility + +- Client `timeout` is milliseconds. Normal runtime requests have separate header + and body budgets. Streaming transfers reset inactivity budgets as data moves. +- Process `timeoutMs` / `timeoutSec` limit remote execution. `wait({ timeoutMs })` + only limits local waiting and leaves collection running. +- `exec()` / `processes.start()` accept an `AbortSignal`. Aborting stops local + collection and closes its connection; the detached remote process continues. + Use `signal()` or `kill()` when you intend to stop the command. +- Process stream inactivity is limited to 60 seconds; receiver keepalives reset it. + Incomplete output, exceeded output limits, and unsupported receivers produce + structured errors. A failed streaming start is never automatically re-executed. +- Image polling `pollInterval`, `waitTimeout`, and `uploadTimeout` are seconds. + `getOrBuildImage()` defaults uploads to 600 seconds of inactivity and polling to + 35 minutes. Explicit `null` disables the corresponding timeout. Polling deadlines + include control GET retries. A polling `signal` or timeout leaves accepted builds + running for other callers. Lower-level build helpers leave upload timeouts unset + unless explicitly supplied. +- `HYPERBROWSER_BASE_URL` supplies the API base URL when `baseUrl` is not provided. + +Public handle classes are available from `@hyperbrowser/sdk/sandbox`; request and +response types remain under `@hyperbrowser/sdk/types`. + +### Development checks + +The supported Node baseline is 20.20.2. Run `yarn build`, `yarn typecheck`, +`yarn lint`, and `yarn test` for local verification. Tests share `vitest.config.ts`: + +- `tests/unit`: isolated logic and mocked contracts (`yarn test:unit`). +- `tests/integration`: local HTTP, WebSocket, filesystem, and subprocess tests + (`yarn test:integration`). +- `tests/e2e`: live Hyperbrowser API tests (`yarn test:e2e`). + +`yarn test` and `yarn test:watch` select unit and integration tests by default; +neither requires API credentials or a Docker daemon. + +Live tests are opt-in: set `HYPERBROWSER_API_KEY` and `HYPERBROWSER_BASE_URL`, then +run `yarn test:e2e`. They create remote resources and require a receiver supporting +streamed process starts. diff --git a/package.json b/package.json index 6179ad2..65ed6e8 100644 --- a/package.json +++ b/package.json @@ -17,13 +17,15 @@ "license": "MIT", "scripts": { "build": "tsc", - "lint": "eslint src/**/*.ts", + "lint": "eslint 'src/**/*.ts'", "prepare": "yarn build", "test": "vitest run", - "test:e2e": "vitest run tests/sandbox/e2e", - "test:integration": "vitest run tests/integration", + "test:unit": "vitest run --project unit", + "test:e2e": "vitest run --project e2e", + "test:integration": "vitest run --project integration", "test:watch": "vitest", - "format": "prettier --write 'src/**/*.ts'" + "format": "prettier --write 'src/**/*.ts'", + "typecheck": "tsc --noEmit && tsc -p tsconfig.tests.json" }, "files": [ "dist", @@ -73,6 +75,14 @@ "./tools": { "types": "./dist/tools/index.d.ts", "default": "./dist/tools/index.js" + }, + "./image-builds": { + "types": "./dist/image-builds.d.ts", + "default": "./dist/image-builds.js" + }, + "./sandbox": { + "types": "./dist/sandbox/index.d.ts", + "default": "./dist/sandbox/index.js" } }, "typesVersions": { @@ -82,7 +92,16 @@ ], "tools": [ "./dist/tools/index.d.ts" + ], + "image-builds": [ + "./dist/image-builds.d.ts" + ], + "sandbox": [ + "./dist/sandbox/index.d.ts" ] } + }, + "engines": { + "node": ">=20.20.2" } } diff --git a/src/client.ts b/src/client.ts index 5f585dc..7e29b31 100644 --- a/src/client.ts +++ b/src/client.ts @@ -19,42 +19,9 @@ import { WebService } from "./services/web"; import { SandboxesService } from "./services/sandboxes"; import { VolumesService } from "./services/volumes"; -export type HyperbrowserService = "control" | "runtime"; - -export interface HyperbrowserErrorOptions { - statusCode?: number; - code?: string; - requestId?: string; - retryable?: boolean; - service?: HyperbrowserService; - details?: unknown; - cause?: unknown; -} - -export class HyperbrowserError extends Error { - public readonly statusCode?: number; - public readonly code?: string; - public readonly requestId?: string; - public readonly retryable: boolean; - public readonly service?: HyperbrowserService; - public readonly details?: unknown; - public readonly cause?: unknown; - - constructor(message: string, options: number | HyperbrowserErrorOptions = {}) { - super(`[Hyperbrowser]: ${message}`); - this.name = "HyperbrowserError"; - - const normalized = typeof options === "number" ? { statusCode: options } : options; - - this.statusCode = normalized.statusCode; - this.code = normalized.code; - this.requestId = normalized.requestId; - this.retryable = normalized.retryable ?? false; - this.service = normalized.service; - this.details = normalized.details; - this.cause = normalized.cause; - } -} +import { HyperbrowserError } from "./error"; +export { HyperbrowserError } from "./error"; +export type { HyperbrowserErrorOptions, HyperbrowserService } from "./error"; export class HyperbrowserClient { public readonly sessions: SessionsService; @@ -81,7 +48,8 @@ export class HyperbrowserClient { constructor(config: HyperbrowserConfig = {}) { const apiKey = config.apiKey || process.env["HYPERBROWSER_API_KEY"]; - const baseUrl = config.baseUrl || "https://api.hyperbrowser.ai"; + const baseUrl = + config.baseUrl || process.env["HYPERBROWSER_BASE_URL"] || "https://api.hyperbrowser.ai"; const timeout = config.timeout || 30000; const runtimeProxyOverride = config.runtimeProxyOverride?.trim() || undefined; if (!apiKey) { diff --git a/src/error.ts b/src/error.ts new file mode 100644 index 0000000..910e9b7 --- /dev/null +++ b/src/error.ts @@ -0,0 +1,54 @@ +export type HyperbrowserService = "control" | "runtime"; + +/** Keep diagnostics useful without retaining query credentials or URL hosts. */ +export const errorRequestPath = (target: string): string => { + try { + return new URL(target, "http://sdk.invalid").pathname; + } catch { + return target.split("?", 1)[0]; + } +}; + +export const networkErrorMessage = (error: unknown, fallback: string): string => + error instanceof Error ? error.message || `${fallback} (${error.name || "Error"})` : fallback; + +export interface HyperbrowserErrorOptions { + statusCode?: number; + code?: string; + requestId?: string; + retryable?: boolean; + service?: HyperbrowserService; + details?: unknown; + cause?: unknown; + method?: string; + path?: string; +} + +export class HyperbrowserError extends Error { + public readonly statusCode?: number; + public readonly code?: string; + public readonly requestId?: string; + public readonly retryable: boolean; + public readonly service?: HyperbrowserService; + public readonly details?: unknown; + public readonly cause?: unknown; + public readonly method?: string; + public readonly path?: string; + + constructor(message: string, options: number | HyperbrowserErrorOptions = {}) { + super(`[Hyperbrowser]: ${message}`); + this.name = "HyperbrowserError"; + + const normalized = typeof options === "number" ? { statusCode: options } : options; + + this.statusCode = normalized.statusCode; + this.code = normalized.code; + this.requestId = normalized.requestId; + this.retryable = normalized.retryable ?? false; + this.service = normalized.service; + this.details = normalized.details; + this.cause = normalized.cause; + this.method = normalized.method; + this.path = normalized.path; + } +} diff --git a/src/image-builds.ts b/src/image-builds.ts new file mode 100644 index 0000000..af4249e --- /dev/null +++ b/src/image-builds.ts @@ -0,0 +1,9 @@ +/** Public, offline image identity helpers. */ +export { + dockerBuildContextFingerprint, + DockerBuildContextChangedError, +} from "./sandbox/image-build/context"; +export type { DockerBuildContextOptions } from "./sandbox/image-build/context"; +export { imageBuildName } from "./sandbox/image-build/resolution"; +export type { ImageBuildNameOptions, ImageBuildSource } from "./sandbox/image-build/resolution"; +export { dockerImageDigest, DockerCommandError } from "./sandbox/image-build/docker-image"; diff --git a/src/index.ts b/src/index.ts index 5691064..b8d510a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,22 @@ +import { + dockerBuildContextFingerprint, + DockerBuildContextChangedError, + imageBuildName, + dockerImageDigest, + DockerCommandError, +} from "./image-builds"; +export { + dockerBuildContextFingerprint, + DockerBuildContextChangedError, + imageBuildName, + dockerImageDigest, + DockerCommandError, +}; +export type { + DockerBuildContextOptions, + ImageBuildNameOptions, + ImageBuildSource, +} from "./image-builds"; import { HyperbrowserClient, HyperbrowserError } from "./client"; // Export HyperbrowserClient as Hyperbrowser for named imports @@ -13,5 +32,10 @@ if (typeof module !== "undefined" && module.exports) { module.exports.Hyperbrowser = HyperbrowserClient; module.exports.HyperbrowserClient = HyperbrowserClient; module.exports.HyperbrowserError = HyperbrowserError; + module.exports.dockerBuildContextFingerprint = dockerBuildContextFingerprint; + module.exports.DockerBuildContextChangedError = DockerBuildContextChangedError; + module.exports.imageBuildName = imageBuildName; + module.exports.dockerImageDigest = dockerImageDigest; + module.exports.DockerCommandError = DockerCommandError; module.exports.default = HyperbrowserClient; } diff --git a/src/retry.ts b/src/retry.ts new file mode 100644 index 0000000..0dd1649 --- /dev/null +++ b/src/retry.ts @@ -0,0 +1,67 @@ +import { HyperbrowserError } from "./error"; + +export const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504]); +export const GET_RETRY_MAX_ATTEMPTS = 3; +export const GET_RETRY_INITIAL_DELAY_MS = 250; +export const GET_RETRY_MAX_DELAY_MS = 1_000; + +const RETRYABLE_NETWORK_CODES = new Set([ + "ECONNRESET", + "ECONNREFUSED", + "EAI_AGAIN", + "ETIMEDOUT", + "ESOCKETTIMEDOUT", +]); + +export const isRetryableNetworkError = (error: unknown): boolean => { + if (!(error instanceof Error)) { + return false; + } + + const networkError = error as Error & { code?: string; type?: string }; + return ( + networkError.name === "AbortError" || + networkError.type === "request-timeout" || + (networkError.code ? RETRYABLE_NETWORK_CODES.has(networkError.code) : false) + ); +}; + +export const shouldRetryGet = ( + method: string, + error: HyperbrowserError, + failedAttempt: number +): boolean => + method.toUpperCase() === "GET" && error.retryable && failedAttempt < GET_RETRY_MAX_ATTEMPTS; + +/** Exponential backoff with jitter, capped at one second. */ +export const getRetryDelayMs = (failedAttempt: number): number => { + const maximumDelay = Math.min( + GET_RETRY_INITIAL_DELAY_MS * 2 ** (failedAttempt - 1), + GET_RETRY_MAX_DELAY_MS + ); + return maximumDelay / 2 + Math.random() * (maximumDelay / 2); +}; + +export const retryDelay = ( + ms: number, + signal?: { + readonly aborted: boolean; + addEventListener: AbortSignal["addEventListener"]; + removeEventListener: AbortSignal["removeEventListener"]; + } +): Promise => + new Promise((resolve, reject) => { + const aborted = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", aborted); + reject( + new HyperbrowserError("Request canceled", { code: "request_aborted", service: "control" }) + ); + }; + const timer = setTimeout(() => { + signal?.removeEventListener("abort", aborted); + resolve(); + }, ms); + if (signal?.aborted) aborted(); + else signal?.addEventListener("abort", aborted, { once: true }); + }); diff --git a/src/sandbox/base.ts b/src/sandbox/base.ts index b1b020a..c73eac2 100644 --- a/src/sandbox/base.ts +++ b/src/sandbox/base.ts @@ -1,5 +1,8 @@ +import { Readable } from "stream"; +import { StringDecoder } from "string_decoder"; import fetch, { RequestInit, Response } from "node-fetch"; -import { HyperbrowserError } from "../client"; +import { errorRequestPath, HyperbrowserError, networkErrorMessage } from "../error"; +import { isRetryableNetworkError, RETRYABLE_STATUS_CODES } from "../retry"; import { resolveRuntimeTransportTarget } from "./ws"; export interface RuntimeConnection { @@ -17,33 +20,67 @@ export interface RuntimeSSEEvent { type RuntimeParams = Record; -const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504]); -const RETRYABLE_NETWORK_CODES = new Set([ - "ECONNRESET", - "ECONNREFUSED", - "EAI_AGAIN", - "ETIMEDOUT", - "ESOCKETTIMEDOUT", -]); +// The receiver sends process SSE keepalives every 15 seconds. +export const PROCESS_STREAM_IDLE_TIMEOUT_MS = 60_000; + +export interface RuntimeSSEInit { + method?: "GET" | "POST"; + body?: string; + headers?: Record; + /** Milliseconds without any bytes before the stream fails. Defaults to 60s. */ + idleTimeoutMs?: number; + signal?: AbortSignal; +} + +export interface RuntimeSSEStream { + events: AsyncGenerator; + /** Close the underlying connection; the remote process keeps running. */ + close(): void; +} const getRequestId = (response: Response): string | undefined => { return response.headers.get("x-request-id") || response.headers.get("request-id") || undefined; }; -const isRetryableNetworkError = (error: unknown): boolean => { - if (!(error instanceof Error)) { - return false; - } - - const networkError = error as Error & { code?: string; type?: string }; - return ( - networkError.name === "AbortError" || - networkError.type === "request-timeout" || - (networkError.code ? RETRYABLE_NETWORK_CODES.has(networkError.code) : false) - ); -}; +const isEventStream = (response: Response): boolean => + (response.headers.get("content-type") || "").includes("text/event-stream"); export class RuntimeTransport { + private readonly responseCleanup = new WeakMap void>(); + + private closeResponse(response: Response): void { + this.responseCleanup.get(response)?.(); + } + + private failure(error: unknown, path: string, init?: RequestInit, response?: Response): HyperbrowserError { + if (error instanceof HyperbrowserError) { + return new HyperbrowserError(error.message.replace(/^\[Hyperbrowser\]: /, ""), { + statusCode: error.statusCode ?? response?.status, + code: error.code, + requestId: error.requestId ?? (response ? getRequestId(response) : undefined), + service: error.service ?? "runtime", + retryable: error.retryable, + details: error.details, + cause: error.cause ?? error, + method: error.method ?? init?.method ?? "GET", + path: error.path ?? errorRequestPath(path), + }); + } + const canceled = init?.signal?.aborted; + return new HyperbrowserError( + networkErrorMessage(error, "Runtime request failed"), + { + code: canceled ? "request_aborted" : undefined, + statusCode: response?.status, + requestId: response ? getRequestId(response) : undefined, + service: "runtime", + retryable: !canceled && isRetryableNetworkError(error), + method: init?.method ?? "GET", + path: errorRequestPath(path), + cause: error, + } + ); + } constructor( private readonly resolveConnection: (forceRefresh?: boolean) => Promise, private readonly timeout: number = 30000, @@ -52,45 +89,159 @@ export class RuntimeTransport { async requestJSON(path: string, init?: RequestInit, params?: RuntimeParams): Promise { const response = await this.fetchWithAuth(path, init, params); - if (response.headers.get("content-length") === "0") { - return {} as T; - } - try { - return (await response.json()) as T; - } catch { - throw new HyperbrowserError("Failed to parse JSON response", { - statusCode: response.status, - requestId: getRequestId(response), - retryable: false, - service: "runtime", - }); + const text = await response.text(); + if (!text) return {} as T; + try { + return JSON.parse(text) as T; + } catch (cause) { + throw new HyperbrowserError("Failed to parse JSON response", { + statusCode: response.status, + requestId: getRequestId(response), + service: "runtime", + cause, + method: init?.method ?? "GET", + path: errorRequestPath(path), + }); + } + } catch (error) { + throw this.failure(error, path, init, response); + } finally { + this.closeResponse(response); } } async requestBuffer(path: string, init?: RequestInit, params?: RuntimeParams): Promise { const response = await this.fetchWithAuth(path, init, params); - return response.buffer(); + try { + return await response.buffer(); + } catch (error) { + throw this.failure(error, path, init, response); + } finally { + this.closeResponse(response); + } + } + + /** Pull-based binary transfer. Breaking iteration closes the HTTP connection. */ + async *streamBytes( + path: string, + init?: RequestInit, + params?: RuntimeParams + ): AsyncGenerator { + const response = await this.fetchWithAuth(path, init, params, true, true); + try { + if (!response.body) return; + const iterator = (response.body as Readable)[Symbol.asyncIterator](); + for (;;) { + const next = await this.readChunk(iterator, this.timeout); + if (next.done) return; + yield Buffer.from(next.value); + } + } catch (error) { + throw this.failure(error, path, init, response); + } finally { + this.closeResponse(response); + } + } + + private readChunk(iterator: AsyncIterator, timeout: number): Promise> { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => + reject( + new HyperbrowserError( + `Runtime stream idle for ${timeout}ms without data or keepalives`, + { code: "stream_idle_timeout", service: "runtime", retryable: true } + ) + ), + timeout + ); + iterator.next().then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + } + ); + }); } async *streamSSE(path: string, params?: RuntimeParams): AsyncGenerator { + const stream = await this.openSSE(path, params); + try { + yield* stream.events; + } finally { + stream.close(); + } + } + + /** + * Open a server-sent event stream. POST streams carry a JSON body and + * require an event-stream response, since the request may have side effects + * that must not be retried blindly. + */ + async openSSE( + path: string, + params?: RuntimeParams, + init: RuntimeSSEInit = {} + ): Promise { + const method = init.method ?? "GET"; + const headers: Record = { + Accept: "text/event-stream", + ...(init.body !== undefined ? { "content-type": "application/json" } : {}), + ...(init.headers ?? {}), + }; const response = await this.fetchWithAuth( path, - { - method: "GET", - headers: { - Accept: "text/event-stream", - }, - }, - params + { method, headers, body: init.body, signal: init.signal }, + params, + true, + true ); - + if (method === "POST" && !isEventStream(response)) { + this.closeResponse(response); + throw new HyperbrowserError( + "Receiver does not support streaming command start; update the receiver. " + + "The command may have started; do not retry it automatically.", + { code: "streaming_not_supported", service: "runtime", retryable: false, + method, path: errorRequestPath(path), requestId: getRequestId(response), statusCode: response.status } + ); + } const body = response.body; + const idleTimeoutMs = init.idleTimeoutMs ?? PROCESS_STREAM_IDLE_TIMEOUT_MS; + let closed = false; + const close = (): void => { + if (closed) { + return; + } + closed = true; + this.closeResponse(response); + }; + const parsed = this.parseSSE(body, idleTimeoutMs, () => closed, close, init.signal); + const failure = (error: unknown) => this.failure(error, path, { method, signal: init.signal }, response); + const events = (async function* () { + try { yield* parsed; } catch (error) { throw failure(error); } + })(); + return { events, close }; + } + + private async *parseSSE( + body: NodeJS.ReadableStream | null, + idleTimeoutMs: number, + isClosed: () => boolean, + close: () => void, + signal?: AbortSignal + ): AsyncGenerator { if (!body) { return; } let buffer = ""; + let skipLineFeed = false; + const decoder = new StringDecoder("utf8"); let eventName = "message"; let eventId: string | undefined; let dataLines: string[] = []; @@ -122,56 +273,85 @@ export class RuntimeTransport { return event; }; - for await (const chunk of body) { - buffer += Buffer.from(chunk).toString("utf8"); - - while (true) { - const newlineIndex = buffer.indexOf("\n"); - if (newlineIndex === -1) { + const readLine = (line: string): RuntimeSSEEvent | null => { + if (line === "") return flushEvent(); + if (line.startsWith(":")) return null; + const separator = line.indexOf(":"); + const field = separator === -1 ? line : line.slice(0, separator); + // Match the Python transport's field-value space normalization. + const value = separator === -1 ? "" : line.slice(separator + 1).replace(/^ +/, ""); + switch (field) { + case "event": + eventName = value || "message"; break; - } + case "data": + dataLines.push(value); + break; + case "id": + eventId = value; + break; + } + return null; + }; - let line = buffer.slice(0, newlineIndex); - buffer = buffer.slice(newlineIndex + 1); - if (line.endsWith("\r")) { - line = line.slice(0, -1); - } + const iterator = (body as AsyncIterable)[Symbol.asyncIterator](); - if (line === "") { - const event = flushEvent(); - if (event) { - yield event; + try { + for (;;) { + let next: IteratorResult; + try { + next = await this.readChunk(iterator, idleTimeoutMs); + } catch (error) { + if (signal?.aborted) + throw new HyperbrowserError("Command collection canceled", { + code: "request_aborted", + service: "runtime", + cause: error, + }); + if (isClosed()) { + return; + } + if (error instanceof HyperbrowserError) { + throw error; } - continue; + throw new HyperbrowserError( + error instanceof Error ? error.message : "Runtime stream failed", + { service: "runtime", retryable: isRetryableNetworkError(error), cause: error } + ); } - - if (line.startsWith(":")) { - continue; + if (next.done) { + break; } + buffer += decoder.write(Buffer.from(next.value)); - const separator = line.indexOf(":"); - const field = separator === -1 ? line : line.slice(0, separator); - const value = separator === -1 ? "" : line.slice(separator + 1).replace(/^ /, ""); - - switch (field) { - case "event": - eventName = value || "message"; - break; - case "data": - dataLines.push(value); - break; - case "id": - eventId = value; - break; - default: + while (true) { + if (skipLineFeed) { + if (!buffer) break; + if (buffer.startsWith("\n")) buffer = buffer.slice(1); + skipLineFeed = false; + } + const newlineIndex = buffer.search(/[\r\n]/); + if (newlineIndex === -1) { break; + } + + const line = buffer.slice(0, newlineIndex); + // Deliver bare CR immediately; absorb a following LF even across chunks. + skipLineFeed = buffer[newlineIndex] === "\r"; + buffer = buffer.slice(newlineIndex + 1); + const event = readLine(line); + if (event) yield event; } } - } - const trailing = flushEvent(); - if (trailing) { - yield trailing; + buffer += decoder.end(); + if (buffer) readLine(buffer); + const trailing = flushEvent(); + if (trailing) { + yield trailing; + } + } finally { + close(); } } @@ -179,25 +359,42 @@ export class RuntimeTransport { path: string, init?: RequestInit, params?: RuntimeParams, - allowRefresh: boolean = true + allowRefresh: boolean = true, + streaming: boolean = false ): Promise { const connection = await this.resolveConnection(false); - const response = await this.fetchForConnection(connection, path, init, params); + const response = await this.fetchForConnection(connection, path, init, params, streaming); if (response.status === 401 && allowRefresh) { + this.closeResponse(response); + if (init?.body instanceof Readable) { + throw new HyperbrowserError( + "Runtime authentication expired during a streaming upload; obtain a new stream before retrying", + { + statusCode: 401, + code: "stream_not_replayable", + service: "runtime", + retryable: false, + method: init?.method ?? "GET", + path: errorRequestPath(path), + requestId: getRequestId(response), + } + ); + } const refreshed = await this.resolveConnection(true); - const retryResponse = await this.fetchForConnection(refreshed, path, init, params); - return this.assertResponse(retryResponse); + const retryResponse = await this.fetchForConnection(refreshed, path, init, params, streaming); + return this.assertResponse(retryResponse, path, init); } - return this.assertResponse(response); + return this.assertResponse(response, path, init); } private async fetchForConnection( connection: RuntimeConnection, path: string, init?: RequestInit, - params?: RuntimeParams + params?: RuntimeParams, + streaming = false ): Promise { const target = resolveRuntimeTransportTarget( connection.baseUrl, @@ -206,32 +403,52 @@ export class RuntimeTransport { ); const headers = this.buildHeaders(connection, init?.headers, target.hostHeader); const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), this.timeout); - + let timer: NodeJS.Timeout | undefined = setTimeout(() => controller.abort(), this.timeout); + const upload = init?.body instanceof Readable ? init.body : undefined; + const uploadProgress = () => { + clearTimeout(timer); + timer = setTimeout(() => controller.abort(), this.timeout); + }; + upload?.on("data", uploadProgress); + const callerSignal = init?.signal; + const abortFromCaller = () => controller.abort(); + if (callerSignal?.aborted) controller.abort(); + else callerSignal?.addEventListener("abort", abortFromCaller, { once: true }); + const detach = () => { + clearTimeout(timer); + timer = undefined; + callerSignal?.removeEventListener("abort", abortFromCaller); + upload?.removeListener("data", uploadProgress); + }; try { - return await fetch(target.url, { - ...init, - headers, - signal: controller.signal, - }); + const response = await fetch(target.url, { ...init, headers, signal: controller.signal }); + clearTimeout(timer); + upload?.removeListener("data", uploadProgress); + // Header and body budgets are separate. Streams use a read-idle budget. + timer = + streaming && response.ok ? undefined : setTimeout(() => controller.abort(), this.timeout); + const body = response.body as Readable | null; + let disposed = false; + const close = () => { + if (disposed) return; + disposed = true; + detach(); + if (!body?.readableEnded) controller.abort(); + body?.destroy(); + }; + this.responseCleanup.set(response, close); + if (body) { + body.once("end", detach); + body.once("close", close); + } else close(); + return response; } catch (error) { - if (error instanceof HyperbrowserError) { - throw error; - } - throw new HyperbrowserError( - error instanceof Error ? error.message : "Unknown runtime request error", - { - retryable: isRetryableNetworkError(error), - service: "runtime", - cause: error, - } - ); - } finally { - clearTimeout(timeoutId); + detach(); + throw this.failure(error, path, init); } } - private async assertResponse(response: Response): Promise { + private async assertResponse(response: Response, path: string, init?: RequestInit): Promise { if (response.ok) { return response; } @@ -256,8 +473,11 @@ export class RuntimeTransport { message = rawText; } } - } catch { + } catch (error) { + if (init?.signal?.aborted) throw this.failure(error, path, init, response); // Keep the fallback message. + } finally { + this.closeResponse(response); } throw new HyperbrowserError(message, { @@ -267,6 +487,8 @@ export class RuntimeTransport { retryable: RETRYABLE_STATUS_CODES.has(response.status), service: "runtime", details, + method: init?.method ?? "GET", + path: errorRequestPath(path), }); } diff --git a/src/sandbox/files.ts b/src/sandbox/files.ts index 81858b1..5aada00 100644 --- a/src/sandbox/files.ts +++ b/src/sandbox/files.ts @@ -2,10 +2,20 @@ import { Blob, Buffer } from "buffer"; import nodePath from "path"; import { ReadableStream } from "node:stream/web"; import WebSocket from "ws"; -import { HyperbrowserError } from "../client"; +import { Readable } from "stream"; +import { HyperbrowserError } from "../error"; import { RuntimeTransport } from "./base"; import { AsyncEventQueue, openRuntimeWebSocket, toWebSocketUrl } from "./ws"; import { + SandboxFileUploadStream, + SandboxFileUploadStreamOptions, + SandboxFileDownloadStreamOptions, + SandboxFileMoveParams, + SandboxFileRenameOptions, + SandboxFileWatchParams, + SandboxFileWatchStatus, + SandboxFileWatchEventsParams, + SandboxFileWatchStreamEvent, SandboxFileChmodParams, SandboxFileChownParams, SandboxFileCopyParams, @@ -78,27 +88,7 @@ interface FileMoveCopyWireResponse { } interface FileWatchStatusResponse { - watch: RawFileWatchStatus; -} - -interface RawFileWatchEvent { - seq: number; - path: string; - op: string; - timestamp: number; -} - -interface RawFileWatchStatus { - id: string; - path: string; - recursive: boolean; - active: boolean; - error?: string; - createdAt: number; - stoppedAt?: number; - oldestSeq?: number; - lastSeq?: number; - eventCount?: number; + watch: SandboxFileWatchStatus; } interface RuntimeConnectionInfo { @@ -245,11 +235,12 @@ const encodeWriteData = async ( throw new Error("Unsupported write data type"); }; -class RuntimeFileWatchHandle { +export class SandboxFileWatchHandle { + private readonly connections = new Set(); constructor( private readonly transport: RuntimeTransport, private readonly getConnectionInfo: () => Promise, - private readonly status: RawFileWatchStatus, + private status: SandboxFileWatchStatus, private readonly runtimeProxyOverride?: string ) {} @@ -261,20 +252,55 @@ class RuntimeFileWatchHandle { return this.status.path; } - async stop(): Promise { - await this.transport.requestJSON<{ success: boolean }>( - `/sandbox/files/watch/${this.status.id}`, - { - method: "DELETE", - } + get current(): SandboxFileWatchStatus { + return this.toJSON(); + } + + toJSON(): SandboxFileWatchStatus { + return { + ...this.status, + ...(this.status.events ? { events: this.status.events.map((event) => ({ ...event })) } : {}), + }; + } + + async refresh(includeEvents = false): Promise { + const response = await this.transport.requestJSON( + `/sandbox/files/watch/${encodeURIComponent(this.id)}`, + undefined, + includeEvents ? { includeEvents: true } : undefined ); + this.status = response.watch; + return this; } - async *events(cursor?: number): AsyncGenerator { + async stop(): Promise { + try { + await this.transport.requestJSON(`/sandbox/files/watch/${encodeURIComponent(this.id)}`, { + method: "DELETE", + }); + } catch (error) { + if (!(error instanceof HyperbrowserError) || ![404, 409].includes(error.statusCode ?? 0)) + throw error; + } + this.status = { ...this.status, active: false, stoppedAt: this.status.stoppedAt || Date.now() }; + for (const socket of this.connections) socket.terminate(); + } + + async *events( + options: SandboxFileWatchEventsParams = {} + ): AsyncGenerator { + const { cursor, route = "ws", signal } = options; + if (route !== "ws" && route !== "stream") + throw new HyperbrowserError("Watch route must be ws or stream"); + if (signal?.aborted) + throw new HyperbrowserError("Watch canceled", { + code: "request_aborted", + service: "runtime", + }); const connectionInfo = await this.getConnectionInfo(); const target = toWebSocketUrl( connectionInfo.baseUrl, - `/sandbox/files/watch/${this.status.id}/ws?sessionId=${encodeURIComponent( + `/sandbox/files/watch/${encodeURIComponent(this.status.id)}/${route}?sessionId=${encodeURIComponent( connectionInfo.sandboxId )}${cursor !== undefined ? `&cursor=${encodeURIComponent(String(cursor))}` : ""}`, this.runtimeProxyOverride @@ -287,14 +313,22 @@ class RuntimeFileWatchHandle { headers.Host = target.hostHeader; } - const ws = await openRuntimeWebSocket(target, headers); - const queue = new AsyncEventQueue(); + const ws = await openRuntimeWebSocket(target, headers, { signal }); + this.connections.add(ws); + const queue = new AsyncEventQueue(); + const abort = () => { + queue.fail( + new HyperbrowserError("Watch canceled", { code: "request_aborted", service: "runtime" }) + ); + ws.terminate(); + }; + signal?.addEventListener("abort", abort, { once: true }); + if (signal?.aborted) abort(); ws.on("message", (data) => { try { const parsed = JSON.parse(data.toString()) as - | { type: "event"; event: RawFileWatchEvent } - | { type: "done"; status: RawFileWatchStatus } + | SandboxFileWatchStreamEvent | { error: string; code?: string }; if ("error" in parsed) { @@ -310,10 +344,18 @@ class RuntimeFileWatchHandle { } if (parsed.type === "event") { - queue.push(parsed.event); + this.status = { + ...this.status, + // Receiver sequences start at 1; zero denotes an empty buffer. + oldestSeq: this.status.oldestSeq || parsed.event.seq, + lastSeq: Math.max(this.status.lastSeq ?? 0, parsed.event.seq), + }; + queue.push(parsed); return; } + this.status = parsed.status; + queue.push({ type: "done", status: this.current }); queue.close(); } catch (error) { queue.fail(error); @@ -322,18 +364,17 @@ class RuntimeFileWatchHandle { ws.on("close", () => queue.close()); ws.on("error", (error) => queue.fail(error)); + ws.resume?.(); try { for await (const event of queue) { yield event; } } finally { - if (ws.readyState !== WebSocket.CLOSING && ws.readyState !== WebSocket.CLOSED) { - await new Promise((resolve) => { - ws.once("close", () => resolve()); - ws.close(); - }); - } + signal?.removeEventListener("abort", abort); + this.connections.delete(ws); + // terminate also releases peers which never acknowledge a close frame. + if (ws.readyState !== WebSocket.CLOSED) ws.terminate(); } } } @@ -343,20 +384,23 @@ export class SandboxWatchDirHandle { private timeout?: NodeJS.Timeout; private stopRequested = false; private exitNotified = false; + private invokingCallback = false; constructor( - private readonly watch: RuntimeFileWatchHandle, + private readonly watch: SandboxFileWatchHandle, onEvent: (event: SandboxFileSystemEvent) => void | Promise, private readonly onExit?: (error?: Error) => void | Promise, timeoutMs?: number ) { if (timeoutMs !== undefined && timeoutMs > 0) { this.timeout = setTimeout(() => { - void this.stop(); + void this.stop().catch(() => undefined); }, timeoutMs); this.timeout.unref?.(); } this.runPromise = this.run(onEvent); + // Callback failures must not become unhandled background rejections. + this.runPromise.catch(() => undefined); } async stop(): Promise { @@ -369,7 +413,7 @@ export class SandboxWatchDirHandle { this.timeout = undefined; } await this.watch.stop(); - await this.runPromise.catch(() => undefined); + if (!this.invokingCallback) await this.runPromise.catch(() => undefined); } private async run( @@ -377,15 +421,19 @@ export class SandboxWatchDirHandle { ): Promise { let exitError: Error | undefined; try { - for await (const event of this.watch.events()) { + for await (const message of this.watch.events()) { + if (message.type === "done") break; + const event = message.event; const type = normalizeEventType(event.op); if (!type) { continue; } - await onEvent({ - type, - name: relativeWatchName(this.watch.path, event.path), - }); + this.invokingCallback = true; + try { + await onEvent({ type, name: relativeWatchName(this.watch.path, event.path) }); + } finally { + this.invokingCallback = false; + } } } catch (error) { exitError = error as Error; @@ -396,7 +444,12 @@ export class SandboxWatchDirHandle { } if (!this.exitNotified) { this.exitNotified = true; - await this.onExit?.(exitError); + this.invokingCallback = true; + try { + await this.onExit?.(exitError); + } finally { + this.invokingCallback = false; + } } } } @@ -622,7 +675,11 @@ export class SandboxFilesApi { return Boolean(response.created); } - async rename(oldPath: string, newPath: string): Promise { + async rename( + oldPath: string, + newPath: string, + options: SandboxFileRenameOptions = {} + ): Promise { const response = await this.transport.requestJSON( "/sandbox/files/move", { @@ -630,6 +687,7 @@ export class SandboxFilesApi { body: this.withRunAsBody({ from: oldPath, to: newPath, + overwrite: options.overwrite, }), headers: { "content-type": "application/json", @@ -698,28 +756,98 @@ export class SandboxFilesApi { onEvent: (event: SandboxFileSystemEvent) => void | Promise, options: SandboxWatchDirOptions = {} ): Promise { + const watch = await this.watch(path, options); + return new SandboxWatchDirHandle(watch, onEvent, options.onExit, options.timeoutMs); + } + + async watch(path: string, options: SandboxFileWatchParams = {}): Promise { const response = await this.transport.requestJSON( "/sandbox/files/watch", { method: "POST", - body: this.withRunAsBody({ - path, - recursive: options.recursive, - }), - headers: { - "content-type": "application/json", - }, + body: this.withRunAsBody({ path, recursive: options.recursive }), + headers: { "content-type": "application/json" }, } ); + return new SandboxFileWatchHandle( + this.transport, + this.getConnectionInfo, + response.watch, + this.runtimeProxyOverride + ); + } - const watch = new RuntimeFileWatchHandle( + async getWatch(id: string, includeEvents = false): Promise { + const response = await this.transport.requestJSON( + `/sandbox/files/watch/${encodeURIComponent(id)}`, + undefined, + includeEvents ? { includeEvents: true } : undefined + ); + return new SandboxFileWatchHandle( this.transport, this.getConnectionInfo, response.watch, this.runtimeProxyOverride ); + } - return new SandboxWatchDirHandle(watch, onEvent, options.onExit, options.timeoutMs); + stat(path: string): Promise { + return this.getInfo(path); + } + mkdir(path: string, options: SandboxFileMakeDirOptions = {}): Promise { + return this.makeDir(path, options); + } + move(params: SandboxFileMoveParams): Promise { + return this.rename(params.source, params.destination, params); + } + delete(path: string, options: { recursive?: boolean } = {}): Promise { + return this.remove(path, options); + } + + /** Upload without buffering; the source is closed on completion, rejection or cancellation. */ + async uploadStream( + path: string, + source: SandboxFileUploadStream, + options: SandboxFileUploadStreamOptions = {} + ): Promise { + if ( + options.contentLength !== undefined && + (!Number.isSafeInteger(options.contentLength) || options.contentLength < 0) + ) { + throw new HyperbrowserError("contentLength must be a nonnegative safe integer"); + } + const body = Readable.from(source, { objectMode: false, highWaterMark: 64 * 1024 }); + try { + return await this.transport.requestJSON( + "/sandbox/files/upload", + { + method: "PUT", + body, + signal: options.signal, + headers: { + "content-type": "application/octet-stream", + ...(options.contentLength !== undefined + ? { "content-length": String(options.contentLength) } + : {}), + }, + }, + this.withRunAsQuery({ path }) + ); + } finally { + body.destroy(); + } + } + + /** Pull-based download; consume with for-await or Readable.from(). */ + downloadStream( + path: string, + options: SandboxFileDownloadStreamOptions = {} + ): AsyncGenerator { + return this.transport.streamBytes( + "/sandbox/files/download", + { signal: options.signal }, + this.withRunAsQuery({ path }) + ); } async uploadUrl( diff --git a/src/sandbox/image-build/artifacts.ts b/src/sandbox/image-build/artifacts.ts new file mode 100644 index 0000000..0d3aa7a --- /dev/null +++ b/src/sandbox/image-build/artifacts.ts @@ -0,0 +1,27 @@ +import { rmSync } from "fs"; +import { SandboxImageBuildInputFormat, SandboxImageInit } from "../../types/sandbox"; + +export const IMAGE_BUILD_INPUT_FORMAT: SandboxImageBuildInputFormat = "rootfs_export_tar_gz"; +export const CONTEXT_MANIFEST_INPUT_FORMAT: SandboxImageBuildInputFormat = + "dockerfile_context_manifest_v1"; +export const DOCKER_IMAGE_MANIFEST_INPUT_FORMAT: SandboxImageBuildInputFormat = + "docker_image_manifest_v1"; +export const IMAGE_BUILD_SOURCE_PLATFORM = "linux/amd64"; + +export interface DockerImageBuildArtifact { + path: string; + sha256Hex: string; + sizeBytes: number; + inputFormat: SandboxImageBuildInputFormat; + sourcePlatform: string; + imageConfigUser: string; + imageInit?: SandboxImageInit; +} + +export const removeArtifact = (artifact: DockerImageBuildArtifact): void => { + rmSync(artifact.path, { force: true }); +}; + +export const removeWorkspace = (workspace: string): void => { + rmSync(workspace, { recursive: true, force: true }); +}; diff --git a/src/sandbox/image-build/blake2b.ts b/src/sandbox/image-build/blake2b.ts new file mode 100644 index 0000000..8124ad8 --- /dev/null +++ b/src/sandbox/image-build/blake2b.ts @@ -0,0 +1,97 @@ +/** Minimal BLAKE2b (RFC 7693) supporting arbitrary digest lengths. */ + +const IV: bigint[] = [ + 0x6a09e667f3bcc908n, + 0xbb67ae8584caa73bn, + 0x3c6ef372fe94f82bn, + 0xa54ff53a5f1d36f1n, + 0x510e527fade682d1n, + 0x9b05688c2b3e6c1fn, + 0x1f83d9abfb41bd6bn, + 0x5be0cd19137e2179n, +]; + +const SIGMA: number[][] = [ + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], + [11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4], + [7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8], + [9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13], + [2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9], + [12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11], + [13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10], + [6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5], + [10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0], + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], +]; + +const MASK = (1n << 64n) - 1n; + +const rotr = (value: bigint, bits: bigint): bigint => + ((value >> bits) | (value << (64n - bits))) & MASK; + +const compress = (h: bigint[], block: Buffer, counter: bigint, last: boolean): void => { + const m: bigint[] = []; + for (let index = 0; index < 16; index += 1) { + m.push(block.readBigUInt64LE(index * 8)); + } + const v = [...h, ...IV]; + v[12] ^= counter & MASK; + v[13] ^= (counter >> 64n) & MASK; + if (last) { + v[14] ^= MASK; + } + + const mix = (a: number, b: number, c: number, d: number, x: bigint, y: bigint): void => { + v[a] = (v[a] + v[b] + x) & MASK; + v[d] = rotr(v[d] ^ v[a], 32n); + v[c] = (v[c] + v[d]) & MASK; + v[b] = rotr(v[b] ^ v[c], 24n); + v[a] = (v[a] + v[b] + y) & MASK; + v[d] = rotr(v[d] ^ v[a], 16n); + v[c] = (v[c] + v[d]) & MASK; + v[b] = rotr(v[b] ^ v[c], 63n); + }; + + for (let round = 0; round < 12; round += 1) { + const s = SIGMA[round]; + mix(0, 4, 8, 12, m[s[0]], m[s[1]]); + mix(1, 5, 9, 13, m[s[2]], m[s[3]]); + mix(2, 6, 10, 14, m[s[4]], m[s[5]]); + mix(3, 7, 11, 15, m[s[6]], m[s[7]]); + mix(0, 5, 10, 15, m[s[8]], m[s[9]]); + mix(1, 6, 11, 12, m[s[10]], m[s[11]]); + mix(2, 7, 8, 13, m[s[12]], m[s[13]]); + mix(3, 4, 9, 14, m[s[14]], m[s[15]]); + } + + for (let index = 0; index < 8; index += 1) { + h[index] = h[index] ^ v[index] ^ v[index + 8]; + } +}; + +export const blake2b = (data: Buffer, digestSize: number): Buffer => { + if (!Number.isInteger(digestSize) || digestSize < 1 || digestSize > 64) { + throw new RangeError("digestSize must be between 1 and 64"); + } + const h = [...IV]; + h[0] ^= BigInt(0x01010000 ^ digestSize); + + let offset = 0; + let counter = 0n; + while (data.length - offset > 128) { + counter += 128n; + compress(h, data.subarray(offset, offset + 128), counter, false); + offset += 128; + } + const remaining = data.subarray(offset); + const block = Buffer.alloc(128); + remaining.copy(block); + counter += BigInt(remaining.length); + compress(h, block, counter, true); + + const out = Buffer.alloc(64); + h.forEach((word, index) => out.writeBigUInt64LE(word, index * 8)); + return out.subarray(0, digestSize); +}; diff --git a/src/sandbox/image-build/common.ts b/src/sandbox/image-build/common.ts new file mode 100644 index 0000000..d968f41 --- /dev/null +++ b/src/sandbox/image-build/common.ts @@ -0,0 +1,112 @@ +/** Python `posixpath.normpath` semantics (keeps exactly two leading slashes). */ +export const posixNormpath = (value: string): string => { + if (value === "") { + return "."; + } + let initialSlashes = value.startsWith("/") ? 1 : 0; + if (initialSlashes && value.startsWith("//") && !value.startsWith("///")) { + initialSlashes = 2; + } + const components: string[] = []; + for (const component of value.split("/")) { + if (component === "" || component === ".") { + continue; + } + if ( + component !== ".." || + (!initialSlashes && components.length === 0) || + (components.length > 0 && components[components.length - 1] === "..") + ) { + components.push(component); + } else if (components.length > 0) { + components.pop(); + } + } + const joined = "/".repeat(initialSlashes) + components.join("/"); + return joined || "."; +}; + +export const posixDirname = (value: string): string => { + const index = value.lastIndexOf("/") + 1; + let head = value.slice(0, index); + if (head && head !== "/".repeat(head.length)) { + head = head.replace(/\/+$/, ""); + } + return head; +}; + +export const posixJoin = (...parts: string[]): string => { + let joined = ""; + for (const part of parts) { + if (part.startsWith("/")) { + joined = part; + } else if (!joined || joined.endsWith("/")) { + joined += part; + } else { + joined += "/" + part; + } + } + return joined; +}; + +/** Compare strings by Unicode code point, like Python's default `str` ordering. */ +export const compareCodePoints = (a: string, b: string): number => { + const left = Array.from(a); + const right = Array.from(b); + const length = Math.min(left.length, right.length); + for (let index = 0; index < length; index += 1) { + const difference = (left[index].codePointAt(0) ?? 0) - (right[index].codePointAt(0) ?? 0); + if (difference !== 0) { + return difference; + } + } + return left.length - right.length; +}; + +const escapeNonAscii = (json: string): string => + json.replace(/[\u007f-\uffff]/g, (character) => { + return "\\u" + character.charCodeAt(0).toString(16).padStart(4, "0"); + }); + +// Serialize keys directly: assigning sorted keys to an object lets JavaScript +// reorder integer-like keys and treats __proto__ as a setter. +const sortedJson = (value: unknown): string => { + if (Array.isArray(value)) return `[${value.map((item) => sortedJson(item) ?? "null").join(",")}]`; + if (value && typeof value === "object") { + const record = value as Record; + const entries = Object.keys(record).sort(compareCodePoints).flatMap((key) => { + const item = sortedJson(record[key]); + return item === undefined ? [] : [`${JSON.stringify(key)}:${item}`]; + }); + return `{${entries.join(",")}}`; + } + return JSON.stringify(value); +}; + +/** + * Compact JSON matching Python `json.dumps(value, separators=(",", ":"))`. + * `sortKeys` mirrors `sort_keys=True`; `ensureAscii` mirrors the default + * `ensure_ascii=True` escaping of non-ASCII characters. + */ +export const compactJson = ( + value: unknown, + options: { sortKeys?: boolean; ensureAscii?: boolean } = {} +): string => { + const json = options.sortKeys ? sortedJson(value) : JSON.stringify(value); + return options.ensureAscii === false ? json : escapeNonAscii(json); +}; + +export const SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/; + +export const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +export const requireRecord = (value: unknown, label: string): Record => { + if (!isRecord(value)) { + throw new Error(`${label} must be a JSON object`); + } + return value; +}; + +export const sleep = (seconds: number): Promise => + new Promise((resolve) => setTimeout(resolve, Math.max(0, seconds) * 1000)); diff --git a/src/sandbox/image-build/context.ts b/src/sandbox/image-build/context.ts new file mode 100644 index 0000000..fb131ff --- /dev/null +++ b/src/sandbox/image-build/context.ts @@ -0,0 +1,747 @@ +/** Remote Dockerfile build context selection, fingerprinting, and packaging. */ + +import { createHash, Hash } from "crypto"; +import { + createReadStream, + existsSync, + lstatSync, + mkdtempSync, + readdirSync, + readFileSync, + readlinkSync, + realpathSync, + statSync, + writeFileSync, + Stats, +} from "fs"; +import { homedir, tmpdir } from "os"; +import * as path from "path"; +import { + SandboxBuildContextBundle, + SandboxBuildContextManifest, + SandboxBuildContextMode, +} from "../../types/sandbox"; +import { + CONTEXT_MANIFEST_INPUT_FORMAT, + DockerImageBuildArtifact, + IMAGE_BUILD_SOURCE_PLATFORM, + removeArtifact, + removeWorkspace, +} from "./artifacts"; +import { + compactJson, + compareCodePoints, + posixDirname, + posixJoin, + posixNormpath, + SHA256_HEX_PATTERN, +} from "./common"; +import { analyzeDockerfileSources } from "./dockerfile-analysis"; +import { DockerIgnoreMatcher } from "./dockerignore"; +import { writeGzipTar } from "./gzip"; +import { TarEntryInfo } from "./tar"; + +const MAX_CONTEXT_SOURCE_GROUPS = 511; +const MAX_CONTEXT_ENTRIES = 1_000_000; + +/** The packaged context no longer matches the caller's cache fingerprint. */ +export class DockerBuildContextChangedError extends Error { + constructor(message: string) { + super(message); + this.name = "DockerBuildContextChangedError"; + } +} + +export interface PackagedDockerBuildContext { + artifact: DockerImageBuildArtifact; + manifest: SandboxBuildContextManifest; + bundles: Record; + workspace: string; + fingerprint: string; + cleanup(): void; +} + +export interface DockerBuildContextOptions { + dockerfile?: string; + forceFullContext?: boolean; +} + +interface DockerBuildContextSelection { + root: string; + dockerfile: string; + mode: SandboxBuildContextMode; + fallbackReason: string | null; + entryGroups: Array>; +} + +const selectionFingerprint = ( + selection: DockerBuildContextSelection, + bundleHashes: string[] +): string => { + // Hash entry metadata and contents, not transport encoding: cache identity + // must not depend on tar headers or a compression library. + const identity = { + version: 1, + dockerfile: selection.dockerfile, + contextMode: selection.mode, + bundles: Array.from(new Set(bundleHashes)).sort(compareCodePoints), + }; + return createHash("sha256") + .update(compactJson(identity, { sortKeys: true })) + .digest("hex"); +}; + +const expandUser = (value: string): string => { + if (value === "~" || value.startsWith("~/")) { + return path.join(homedir(), value.slice(1)); + } + return value; +}; + +const lstatOrNull = (target: string): Stats | null => { + try { + return lstatSync(target); + } catch { + return null; + } +}; + +const statOrNull = (target: string): Stats | null => { + try { + return statSync(target); + } catch { + return null; + } +}; + +const pathExistsOrSymlink = (target: string): boolean => lstatOrNull(target) !== null; + +const pathIsWithin = (parent: string, candidate: string): boolean => { + const relative = path.relative(parent, candidate); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +}; + +const cleanContextRelativePath = (value: string | undefined, fallback: string): string => { + let normalized = (value ?? "").trim() || fallback; + normalized = normalized.split(path.sep).join("/"); + if ( + normalized.includes("\\") || + normalized.includes("\x00") || + normalized.includes("\r") || + normalized.includes("\n") || + normalized.startsWith("/") + ) { + throw new Error("Dockerfile path must be relative to the build context"); + } + if (normalized.split("/").includes("..")) { + throw new Error("Dockerfile path must be a relative path inside the build context"); + } + const cleaned = posixNormpath(normalized); + if (cleaned === "." || cleaned === ".." || cleaned.startsWith("../")) { + throw new Error("Dockerfile path must be a relative path inside the build context"); + } + return cleaned; +}; + +const selectDockerignore = (contextRoot: string, dockerfileRelative: string): string => { + const dockerfileIgnore = `${dockerfileRelative}.dockerignore`; + const candidate = path.join(contextRoot, dockerfileIgnore); + if (existsSync(candidate)) { + if (!statSync(candidate).isFile()) { + throw new Error( + `Dockerfile-specific ignore file "${dockerfileIgnore}" is not a regular file` + ); + } + return dockerfileIgnore; + } + return ".dockerignore"; +}; + +export const loadDockerignore = (ignorePath: string): DockerIgnoreMatcher | null => { + if (!existsSync(ignorePath)) { + return null; + } + try { + return DockerIgnoreMatcher.fromFile(ignorePath); + } catch (error) { + throw new Error(`parse .dockerignore: ${error instanceof Error ? error.message : error}`); + } +}; + +const normalizeContextSource = (source: string): string => { + if (source.includes("\x00")) { + throw new Error("Dockerfile source path contains a NUL byte"); + } + const normalized = String(source).trim().replace(/\\/g, "/"); + if (normalized === "" || normalized === "." || normalized === "/") { + return "."; + } + return posixNormpath("/" + normalized).replace(/^\/+/, "") || "."; +}; + +const deduplicateSourceGroups = (groups: string[][]): string[][] => { + const result: string[][] = []; + const seen = new Set(); + for (const group of groups) { + const normalized = group.map(normalizeContextSource).sort(compareCodePoints); + const key = JSON.stringify(normalized); + if (!seen.has(key)) { + seen.add(key); + result.push(normalized); + } + } + return result; +}; + +const isIgnored = (relative: string, matcher: DockerIgnoreMatcher | null): boolean => + matcher !== null && matcher.matches(relative); + +const addContextEntryWithParents = (entries: Set, relative: string): void => { + let current = relative; + while (current !== "" && current !== ".") { + entries.add(current); + current = posixDirname(current); + } +}; + +const toRelative = (contextRoot: string, target: string): string => { + const relative = path.relative(contextRoot, target).split(path.sep).join("/"); + return relative === "" ? "." : relative; +}; + +/** + * Return a sparse source plus symlinks and their in-context targets. + * + * This mirrors fsutil `FollowPaths`: symlink targets are interpreted inside + * the context root, including absolute or `..` targets, and cycles terminate + * after retaining every symlink encountered. + */ +const followContextSourceSymlinks = (contextRoot: string, relativeSource: string): string[] => { + const pending = [normalizeContextSource(relativeSource)]; + const resolved: string[] = []; + const seen = new Set(); + while (pending.length > 0) { + const relative = pending.pop() as string; + if (seen.has(relative)) { + continue; + } + seen.add(relative); + + const parts = relative === "." ? [] : relative.split("/"); + const currentParts: string[] = []; + let followed = false; + for (let index = 0; index < parts.length; index += 1) { + currentParts.push(parts[index]); + const currentRelative = currentParts.join("/"); + const currentPath = path.join(contextRoot, currentRelative); + const metadata = lstatOrNull(currentPath); + if (metadata === null || !metadata.isSymbolicLink()) { + continue; + } + + resolved.push(currentRelative); + let target: string; + try { + target = readlinkSync(currentPath); + } catch (error) { + throw new Error( + `read build context symlink "${currentRelative}": ${error instanceof Error ? error.message : error}` + ); + } + if (!target || target.includes("\x00")) { + throw new Error(`build context symlink "${currentRelative}" has an invalid target`); + } + + const remainder = parts.slice(index + 1); + const combined = target.startsWith("/") + ? posixJoin(target, ...remainder) + : posixJoin(posixDirname(currentRelative), target, ...remainder); + const normalizedTarget = posixNormpath("/" + combined).replace(/^\/+/, ""); + pending.push(normalizedTarget || "."); + followed = true; + break; + } + + if (!followed) { + resolved.push(relative); + } + } + return resolved; +}; + +const walkDirectory = ( + contextRoot: string, + directory: string, + entries: Set, + ignoreMatcher: DockerIgnoreMatcher | null, + canPruneIgnoredDirectories: boolean +): void => { + const children = readdirSync(directory, { withFileTypes: true }); + const directories = children + .filter((child) => child.isDirectory()) + .map((child) => child.name) + .sort(compareCodePoints); + const files = children + .filter((child) => !child.isDirectory()) + .map((child) => child.name) + .sort(compareCodePoints); + + const retainedDirectories: string[] = []; + for (const name of directories) { + const childRelative = toRelative(contextRoot, path.join(directory, name)); + if (isIgnored(childRelative, ignoreMatcher)) { + if (!canPruneIgnoredDirectories) { + retainedDirectories.push(name); + } + continue; + } + addContextEntryWithParents(entries, childRelative); + retainedDirectories.push(name); + } + for (const name of files) { + const childRelative = toRelative(contextRoot, path.join(directory, name)); + if (isIgnored(childRelative, ignoreMatcher)) { + continue; + } + addContextEntryWithParents(entries, childRelative); + } + for (const name of retainedDirectories) { + walkDirectory( + contextRoot, + path.join(directory, name), + entries, + ignoreMatcher, + canPruneIgnoredDirectories + ); + } +}; + +const collectContextPath = ( + contextRoot: string, + target: string, + entries: Set, + ignoreMatcher: DockerIgnoreMatcher | null +): void => { + const relative = toRelative(contextRoot, target); + const pathIsIgnored = relative !== "." && isIgnored(relative, ignoreMatcher); + const canPruneIgnoredDirectories = !(ignoreMatcher?.hasNegations ?? false); + if (relative !== "." && !pathIsIgnored) { + addContextEntryWithParents(entries, relative); + } + const metadata = lstatOrNull(target); + if (metadata === null || metadata.isSymbolicLink() || !metadata.isDirectory()) { + return; + } + if (pathIsIgnored && canPruneIgnoredDirectories) { + return; + } + walkDirectory(contextRoot, target, entries, ignoreMatcher, canPruneIgnoredDirectories); +}; + +export const collectContextEntries = ( + contextRoot: string, + sources: string[], + options: { ignoreMatcher: DockerIgnoreMatcher | null; required: boolean } +): Set => { + const entries = new Set(); + for (const rawSource of sources) { + const source = normalizeContextSource(rawSource); + const candidate = source === "." ? contextRoot : path.join(contextRoot, source); + const existingMatches = pathExistsOrSymlink(candidate) ? [candidate] : []; + if (options.required && existingMatches.length === 0) { + throw new Error(`Docker build context source not found: "${source}"`); + } + for (const match of existingMatches) { + const relativeMatch = toRelative(contextRoot, match); + for (const followedRelative of followContextSourceSymlinks(contextRoot, relativeMatch)) { + const followedPath = + followedRelative === "." ? contextRoot : path.join(contextRoot, followedRelative); + if (pathExistsOrSymlink(followedPath)) { + collectContextPath(contextRoot, followedPath, entries, options.ignoreMatcher); + } + } + } + } + return entries; +}; + +const isSubset = (left: Set, right: Set): boolean => { + for (const item of left) { + if (!right.has(item)) { + return false; + } + } + return true; +}; + +const removeSubsumedEntryGroups = (groups: Array>): Array> => { + const result: Array> = []; + groups.forEach((entries, index) => { + const subsumed = groups.some((candidate, candidateIndex) => { + if (index === candidateIndex || !isSubset(entries, candidate)) { + return false; + } + const equal = entries.size === candidate.size; + return !equal || index > candidateIndex; + }); + if (!subsumed) { + result.push(entries); + } + }); + return result; +}; + +const selectDockerBuildContext = ( + contextPath: string, + options: DockerBuildContextOptions +): DockerBuildContextSelection => { + const dockerfile = options.dockerfile ?? "Dockerfile"; + const contextRoot = realpathSync(path.resolve(expandUser(contextPath))); + if (!statSync(contextRoot).isDirectory()) { + throw new Error("Docker build context must be a directory"); + } + const dockerfileRelative = cleanContextRelativePath(dockerfile, "Dockerfile"); + const dockerfilePath = path.join(contextRoot, dockerfileRelative); + const dockerfileMetadata = lstatOrNull(dockerfilePath); + const dockerfileStat = statOrNull(dockerfilePath); + if ( + dockerfileMetadata === null || + !((dockerfileStat?.isFile() ?? false) || dockerfileMetadata.isSymbolicLink()) + ) { + throw new Error(`Dockerfile "${dockerfileRelative}" must be a regular file or symlink`); + } + let resolvedDockerfile: string; + try { + resolvedDockerfile = realpathSync(dockerfilePath); + } catch { + throw new Error( + `Dockerfile "${dockerfileRelative}" must resolve to a regular file inside the build context` + ); + } + if (!statSync(resolvedDockerfile).isFile() || !pathIsWithin(contextRoot, resolvedDockerfile)) { + throw new Error( + `Dockerfile "${dockerfileRelative}" must resolve to a regular file inside the build context` + ); + } + + const dockerfileBytes = readFileSync(dockerfilePath); + const ignoreRelative = selectDockerignore(contextRoot, dockerfileRelative); + const ignoreMatcher = loadDockerignore(path.join(contextRoot, ignoreRelative)); + let { groups: sourceGroups, fallbackReason } = analyzeDockerfileSources(dockerfileBytes); + if (options.forceFullContext) { + sourceGroups = []; + fallbackReason = "requested_full_context"; + } + sourceGroups = deduplicateSourceGroups(sourceGroups); + if (sourceGroups.length > MAX_CONTEXT_SOURCE_GROUPS) { + sourceGroups = []; + fallbackReason = "too_many_context_source_groups"; + } + const usesWholeContext = sourceGroups.some((group) => group.includes(".")); + + let contextMode: SandboxBuildContextMode = fallbackReason || usesWholeContext ? "full" : "sparse"; + const controlSources = [ + dockerfileRelative, + ignoreRelative, + toRelative(contextRoot, resolvedDockerfile), + ]; + + const collectFullContext = (): Array> => [ + new Set([ + ...collectContextEntries(contextRoot, ["."], { ignoreMatcher, required: false }), + ...collectContextEntries(contextRoot, controlSources, { + ignoreMatcher: null, + required: false, + }), + ]), + ]; + + let entryGroups: Array>; + if (contextMode === "full") { + entryGroups = collectFullContext(); + } else { + entryGroups = [ + collectContextEntries(contextRoot, controlSources, { ignoreMatcher: null, required: false }), + ]; + for (const group of sourceGroups) { + entryGroups.push( + collectContextEntries(contextRoot, group, { ignoreMatcher, required: true }) + ); + } + const total = entryGroups.reduce((sum, entries) => sum + entries.size, 0); + if (total > MAX_CONTEXT_ENTRIES) { + contextMode = "full"; + fallbackReason = "context_selection_too_large"; + entryGroups = collectFullContext(); + } + } + + return { + root: contextRoot, + dockerfile: dockerfileRelative, + mode: contextMode, + fallbackReason: fallbackReason || null, + entryGroups: removeSubsumedEntryGroups(entryGroups), + }; +}; + +const validateArchiveRelativePath = (relative: string): void => { + if ( + !relative || + relative.startsWith("/") || + relative.includes("\x00") || + posixNormpath(relative) !== relative || + relative === ".." || + relative.startsWith("../") + ) { + throw new Error(`invalid build context path "${relative}"`); + } +}; + +const contextEntryInfo = (contextRoot: string, relative: string): TarEntryInfo | null => { + validateArchiveRelativePath(relative); + const absolute = path.join(contextRoot, relative); + // Every regular path must be present with its own contents (no hard-link + // entries); preserve symlinks without following them. + const metadata = lstatSync(absolute); + const mode = metadata.mode & 0o7777; + if (metadata.isFile()) { + return { name: relative, type: "file", mode, size: metadata.size, linkname: "" }; + } + if (metadata.isDirectory()) { + return { name: `${relative}/`, type: "directory", mode, size: 0, linkname: "" }; + } + if (metadata.isSymbolicLink()) { + const linkname = readlinkSync(absolute); + if (!linkname) { + throw new Error(`build context symlink "${relative}" has an invalid target`); + } + return { name: relative, type: "symlink", mode, size: 0, linkname }; + } + return null; +}; + +const hashContextEntryMetadata = (hasher: Hash, info: TarEntryInfo): void => { + const metadata = Buffer.from( + compactJson([ + info.type === "directory" ? info.name.replace(/\/+$/, "") : info.name, + info.type, + info.mode, + info.size, + info.linkname, + ]), + "utf8" + ); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(metadata.length)); + hasher.update(length); + hasher.update(metadata); +}; + +async function* hashingFileChunks( + filePath: string, + hasher: Hash, + size: number +): AsyncGenerator { + let remaining = size; + if (remaining === 0) { + return; + } + const stream = createReadStream(filePath, { highWaterMark: 64 * 1024 }); + try { + for await (const chunk of stream) { + const buffer = chunk as Buffer; + if (buffer.length > remaining) { + throw new Error(`build context file "${filePath}" changed size while reading`); + } + remaining -= buffer.length; + hasher.update(buffer); + yield buffer; + } + } finally { + stream.destroy(); + } + if (remaining !== 0) { + throw new Error(`build context file "${filePath}" was truncated`); + } +} + +/** + * Fingerprint the effective remote context without compressing or staging it. + * + * Uses the same Dockerfile source selection, ignore rules, and normalized tar + * entries as remote packaging. Pass the result as `expectedContextFingerprint` + * when building to detect context changes. Build options outside the context + * (e.g. platform or imageInit) must also be included in the caller's cache + * key. Mutable base tags and network resources are not resolved. + */ +export const dockerBuildContextFingerprint = async ( + contextPath: string, + options: DockerBuildContextOptions = {} +): Promise => { + const selection = selectDockerBuildContext(contextPath, options); + const hashes: string[] = []; + for (const entries of selection.entryGroups) { + const hasher = createHash("sha256"); + for (const relative of Array.from(entries).sort(compareCodePoints)) { + const info = contextEntryInfo(selection.root, relative); + if (info === null) { + continue; + } + hashContextEntryMetadata(hasher, info); + if (info.type === "file") { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _chunk of hashingFileChunks( + path.join(selection.root, relative), + hasher, + info.size + )) { + // hashing only + } + } + } + hashes.push(hasher.digest("hex")); + } + return selectionFingerprint(selection, hashes); +}; + +const packageContextBundle = async ( + contextRoot: string, + entries: string[], + workspace: string, + index: number +): Promise<[DockerImageBuildArtifact, SandboxBuildContextBundle, string]> => { + const bundlePath = path.join(workspace, `bundle-${String(index).padStart(4, "0")}.tar.gz`); + const contextHasher = createHash("sha256"); + let entryCount = 0; + let uncompressedSize = 0; + const result = await writeGzipTar(bundlePath, async (writer) => { + for (const relative of entries) { + const info = contextEntryInfo(contextRoot, relative); + if (info === null) { + continue; + } + hashContextEntryMetadata(contextHasher, info); + if (info.type === "file") { + await writer.addEntry( + info, + hashingFileChunks(path.join(contextRoot, relative), contextHasher, info.size) + ); + uncompressedSize += info.size; + } else { + await writer.addEntry(info); + } + entryCount += 1; + } + }); + const artifact: DockerImageBuildArtifact = { + path: bundlePath, + sha256Hex: result.sha256Hex, + sizeBytes: result.sizeBytes, + inputFormat: CONTEXT_MANIFEST_INPUT_FORMAT, + sourcePlatform: IMAGE_BUILD_SOURCE_PLATFORM, + imageConfigUser: "", + }; + const descriptor: SandboxBuildContextBundle = { + sha256: result.sha256Hex, + sizeBytes: result.sizeBytes, + uncompressedSizeBytes: uncompressedSize, + entryCount, + }; + return [artifact, descriptor, contextHasher.digest("hex")]; +}; + +/** Serialize a manifest exactly as the Python SDK does (field order, no nulls). */ +export const canonicalManifestJson = (value: unknown): Buffer => + Buffer.from(compactJson(value, { ensureAscii: false }), "utf8"); + +export const writeManifestArtifact = ( + workspace: string, + filename: string, + data: Buffer, + inputFormat: DockerImageBuildArtifact["inputFormat"], + extras: Pick = { + imageConfigUser: "", + } +): DockerImageBuildArtifact => { + const filePath = path.join(workspace, filename); + writeFileSync(filePath, data, { flag: "wx" }); + return { + path: filePath, + sha256Hex: createHash("sha256").update(data).digest("hex"), + sizeBytes: data.length, + inputFormat, + sourcePlatform: IMAGE_BUILD_SOURCE_PLATFORM, + imageConfigUser: extras.imageConfigUser, + imageInit: extras.imageInit, + }; +}; + +export interface PackageDockerBuildContextOptions extends DockerBuildContextOptions { + tempDir?: string; + expectedContextFingerprint?: string; +} + +export const packageDockerBuildContextManifest = async ( + contextPath: string, + options: PackageDockerBuildContextOptions = {} +): Promise => { + const expected = options.expectedContextFingerprint; + if (expected !== undefined && !SHA256_HEX_PATTERN.test(expected)) { + throw new Error("expectedContextFingerprint must be a SHA-256 hex digest"); + } + const selection = selectDockerBuildContext(contextPath, options); + const workspace = mkdtempSync(path.join(options.tempDir ?? tmpdir(), "hb-docker-context-")); + try { + const bundles: Record = {}; + const descriptors: SandboxBuildContextBundle[] = []; + const bundleHashes: string[] = []; + for (const [index, entries] of selection.entryGroups.entries()) { + const [artifact, descriptor, bundleHash] = await packageContextBundle( + selection.root, + Array.from(entries).sort(compareCodePoints), + workspace, + index + ); + bundleHashes.push(bundleHash); + if (descriptor.sha256 in bundles) { + removeArtifact(artifact); + continue; + } + bundles[descriptor.sha256] = artifact; + descriptors.push(descriptor); + } + descriptors.sort((left, right) => compareCodePoints(left.sha256, right.sha256)); + const fingerprint = selectionFingerprint(selection, bundleHashes); + if (expected !== undefined && fingerprint !== expected) { + throw new DockerBuildContextChangedError( + "Docker build context changed after its cache fingerprint was computed. " + + "Retry the build with a fresh fingerprint." + ); + } + const manifest: SandboxBuildContextManifest = { + version: 1, + dockerfilePath: selection.dockerfile, + contextMode: selection.mode, + ...(selection.fallbackReason ? { fallbackReason: selection.fallbackReason } : {}), + bundles: descriptors, + }; + const artifact = writeManifestArtifact( + workspace, + "context-manifest.json", + canonicalManifestJson(manifest), + CONTEXT_MANIFEST_INPUT_FORMAT + ); + return { + artifact, + manifest, + bundles, + workspace, + fingerprint, + cleanup: () => removeWorkspace(workspace), + }; + } catch (error) { + removeWorkspace(workspace); + throw error; + } +}; diff --git a/src/sandbox/image-build/docker-image.ts b/src/sandbox/image-build/docker-image.ts new file mode 100644 index 0000000..05f8a69 --- /dev/null +++ b/src/sandbox/image-build/docker-image.ts @@ -0,0 +1,580 @@ +/** Local Docker image inspection and layer-manifest packaging for prebuilt imports. */ + +import { Readable } from "stream"; +import { pipeline } from "stream/promises"; +import { execFile, spawn, ChildProcess } from "child_process"; +import { createHash } from "crypto"; +import { createWriteStream, mkdtempSync, readFileSync } from "fs"; +import { once } from "events"; +import { tmpdir } from "os"; +import * as path from "path"; +import { promisify } from "util"; +import { + SandboxDockerImageConfig, + SandboxDockerImageLayer, + SandboxDockerImageManifest, + SandboxImageInit, +} from "../../types/sandbox"; +import { + DOCKER_IMAGE_MANIFEST_INPUT_FORMAT, + DockerImageBuildArtifact, + IMAGE_BUILD_SOURCE_PLATFORM, + removeWorkspace, +} from "./artifacts"; +import { isRecord, posixNormpath, requireRecord, SHA256_HEX_PATTERN } from "./common"; +import { canonicalManifestJson, writeManifestArtifact } from "./context"; +import { deriveAutoImageInit } from "./image-init"; +import { readTarEntries } from "./tar"; + +const execFileAsync = promisify(execFile); + +const MAX_DOCKER_SAVE_ENTRIES = 4096; +const MAX_DOCKER_SAVE_ARCHIVE_BYTES = 5 * 1024 * 1024 * 1024; +const MAX_DOCKER_SAVE_METADATA_BYTES = 16 * 1024 * 1024; +const MAX_DOCKER_IMAGE_LAYERS = 512; + +export interface DockerImageManifestSource { + imageDigest: string; + config: Record; + imageConfigUser: string; + imageInit?: SandboxImageInit; + cleanup(): Promise; +} + +export interface PackagedDockerImage { + artifact: DockerImageBuildArtifact; + manifest: SandboxDockerImageManifest; + layers: Record; + workspace: string; + cleanup(): void; +} + +interface StoredDockerSaveEntry { + path: string; + sha256Hex: string; + sizeBytes: number; +} + +export class DockerCommandError extends Error { + constructor(message: string) { + super(message); + this.name = "DockerCommandError"; + } +} + +const describeCommandFailure = ( + args: string[], + error: NodeJS.ErrnoException & { stderr?: string | Buffer; code?: string | number } +): DockerCommandError => { + if (error.code === "ENOENT") { + return new DockerCommandError("docker CLI is required for local Docker image operations"); + } + const stderr = String(error.stderr ?? "").trim(); + const command = ["docker", ...args].join(" "); + if (stderr) { + return new DockerCommandError(`${command}: ${stderr}`); + } + return new DockerCommandError(`${command} failed with code ${String(error.code ?? "unknown")}`); +}; + +export const runDockerCommand = async (args: string[]): Promise => { + try { + const { stdout } = await execFileAsync("docker", args, { + maxBuffer: MAX_DOCKER_SAVE_METADATA_BYTES, + encoding: "utf8", + }); + return stdout; + } catch (error) { + throw describeCommandFailure(args, error as NodeJS.ErrnoException); + } +}; + +const normalizeSha256Digest = (value: unknown): string => { + const digest = String(value ?? "") + .trim() + .toLowerCase(); + if (!digest.startsWith("sha256:") || !SHA256_HEX_PATTERN.test(digest.slice(7))) { + throw new Error("docker inspect returned an invalid linux/amd64 image digest"); + } + return digest; +}; + +const unsupportedDockerImagePlatformError = ( + dockerImage: string, + actualPlatform: string, + expectedPlatform: string +): Error => + new Error( + [ + "docker image platform is not supported for Hyperbrowser image builds: " + + `${dockerImage} is ${actualPlatform} (expected ${expectedPlatform}).`, + `Please rebuild the image for ${expectedPlatform} and try again:`, + " cd ", + ` docker buildx build --platform ${expectedPlatform} -t ${dockerImage} ` + + "-f --load .", + ].join("\n") + ); + +const inspectDockerImage = async ( + dockerImage: string, + platform: string +): Promise> => { + const output = ( + await runDockerCommand([ + "image", + "inspect", + `--platform=${platform}`, + "--format", + "{{json .}}", + dockerImage, + ]) + ).trim(); + let inspection: unknown; + try { + inspection = JSON.parse(output); + } catch { + throw new Error("decode Docker image inspection"); + } + if (!isRecord(inspection)) { + throw new Error("docker inspect returned an invalid image inspection"); + } + const actualPlatform = `${String(inspection.Os ?? "")}/${String( + inspection.Architecture ?? "" + )}`.toLowerCase(); + if (actualPlatform !== platform.trim().toLowerCase()) { + throw unsupportedDockerImagePlatformError(dockerImage, actualPlatform, platform); + } + return inspection; +}; + +const inspectDockerContainerConfig = async ( + containerId: string +): Promise> => { + const output = ( + await runDockerCommand(["container", "inspect", "--format", "{{json .Config}}", containerId]) + ).trim(); + if (!output || output === "null") { + throw new Error("docker inspect returned empty container config"); + } + try { + return requireRecord(JSON.parse(output), "Docker container config"); + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error("decode Docker container config"); + } + throw error; + } +}; + +const removeDockerContainer = async (containerId: string): Promise => { + try { + await runDockerCommand(["rm", "-f", containerId]); + } catch (error) { + if (!(error instanceof DockerCommandError)) { + throw error; + } + } +}; + +/** Inspect platform identity with Docker API 1.49+, without temporary resources. */ +export const dockerImageDigest = async ( + dockerImage: string, + options: { platform?: string } = {} +): Promise => { + const platform = options.platform ?? IMAGE_BUILD_SOURCE_PLATFORM; + let inspection: Record; + try { + inspection = await inspectDockerImage(dockerImage, platform); + } catch (error) { + if (error instanceof DockerCommandError) { + const message = error.message; + if ( + message.includes('"--platform" requires API version') || + message.includes("unknown flag: --platform") + ) { + throw new Error( + "Local Docker image imports require a Docker CLI and Engine supporting API 1.49 " + + "or newer (Docker 28.1+). Upgrade Docker or remove an older DOCKER_API_VERSION override." + ); + } + } + throw error; + } + return normalizeSha256Digest(inspection.Id); +}; + +const manifestSource = ( + imageDigest: string, + config: Record, + cleanup: () => Promise +): DockerImageManifestSource => ({ + imageDigest, + config, + imageConfigUser: String(config.User ?? "").trim(), + imageInit: deriveAutoImageInit(config), + cleanup, +}); + +export const prepareDockerImageManifestSource = async ( + dockerImage: string, + options: { platform?: string } = {} +): Promise => { + const platform = options.platform ?? IMAGE_BUILD_SOURCE_PLATFORM; + try { + const inspection = await inspectDockerImage(dockerImage, platform); + return manifestSource( + normalizeSha256Digest(inspection.Id), + requireRecord(inspection.Config, "Docker image config"), + async () => undefined + ); + } catch (error) { + if (!(error instanceof DockerCommandError)) { + throw error; + } + } + + const containerId = ( + await runDockerCommand(["create", `--platform=${platform}`, dockerImage]) + ).trim(); + if (!containerId) { + throw new Error("docker create returned empty container ID"); + } + try { + const config = await inspectDockerContainerConfig(containerId); + let digest: string; + try { + digest = await runDockerCommand([ + "image", + "inspect", + `--platform=${platform}`, + "--format", + "{{.Id}}", + dockerImage, + ]); + } catch (error) { + if (!(error instanceof DockerCommandError)) { + throw error; + } + digest = await runDockerCommand([ + "container", + "inspect", + "--format", + "{{.Image}}", + containerId, + ]); + } + return manifestSource(normalizeSha256Digest(digest), config, () => + removeDockerContainer(containerId) + ); + } catch (error) { + await removeDockerContainer(containerId); + throw error; + } +}; + +const normalizeDockerSaveEntryName = (raw: string): string => { + if ( + !raw || + raw.includes("\\") || + raw.includes("\x00") || + raw.includes("\r") || + raw.includes("\n") || + raw.startsWith("/") + ) { + throw new Error("docker image save contains an unsafe entry path"); + } + const normalized = posixNormpath(raw); + if ( + normalized === "." || + normalized === ".." || + normalized.startsWith("../") || + normalized !== raw + ) { + throw new Error("docker image save contains an unsafe entry path"); + } + return normalized; +}; + +const storeStreamedEntry = async ( + source: AsyncIterable, + destination: string, + expectedSize: number, + name: string +): Promise => { + const hasher = createHash("sha256"); + const output = createWriteStream(destination, { flags: "wx" }); + let written = 0; + async function* chunks() { + for await (const chunk of source) { + hasher.update(chunk); + written += chunk.length; + yield chunk; + } + } + await pipeline(Readable.from(chunks(), { objectMode: false }), output); + if (written !== expectedSize) { + throw new Error(`docker image save entry "${name}" has truncated content`); + } + return { path: destination, sha256Hex: hasher.digest("hex"), sizeBytes: written }; +}; + +const parseJsonObject = (data: Buffer, label: string): Record => { + let parsed: unknown; + try { + parsed = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(data)); + } catch { + throw new Error(`${label} is not valid JSON`); + } + return requireRecord(parsed, label); +}; + +const resolveDockerSaveManifest = ( + entries: Map +): [StoredDockerSaveEntry, StoredDockerSaveEntry[]] => { + const manifestEntry = entries.get("manifest.json"); + if ( + manifestEntry === undefined || + manifestEntry.sizeBytes <= 0 || + manifestEntry.sizeBytes > MAX_DOCKER_SAVE_METADATA_BYTES + ) { + throw new Error("docker image save manifest.json is missing or too large"); + } + let manifest: unknown; + try { + manifest = JSON.parse(readFileSync(manifestEntry.path, "utf8")); + } catch { + throw new Error("docker image save manifest.json is invalid"); + } + if (!Array.isArray(manifest) || manifest.length !== 1) { + throw new Error("docker image save must contain exactly one manifest entry"); + } + const item = requireRecord(manifest[0], "Docker save manifest entry"); + const configName = normalizeDockerSaveEntryName(String(item.Config ?? "")); + const configEntry = entries.get(configName); + if ( + configEntry === undefined || + configEntry.sizeBytes <= 0 || + configEntry.sizeBytes > MAX_DOCKER_SAVE_METADATA_BYTES + ) { + throw new Error("docker image save config is missing or too large"); + } + const rawLayers = item.Layers; + if (!Array.isArray(rawLayers) || rawLayers.length > MAX_DOCKER_IMAGE_LAYERS) { + throw new Error("docker image save has too many layers"); + } + const layers: StoredDockerSaveEntry[] = []; + for (const rawLayer of rawLayers) { + const layerName = normalizeDockerSaveEntryName(String(rawLayer)); + const layer = entries.get(layerName); + if (layer === undefined || layer.sizeBytes <= 0) { + throw new Error(`docker image save layer "${layerName}" is missing`); + } + layers.push(layer); + } + return [configEntry, layers]; +}; + +const resolveOciImageDescriptor = ( + entries: Map, + imageDigest: string, + config: SandboxDockerImageConfig, + layers: SandboxDockerImageLayer[] +): SandboxDockerImageConfig => { + const digestHex = imageDigest.slice("sha256:".length); + const entry = entries.get(`blobs/sha256/${digestHex}`); + if ( + entry === undefined || + entry.sha256Hex !== digestHex || + entry.sizeBytes <= 0 || + entry.sizeBytes > MAX_DOCKER_SAVE_METADATA_BYTES + ) { + throw new Error("docker image save is missing its inspected OCI image manifest"); + } + const data = readFileSync(entry.path); + const descriptor = parseJsonObject(data, "OCI image manifest"); + if (descriptor.schemaVersion !== 2) { + throw new Error("inspected Docker image descriptor is not a valid OCI image manifest"); + } + const descriptorConfig = requireRecord(descriptor.config, "OCI config"); + if ( + descriptorConfig.digest !== `sha256:${config.sha256}` || + descriptorConfig.size !== config.sizeBytes + ) { + throw new Error("OCI image manifest config does not match Docker save config"); + } + const descriptorLayers = descriptor.layers; + if (!Array.isArray(descriptorLayers) || descriptorLayers.length !== layers.length) { + throw new Error("OCI image manifest layer count does not match Docker save manifest"); + } + descriptorLayers.forEach((rawLayer, index) => { + const item = requireRecord(rawLayer, `OCI layer ${index}`); + const layer = layers[index]; + if (item.digest !== `sha256:${layer.sha256}` || item.size !== layer.sizeBytes) { + throw new Error(`OCI image manifest layer ${index} does not match Docker save manifest`); + } + }); + return { + sha256: entry.sha256Hex, + sizeBytes: entry.sizeBytes, + dataBase64: data.toString("base64"), + }; +}; + +const terminateProcess = async (child: ChildProcess): Promise => { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + child.kill("SIGTERM"); + const timer = setTimeout(() => child.kill("SIGKILL"), 5000); + await once(child, "close"); + clearTimeout(timer); +}; + +export interface PackageDockerImageManifestOptions { + platform?: string; + tempDir?: string; +} + +/** Stream `docker image save` into verified layer artifacts plus a manifest. */ +export const packageDockerImageManifest = async ( + dockerImage: string, + imageDigest: string, + config: Record, + options: PackageDockerImageManifestOptions = {} +): Promise => { + const platform = options.platform ?? IMAGE_BUILD_SOURCE_PLATFORM; + const normalizedDigest = normalizeSha256Digest(imageDigest); + const workspace = mkdtempSync(path.join(options.tempDir ?? tmpdir(), "hb-docker-image-layers-")); + let child: ChildProcess | null = null; + try { + const args = ["image", "save", `--platform=${platform}`, dockerImage]; + child = spawn("docker", args, { stdio: ["ignore", "pipe", "pipe"] }); + const process = child; + const stdout = process.stdout; + if (!stdout || !process.stderr) { + throw new Error("docker image save did not provide stdout"); + } + let stderr = ""; + process.stderr.setEncoding("utf8"); + process.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + const exit = new Promise((resolve, reject) => { + process.on("error", (error: NodeJS.ErrnoException) => + reject(describeCommandFailure(args, error)) + ); + process.on("close", (code) => resolve(code)); + }); + exit.catch(() => undefined); + + const entries = new Map(); + let totalBytes = 0; + let index = 0; + const consume = async (): Promise => { + for await (const member of readTarEntries(stdout)) { + if (index >= MAX_DOCKER_SAVE_ENTRIES) { + throw new Error( + `docker image save contains more than ${MAX_DOCKER_SAVE_ENTRIES} entries` + ); + } + const entryIndex = index; + index += 1; + if (!member.isFile) { + continue; + } + const name = normalizeDockerSaveEntryName(member.name); + if (entries.has(name)) { + throw new Error(`docker image save contains duplicate entry "${name}"`); + } + totalBytes += member.size; + if (member.size < 0 || totalBytes > MAX_DOCKER_SAVE_ARCHIVE_BYTES) { + throw new Error("docker image save archive exceeds the size limit"); + } + const destination = path.join(workspace, `entry-${String(entryIndex).padStart(4, "0")}`); + entries.set( + name, + await storeStreamedEntry(member.content(), destination, member.size, name) + ); + } + }; + try { + await consume(); + } catch (error) { + await terminateProcess(process); + throw error; + } + const returnCode = await exit; + if (returnCode !== 0) { + const message = stderr.trim(); + throw new Error( + message + ? `docker image save ${dockerImage} failed: ${message}` + : `docker image save ${dockerImage} failed with code ${returnCode}` + ); + } + child = null; + + const [configEntry, layerEntries] = resolveDockerSaveManifest(entries); + const configBytes = readFileSync(configEntry.path); + parseJsonObject(configBytes, "Docker image config"); + const configDescriptor: SandboxDockerImageConfig = { + sha256: configEntry.sha256Hex, + sizeBytes: configEntry.sizeBytes, + dataBase64: configBytes.toString("base64"), + }; + const layerDescriptors: SandboxDockerImageLayer[] = []; + const layers: Record = {}; + for (const layer of layerEntries) { + layerDescriptors.push({ sha256: layer.sha256Hex, sizeBytes: layer.sizeBytes }); + const existing = layers[layer.sha256Hex]; + if (existing !== undefined && existing.sizeBytes !== layer.sizeBytes) { + throw new Error(`Docker layer ${layer.sha256Hex} has conflicting sizes`); + } + if (existing === undefined) { + layers[layer.sha256Hex] = { + path: layer.path, + sha256Hex: layer.sha256Hex, + sizeBytes: layer.sizeBytes, + inputFormat: DOCKER_IMAGE_MANIFEST_INPUT_FORMAT, + sourcePlatform: platform, + imageConfigUser: "", + }; + } + } + + const imageDescriptor = + normalizedDigest !== `sha256:${configDescriptor.sha256}` + ? resolveOciImageDescriptor(entries, normalizedDigest, configDescriptor, layerDescriptors) + : undefined; + const manifest: SandboxDockerImageManifest = { + version: 1, + imageDigest: normalizedDigest, + ...(imageDescriptor ? { descriptor: imageDescriptor } : {}), + config: configDescriptor, + layers: layerDescriptors, + }; + const artifact = writeManifestArtifact( + workspace, + "docker-image-manifest.json", + canonicalManifestJson(manifest), + DOCKER_IMAGE_MANIFEST_INPUT_FORMAT, + { + imageConfigUser: String(config.User ?? "").trim(), + imageInit: deriveAutoImageInit(config), + } + ); + return { + artifact, + manifest, + layers, + workspace, + cleanup: () => removeWorkspace(workspace), + }; + } catch (error) { + if (child !== null) { + await terminateProcess(child); + } + removeWorkspace(workspace); + throw error; + } +}; diff --git a/src/sandbox/image-build/dockerfile-analysis.ts b/src/sandbox/image-build/dockerfile-analysis.ts new file mode 100644 index 0000000..e756da5 --- /dev/null +++ b/src/sandbox/image-build/dockerfile-analysis.ts @@ -0,0 +1,417 @@ +/** + * Conservative Dockerfile analysis for remote build context selection. + * + * This module intentionally does not try to execute or fully reproduce the + * BuildKit Dockerfile frontend. It recognizes the context-consuming syntax the + * SDK can analyze safely and requests a full context for anything ambiguous. + * That keeps sparse uploads an optimization rather than a correctness + * requirement. + */ + +const KNOWN_INSTRUCTIONS = new Set([ + "ADD", + "ARG", + "CMD", + "COPY", + "ENTRYPOINT", + "ENV", + "EXPOSE", + "FROM", + "HEALTHCHECK", + "LABEL", + "MAINTAINER", + "ONBUILD", + "RUN", + "SHELL", + "STOPSIGNAL", + "USER", + "VOLUME", + "WORKDIR", +]); +const KNOWN_RUN_FLAGS = new Set(["device", "mount", "network", "security"]); +const INSTRUCTION_PATTERN = /^([A-Za-z]+)(?:[ \t]+(.*))?$/s; +const DIRECTIVE_PATTERN = /^\s*#\s*(escape|syntax)\s*=\s*(\S+)/gim; +const FLAG_NAME_PATTERN = /^[a-z][a-z0-9-]*$/; +const SCP_GIT_SOURCE_PATTERN = /^git@[^:/\s]+:.+/; +// Python splitlines includes these control separators. +// eslint-disable-next-line no-control-regex +const LINE_SEPARATOR_PATTERN = /\r\n|[\n\r\v\f\x1c\x1d\x1e\x85\u2028\u2029]/; + +class AnalysisFallback extends Error { + constructor(readonly reason: string) { + super(reason); + } +} + +class ParseError extends Error {} + +export type DockerfileSourceAnalysis = { groups: string[][]; fallbackReason: string }; + +/** Python `str.strip()` semantics: a UTF-8 BOM is not whitespace and must not be trimmed. */ +const stripWhitespace = (value: string): string => + value.replace(/^(?:(?!\ufeff)\s)+|(?:(?!\ufeff)\s)+$/g, ""); + +/** POSIX `shlex.split` word splitting. */ +export const shlexSplit = (value: string): string[] => { + const words: string[] = []; + let current = ""; + let inWord = false; + let quote: '"' | "'" | null = null; + let index = 0; + while (index < value.length) { + const character = value[index]; + index += 1; + if (quote === "'") { + if (character === "'") { + quote = null; + } else { + current += character; + } + continue; + } + if (quote === '"') { + if (character === '"') { + quote = null; + } else if (character === "\\") { + if (index >= value.length) { + throw new ParseError("No escaped character"); + } + const escaped = value[index]; + index += 1; + if (escaped === '"' || escaped === "\\") { + current += escaped; + } else { + current += "\\" + escaped; + } + } else { + current += character; + } + continue; + } + if (character === "\\") { + if (index >= value.length) { + throw new ParseError("No escaped character"); + } + current += value[index]; + index += 1; + inWord = true; + } else if (character === '"' || character === "'") { + quote = character; + inWord = true; + } else if (/\s/.test(character)) { + if (inWord) { + words.push(current); + current = ""; + inWord = false; + } + } else { + current += character; + inWord = true; + } + } + if (quote !== null) { + throw new ParseError("No closing quotation"); + } + if (inWord) { + words.push(current); + } + return words; +}; + +const isOfficialDockerfileFrontend = (reference: string): boolean => { + let normalized = reference.trim(); + if (normalized.startsWith("docker-image://")) { + normalized = normalized.slice("docker-image://".length); + } + normalized = normalized.split("@", 1)[0]; + const lastSlash = normalized.lastIndexOf("/"); + const lastColon = normalized.lastIndexOf(":"); + if (lastColon > lastSlash) { + normalized = normalized.slice(0, lastColon); + } + return new Set([ + "docker/dockerfile", + "docker.io/docker/dockerfile", + "index.docker.io/docker/dockerfile", + "docker/dockerfile-upstream", + "docker.io/docker/dockerfile-upstream", + "index.docker.io/docker/dockerfile-upstream", + ]).has(normalized); +}; + +const isRemoteAddSource = (source: string): boolean => { + if (SCP_GIT_SOURCE_PATTERN.test(source)) { + return true; + } + return ["http://", "https://", "git://", "ssh://"].some((prefix) => source.startsWith(prefix)); +}; + +const sourceHasPattern = (source: string): boolean => + source.includes("*") || source.includes("?") || source.includes("["); + +const takeWord = (value: string): [string, string] => { + let quote: string | null = null; + let escaped = false; + for (let index = 0; index < value.length; index += 1) { + const character = value[index]; + if (escaped) { + escaped = false; + continue; + } + if (character === "\\" && quote !== "'") { + escaped = true; + continue; + } + if (quote !== null) { + if (character === quote) { + quote = null; + } + continue; + } + if (character === '"' || character === "'") { + quote = character; + } else if (/\s/.test(character)) { + return [value.slice(0, index), value.slice(index)]; + } + } + if (quote !== null || escaped) { + throw new ParseError("unterminated Dockerfile instruction word"); + } + return [value, ""]; +}; + +const splitLeadingFlags = (body: string): [Array<[string, string]>, string] => { + const flags: Array<[string, string]> = []; + let remaining = body.trimStart(); + while (remaining.startsWith("--")) { + const [rawToken, rest] = takeWord(remaining); + remaining = rest; + const decoded = shlexSplit(rawToken); + if (decoded.length !== 1 || !decoded[0].startsWith("--")) { + throw new ParseError("invalid Dockerfile instruction flag"); + } + const flag = decoded[0].slice(2); + const separator = flag.indexOf("="); + const key = (separator === -1 ? flag : flag.slice(0, separator)).toLowerCase(); + const value = separator === -1 ? "" : flag.slice(separator + 1); + if (!FLAG_NAME_PATTERN.test(key)) { + throw new ParseError("invalid Dockerfile instruction flag"); + } + if (key === "mount" && (rawToken.includes('"') || rawToken.includes("'"))) { + // BuildKit parses mount values as CSV after Dockerfile word processing. + // Quoted CSV values are deliberately outside the subset implemented here. + throw new ParseError("quoted RUN mount requires full context"); + } + flags.push([key, value]); + remaining = remaining.trimStart(); + } + return [flags, remaining]; +}; + +const parseCopyAddValues = (body: string): [Map, string[]] => { + const [parsedFlags, remaining] = splitLeadingFlags(body); + const flags = new Map(parsedFlags); + let values: string[]; + if (remaining.startsWith("[")) { + const parsed: unknown = JSON.parse(remaining); + if (!Array.isArray(parsed) || !parsed.every((value) => typeof value === "string")) { + throw new ParseError("invalid JSON COPY/ADD"); + } + values = parsed; + } else { + if (remaining.includes("\\")) { + // BuildKit's shell-form COPY/ADD word splitting is not POSIX shlex. + // Backslash-containing forms use the full context rather than risk + // selecting a different set of sources. + throw new ParseError("escaped shell COPY/ADD requires full context"); + } + values = shlexSplit(remaining); + } + if (values.length < 2) { + throw new ParseError("COPY/ADD is missing source or destination"); + } + return [flags, values]; +}; + +const parseMountOptions = (value: string): Map => { + const options = new Map(); + for (const field of value.split(",")) { + const separator = field.indexOf("="); + const key = (separator === -1 ? field : field.slice(0, separator)).trim().toLowerCase(); + if (!key) { + throw new AnalysisFallback("run_mount_parse_failed"); + } + options.set(key, separator === -1 ? "" : field.slice(separator + 1).trim()); + } + return options; +}; + +const parseInstruction = (line: string): [string, string] => { + const match = INSTRUCTION_PATTERN.exec(line); + if (match === null) { + throw new AnalysisFallback("dockerfile_parse_failed"); + } + return [match[1].toUpperCase(), match[2] || ""]; +}; + +const validateParserDirectives = (text: string): void => { + for (const match of text.matchAll(DIRECTIVE_PATTERN)) { + const name = match[1].toLowerCase(); + const value = match[2]; + if (name === "escape" && value !== "\\") { + throw new AnalysisFallback("dockerfile_parse_failed"); + } + if (name === "syntax" && !isOfficialDockerfileFrontend(value)) { + throw new AnalysisFallback("custom_dockerfile_frontend"); + } + } +}; + +const logicalLines = (text: string): string[] => { + const lines: string[] = []; + let parts: string[] = []; + const rawLines = text.split(LINE_SEPARATOR_PATTERN); + if (rawLines.length > 0 && rawLines[rawLines.length - 1] === "") { + rawLines.pop(); + } + for (const rawLine of rawLines) { + const stripped = stripWhitespace(rawLine); + if (parts.length === 0 && (!stripped || stripped.startsWith("#"))) { + continue; + } + if (parts.length > 0 && (!stripped || stripped.startsWith("#"))) { + // BuildKit has nuanced rules for comments and empty lines in a + // continuation. Full context is the safe answer here. + throw new AnalysisFallback("dockerfile_parse_failed"); + } + if (stripped.endsWith("\\")) { + parts.push(stripped.slice(0, -1).trimEnd()); + continue; + } + parts.push(stripped); + lines.push(parts.join(" ")); + parts = []; + } + if (parts.length > 0) { + throw new AnalysisFallback("dockerfile_parse_failed"); + } + return lines; +}; + +const analyzeCopyOrAdd = (instruction: string, body: string): string[] => { + let flags: Map; + let values: string[]; + try { + [flags, values] = parseCopyAddValues(body); + } catch (error) { + if (error instanceof ParseError || error instanceof SyntaxError) { + throw new AnalysisFallback("dockerfile_instruction_parse_failed"); + } + throw error; + } + + if (instruction === "COPY" && flags.has("from")) { + return []; + } + + const localSources: string[] = []; + for (const source of values.slice(0, -1)) { + if (source.includes("$") || source.includes("\x00")) { + throw new AnalysisFallback( + instruction === "COPY" ? "copy_source_requires_expansion" : "add_source_requires_expansion" + ); + } + if (source.includes("\\")) { + // A backslash is a path character on Unix but a separator on Windows. + // Full context keeps selection platform-independent. + throw new AnalysisFallback("dockerfile_source_path"); + } + if (instruction === "ADD" && isRemoteAddSource(source)) { + continue; + } + if (sourceHasPattern(source)) { + throw new AnalysisFallback("dockerfile_source_pattern"); + } + localSources.push(source); + } + return localSources; +}; + +const analyzeRun = (body: string): string[][] => { + let flags: Array<[string, string]>; + try { + [flags] = splitLeadingFlags(body); + } catch (error) { + if (error instanceof ParseError) { + throw new AnalysisFallback("run_mount_parse_failed"); + } + throw error; + } + + const groups: string[][] = []; + for (const [name, value] of flags) { + if (!KNOWN_RUN_FLAGS.has(name)) { + throw new AnalysisFallback("dockerfile_instruction_parse_failed"); + } + if (name !== "mount") { + continue; + } + if (!value || value.includes('"') || value.includes("'")) { + throw new AnalysisFallback("run_mount_parse_failed"); + } + const options = parseMountOptions(value); + if ((options.get("type") ?? "bind") !== "bind" || options.get("from")) { + continue; + } + const source = options.get("source") || options.get("src") || "."; + if (source.includes("$") || source.includes("\x00")) { + throw new AnalysisFallback("run_bind_source_requires_expansion"); + } + groups.push([source]); + } + return groups; +}; + +const analyzeLines = (lines: string[]): string[][] => { + const groups: string[][] = []; + for (const line of lines) { + const [instruction, body] = parseInstruction(line); + if (!KNOWN_INSTRUCTIONS.has(instruction)) { + throw new AnalysisFallback("dockerfile_instruction_parse_failed"); + } + if (instruction === "COPY" || instruction === "ADD") { + const group = analyzeCopyOrAdd(instruction, body); + if (group.length > 0) { + groups.push(group); + } + } else if (instruction === "RUN") { + groups.push(...analyzeRun(body)); + } + } + return groups; +}; + +/** Return local source groups or a reason to upload the full context. */ +export const analyzeDockerfileSources = (data: Buffer): DockerfileSourceAnalysis => { + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(data); + } catch { + return { groups: [], fallbackReason: "dockerfile_parse_failed" }; + } + + try { + validateParserDirectives(text); + // Heredocs are valid BuildKit syntax, but treating the entire Dockerfile + // as full context is safer than partially parsing them. + if (text.includes("<<")) { + throw new AnalysisFallback("dockerfile_instruction_parse_failed"); + } + return { groups: analyzeLines(logicalLines(text)), fallbackReason: "" }; + } catch (error) { + if (error instanceof AnalysisFallback) { + return { groups: [], fallbackReason: error.reason }; + } + throw error; + } +}; diff --git a/src/sandbox/image-build/dockerignore.ts b/src/sandbox/image-build/dockerignore.ts new file mode 100644 index 0000000..cfcb6d6 --- /dev/null +++ b/src/sandbox/image-build/dockerignore.ts @@ -0,0 +1,294 @@ +/** + * Docker-compatible `.dockerignore` parsing and matching. + * + * The matching contract intentionally follows Moby `patternmatcher` v0.6.1, + * which is also what the Hyperbrowser CLI reaches through BuildKit's + * filesystem utilities. + */ + +import { readFileSync } from "fs"; +import { posixDirname, posixNormpath } from "./common"; + +const REGEX_META_WITHOUT_GLOB_MEANING = new Set(".+()|{}$"); + +/** Apply the Unix `filepath.Clean` behavior used by the CLI. */ +const cleanPath = (value: string): string => { + if (value.startsWith("//")) { + value = "/" + value.replace(/^\/+/, ""); + } + return posixNormpath(value); +}; + +/** Parse ignore-file lines like Moby's `ignorefile.ReadAll`. */ +export const readIgnorePatterns = (text: string): string[] => { + const patterns: string[] = []; + text.split("\n").forEach((rawLine, index) => { + if (rawLine.endsWith("\r")) { + rawLine = rawLine.slice(0, -1); + } + if (index === 0 && rawLine.startsWith("\ufeff")) { + rawLine = rawLine.slice(1); + } + if (rawLine.startsWith("#")) { + return; + } + let value = rawLine.trim(); + if (!value) { + return; + } + const exclusion = value.startsWith("!"); + if (exclusion) { + value = value.slice(1).trim(); + } + if (value) { + value = cleanPath(value); + if (value.length > 1 && value.startsWith("/")) { + value = value.slice(1); + } + } + patterns.push((exclusion ? "!" : "") + value); + }); + return patterns; +}; + +type MatchType = "exact" | "prefix" | "suffix" | "regexp"; + +const validateCharacterClass = (pattern: string, cursor: number): number => { + const consumeValue = (position: number): number => { + if (position >= pattern.length || pattern[position] === "-" || pattern[position] === "]") { + throw new Error(`invalid Docker ignore pattern "${pattern}"`); + } + if (pattern[position] === "\\") { + position += 1; + if (position >= pattern.length) { + throw new Error(`invalid Docker ignore pattern "${pattern}"`); + } + } + position += 1; + if (position >= pattern.length) { + throw new Error(`invalid Docker ignore pattern "${pattern}"`); + } + return position; + }; + + if (cursor < pattern.length && pattern[cursor] === "^") { + cursor += 1; + } + let ranges = 0; + while (true) { + if (cursor < pattern.length && pattern[cursor] === "]" && ranges) { + return cursor + 1; + } + cursor = consumeValue(cursor); + if (cursor < pattern.length && pattern[cursor] === "-") { + cursor = consumeValue(cursor + 1); + } + ranges += 1; + } +}; + +/** Reject malformed patterns using Go filepath.Match's Unix grammar. */ +const validateFilepathPattern = (pattern: string): void => { + let cursor = 0; + while (cursor < pattern.length) { + const character = pattern[cursor]; + if (character === "\\") { + cursor += 1; + if (cursor === pattern.length) { + throw new Error(`invalid Docker ignore pattern "${pattern}": trailing escape`); + } + } else if (character === "[") { + cursor = validateCharacterClass(pattern, cursor + 1); + continue; + } + cursor += 1; + } +}; + +/** Compile one cleaned Moby pattern into its optimized match form. */ +const compilePattern = (pattern: string): { matchType: MatchType; regexp: RegExp | null } => { + const parts = ["^"]; + let matchType: MatchType = "exact"; + let cursor = 0; + let tokenIndex = 0; + let inClass = false; + while (cursor < pattern.length) { + const character = pattern[cursor]; + cursor += 1; + + if (character === "*") { + if (cursor < pattern.length && pattern[cursor] === "*") { + cursor += 1; + if (cursor < pattern.length && pattern[cursor] === "/") { + cursor += 1; + } + + if (cursor === pattern.length) { + if (matchType === "exact") { + matchType = "prefix"; + } else { + parts.push(".*"); + matchType = "regexp"; + } + } else { + parts.push("(?:.*/)?"); + matchType = "regexp"; + } + + if (tokenIndex === 0) { + matchType = "suffix"; + } + } else { + parts.push("[^/]*"); + matchType = "regexp"; + } + } else if (character === "?") { + parts.push("[^/]"); + matchType = "regexp"; + } else if (REGEX_META_WITHOUT_GLOB_MEANING.has(character)) { + parts.push("\\" + character); + } else if (character === "\\") { + if (cursor < pattern.length) { + const codepoint = pattern.codePointAt(cursor)!; + parts.push(`\\u{${codepoint.toString(16)}}`); + cursor += codepoint > 0xffff ? 2 : 1; + matchType = "regexp"; + } else { + throw new Error(`invalid Docker ignore pattern "${pattern}": trailing escape`); + } + } else { + // Brackets remain regex syntax because they are also filepath glob + // syntax. All other characters are literal in the Moby compiler. + parts.push(character === "]" && !inClass ? "\\]" : character); + if (character === "[") inClass = true; + else if (character === "]") inClass = false; + if (character === "[" || character === "]") { + matchType = "regexp"; + } + } + + tokenIndex += 1; + } + + if (matchType !== "regexp") { + return { matchType, regexp: null }; + } + + parts.push("$"); + try { + // Python and Go wildcards consume Unicode codepoints, not UTF-16 units. + return { matchType, regexp: new RegExp(parts.join(""), "u") }; + } catch (error) { + throw new Error( + `invalid Docker ignore pattern "${pattern}": ${error instanceof Error ? error.message : error}` + ); + } +}; + +class DockerIgnorePattern { + readonly value: string; + readonly exclusion: boolean; + private readonly matchType: MatchType; + private readonly regexp: RegExp | null; + + constructor(rawPattern: string) { + let value = cleanPath(rawPattern.trim()); + const exclusion = value.startsWith("!"); + if (exclusion) { + if (value === "!") { + throw new Error('illegal exclusion pattern: "!"'); + } + value = value.slice(1); + } + validateFilepathPattern(value); + const compiled = compilePattern(value); + this.value = value; + this.exclusion = exclusion; + this.matchType = compiled.matchType; + this.regexp = compiled.regexp; + } + + matchesPath(path: string): boolean { + if (this.matchType === "exact") { + return path === this.value; + } + if (this.matchType === "prefix") { + return path.startsWith(this.value.slice(0, -2)); + } + if (this.matchType === "suffix") { + const suffix = this.value.slice(2); + return path.endsWith(suffix) || (suffix.startsWith("/") && path === suffix.slice(1)); + } + if (this.regexp === null) { + throw new Error(`invalid Docker ignore pattern: "${this.value}"`); + } + return this.regexp.test(path); + } +} + +/** Ordered Docker ignore matcher with parent-directory semantics. */ +export class DockerIgnoreMatcher { + private readonly patterns: DockerIgnorePattern[]; + readonly hasNegations: boolean; + + constructor(patterns: Iterable) { + const compiled: DockerIgnorePattern[] = []; + for (const pattern of patterns) { + const cleaned = pattern.trim(); + if (!cleaned) { + continue; + } + compiled.push(new DockerIgnorePattern(cleaned)); + } + this.patterns = compiled; + this.hasNegations = compiled.some((pattern) => pattern.exclusion); + } + + static fromFile(path: string): DockerIgnoreMatcher { + let text = readFileSync(path).toString("utf8"); + if (text.startsWith("\ufeff")) { + text = text.slice(1); + } + return new DockerIgnoreMatcher(readIgnorePatterns(text)); + } + + static fromText(text: string): DockerIgnoreMatcher { + return new DockerIgnoreMatcher(readIgnorePatterns(text)); + } + + matches(relativePath: string): boolean { + // Traversal supplies slash-delimited paths. On Unix, a backslash can be + // part of a filename and must remain available for glob escaping. + const path = cleanPath(relativePath); + if (path === ".") { + return false; + } + + const parent = posixDirname(path); + const parentParts = parent && parent !== "." ? parent.split("/") : []; + let matched = false; + for (const pattern of this.patterns) { + // An exclusion can only re-include an ignored path, and a normal pattern + // only needs checking while the path is included. + if (pattern.exclusion !== matched) { + continue; + } + + let patternMatches = pattern.matchesPath(path); + if (!patternMatches) { + for (let length = 1; length <= parentParts.length; length += 1) { + if (pattern.matchesPath(parentParts.slice(0, length).join("/"))) { + patternMatches = true; + break; + } + } + } + + if (patternMatches) { + matched = !pattern.exclusion; + } + } + + return matched; + } +} diff --git a/src/sandbox/image-build/gzip.ts b/src/sandbox/image-build/gzip.ts new file mode 100644 index 0000000..438c0dd --- /dev/null +++ b/src/sandbox/image-build/gzip.ts @@ -0,0 +1,102 @@ +import { createHash } from "crypto"; +import { createWriteStream } from "fs"; +import { once } from "events"; +import { createDeflateRaw } from "zlib"; +import { PaxTarWriter } from "./tar"; + +const CRC_TABLE = (() => { + const table = new Uint32Array(256); + for (let index = 0; index < 256; index += 1) { + let value = index; + for (let bit = 0; bit < 8; bit += 1) { + value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1; + } + table[index] = value >>> 0; + } + return table; +})(); + +const updateCrc32 = (crc: number, chunk: Buffer): number => { + let value = ~crc >>> 0; + for (const byte of chunk) { + value = CRC_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8); + } + return ~value >>> 0; +}; + +/** gzip member header matching CPython `GzipFile(filename="", mtime=0)` at compresslevel 1. */ +const GZIP_HEADER = Buffer.from([0x1f, 0x8b, 0x08, 0x00, 0, 0, 0, 0, 0x00, 0xff]); + +export interface GzipTarResult { + sizeBytes: number; + sha256Hex: string; + uncompressedSizeBytes: number; +} + +/** + * Write a deterministic gzip-compressed tar archive to `path`. The `populate` + * callback adds entries through the supplied writer. + */ +export const writeGzipTar = async ( + path: string, + populate: (writer: PaxTarWriter) => Promise +): Promise => { + const output = createWriteStream(path, { flags: "wx" }); + // Observe write errors immediately, including failures before the first drain. + let outputError: Error | undefined; + output.on("error", (error) => { outputError = error; deflate.destroy(error); }); + const hasher = createHash("sha256"); + let compressedSize = 0; + let crc = 0; + let uncompressedSize = 0; + + const writeOutput = async (chunk: Buffer): Promise => { + if (outputError) throw outputError; + hasher.update(chunk); + compressedSize += chunk.length; + if (!output.write(chunk)) { + await once(output, "drain"); + } + }; + + const deflate = createDeflateRaw({ level: 1 }); + const drainDeflate = (async () => { + for await (const chunk of deflate) { + await writeOutput(chunk as Buffer); + } + })(); + // A source failure can occur while compression is still draining. + void drainDeflate.catch(() => undefined); + + try { + await writeOutput(GZIP_HEADER); + const writer = new PaxTarWriter(async (chunk) => { + crc = updateCrc32(crc, chunk); + uncompressedSize += chunk.length; + if (!deflate.write(chunk)) { + await once(deflate, "drain"); + } + }); + await populate(writer); + await writer.close(); + deflate.end(); + await drainDeflate; + const trailer = Buffer.alloc(8); + trailer.writeUInt32LE(crc >>> 0, 0); + trailer.writeUInt32LE(uncompressedSize % 2 ** 32, 4); + await writeOutput(trailer); + output.end(); + await once(output, "finish"); + } catch (error) { + deflate.destroy(); + output.destroy(); + await drainDeflate.catch(() => undefined); + throw error; + } + + return { + sizeBytes: compressedSize, + sha256Hex: hasher.digest("hex"), + uncompressedSizeBytes: uncompressedSize, + }; +}; diff --git a/src/sandbox/image-build/image-init.ts b/src/sandbox/image-build/image-init.ts new file mode 100644 index 0000000..f546092 --- /dev/null +++ b/src/sandbox/image-build/image-init.ts @@ -0,0 +1,111 @@ +import { SandboxImageInit } from "../../types/sandbox"; + +const IMAGE_INIT_ENV_KEY_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const RESERVED_IMAGE_INIT_ENV_KEYS = new Set([ + "SANDBOX_ENABLED", + "SANDBOX_DEFAULT_WORKING_DIR", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "PWD", + "DISPLAY", +]); + +const listStringConfig = (value: unknown): string[] => + Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; + +const normalizeInitArgs = (values: string[] | undefined | null): string[] => + (values ?? []).filter((value) => value); + +const deriveAutoImageEnv = (entries: string[]): Record => { + const env = new Map(); + for (const entry of entries) { + const separator = entry.indexOf("="); + if (separator === -1) { + continue; + } + const key = entry.slice(0, separator).trim(); + if (!key || !IMAGE_INIT_ENV_KEY_PATTERN.test(key) || RESERVED_IMAGE_INIT_ENV_KEYS.has(key)) { + continue; + } + env.set(key, entry.slice(separator + 1)); + } + return Object.fromEntries(env); +}; + +const deriveAutoStartupArgs = (entrypoint: string[], cmd: string[]): string[] => + (entrypoint.length > 0 ? [...entrypoint, ...cmd] : [...cmd]).filter((arg) => arg); + +/** Derive default sandbox initialization from a Docker image config. */ +export const deriveAutoImageInit = ( + config: Record +): SandboxImageInit | undefined => { + const env = deriveAutoImageEnv(listStringConfig(config.Env)); + const args = deriveAutoStartupArgs( + listStringConfig(config.Entrypoint), + listStringConfig(config.Cmd) + ); + const workingDir = String(config.WorkingDir ?? "").trim(); + if (Object.keys(env).length === 0 && args.length === 0 && !workingDir) { + return undefined; + } + const init: SandboxImageInit = {}; + if (Object.keys(env).length > 0) { + init.env = env; + } + if (args.length > 0) { + init.args = args; + } + if (workingDir) { + init.workingDir = workingDir; + } + return init; +}; + +/** Layer explicit overrides over image-derived initialization defaults. */ +export const mergeImageInit = ( + automatic: SandboxImageInit | undefined, + explicit: SandboxImageInit | undefined +): SandboxImageInit | undefined => { + if (automatic === undefined && explicit === undefined) { + return undefined; + } + const env: Record = { ...(automatic?.env ?? {}), ...(explicit?.env ?? {}) }; + let command = (automatic?.command ?? "").trim(); + let args = normalizeInitArgs(automatic?.args); + let workingDir = (automatic?.workingDir ?? "").trim(); + if (explicit !== undefined) { + const explicitWorkingDir = (explicit.workingDir ?? "").trim(); + if (explicitWorkingDir) { + workingDir = explicitWorkingDir; + } + const explicitArgs = normalizeInitArgs(explicit.args); + if (explicitArgs.length > 0) { + args = explicitArgs; + command = ""; + } + const explicitCommand = (explicit.command ?? "").trim(); + if (explicitCommand) { + command = explicitCommand; + args = []; + } + } + if (Object.keys(env).length === 0 && !command && args.length === 0 && !workingDir) { + return undefined; + } + const merged: SandboxImageInit = {}; + if (Object.keys(env).length > 0) { + merged.env = env; + } + if (command) { + merged.command = command; + } + if (args.length > 0) { + merged.args = args; + } + if (workingDir) { + merged.workingDir = workingDir; + } + return merged; +}; diff --git a/src/sandbox/image-build/index.ts b/src/sandbox/image-build/index.ts new file mode 100644 index 0000000..6a52b21 --- /dev/null +++ b/src/sandbox/image-build/index.ts @@ -0,0 +1,44 @@ +export { + CONTEXT_MANIFEST_INPUT_FORMAT, + DOCKER_IMAGE_MANIFEST_INPUT_FORMAT, + IMAGE_BUILD_INPUT_FORMAT, + IMAGE_BUILD_SOURCE_PLATFORM, +} from "./artifacts"; +export type { DockerImageBuildArtifact } from "./artifacts"; +export { + DockerBuildContextChangedError, + dockerBuildContextFingerprint, + packageDockerBuildContextManifest, +} from "./context"; +export type { + DockerBuildContextOptions, + PackageDockerBuildContextOptions, + PackagedDockerBuildContext, +} from "./context"; +export { + DockerCommandError, + dockerImageDigest, + packageDockerImageManifest, + prepareDockerImageManifestSource, +} from "./docker-image"; +export type { + DockerImageManifestSource, + PackagedDockerImage, + PackageDockerImageManifestOptions, +} from "./docker-image"; +export { DockerIgnoreMatcher } from "./dockerignore"; +export { analyzeDockerfileSources } from "./dockerfile-analysis"; +export { deriveAutoImageInit, mergeImageInit } from "./image-init"; +export { + buildDockerImageFromDockerfile, + makeTempDockerTag, + removeDockerImage, +} from "./local-docker"; +export { + completedImageId, + imageBuildName, + isTerminalImageBuildStatus, + matchingImageBuild, +} from "./resolution"; +export type { ImageBuildNameOptions, ImageBuildSource } from "./resolution"; +export { uploadImageBuildArtifact, uploadMissingImageBuildArtifacts } from "./upload"; diff --git a/src/sandbox/image-build/local-docker.ts b/src/sandbox/image-build/local-docker.ts new file mode 100644 index 0000000..231d661 --- /dev/null +++ b/src/sandbox/image-build/local-docker.ts @@ -0,0 +1,56 @@ +import { randomUUID } from "crypto"; +import { existsSync } from "fs"; +import * as path from "path"; +import { IMAGE_BUILD_SOURCE_PLATFORM } from "./artifacts"; +import { DockerCommandError, runDockerCommand } from "./docker-image"; + +export const makeTempDockerTag = (prefix = "hyperbrowser-sdk-build"): string => + `${prefix}:${randomUUID().replace(/-/g, "")}`; + +export const removeDockerImage = async (image: string): Promise => { + try { + await runDockerCommand(["image", "rm", image]); + } catch (error) { + if (!(error instanceof DockerCommandError)) { + throw error; + } + } +}; + +export interface LocalDockerBuildOptions { + contextPath: string; + dockerfile?: string; + tag: string; + platform?: string; + buildArgs?: Record; +} + +/** Build a Dockerfile locally with `docker buildx build --load`. */ +export const buildDockerImageFromDockerfile = async ( + options: LocalDockerBuildOptions +): Promise => { + const context = options.contextPath; + if (!existsSync(context)) { + throw new Error(`Docker build context not found: ${context}`); + } + let dockerfilePath = options.dockerfile ?? "Dockerfile"; + if (!path.isAbsolute(dockerfilePath)) { + dockerfilePath = path.join(context, dockerfilePath); + } + const args = [ + "buildx", + "build", + "--platform", + options.platform ?? IMAGE_BUILD_SOURCE_PLATFORM, + "-t", + options.tag, + "-f", + dockerfilePath, + "--load", + ]; + for (const [key, value] of Object.entries(options.buildArgs ?? {})) { + args.push("--build-arg", `${key}=${value}`); + } + args.push(context); + await runDockerCommand(args); +}; diff --git a/src/sandbox/image-build/resolution.ts b/src/sandbox/image-build/resolution.ts new file mode 100644 index 0000000..fced714 --- /dev/null +++ b/src/sandbox/image-build/resolution.ts @@ -0,0 +1,158 @@ +/** Shared identity and validation for opt-in image resolution. */ + +import { HyperbrowserError } from "../../error"; +import { SandboxImageBuild, SandboxImageInit } from "../../types/sandbox"; +import { blake2b } from "./blake2b"; +import { compactJson, isRecord } from "./common"; + +export type ImageBuildSource = "dockerfile" | "prebuilt"; + +export interface ImageBuildNameOptions { + source: ImageBuildSource; + fingerprint: string; + namePrefix?: string; + platform?: string; + imageInit?: SandboxImageInit; + imageConfigUser?: string; +} + +const normalizeInitEnv = (env: Record | undefined) => + env === undefined ? undefined : env; + +/** Drop undefined/null fields so identity matches `exclude_none` serialization. */ +export const normalizeImageInit = ( + imageInit: SandboxImageInit | undefined +): SandboxImageInit | undefined => { + if (imageInit === undefined) { + return undefined; + } + const normalized: SandboxImageInit = {}; + const env = normalizeInitEnv(imageInit.env); + if (env !== undefined && env !== null) { + normalized.env = env; + } + if (imageInit.command !== undefined && imageInit.command !== null) { + normalized.command = imageInit.command; + } + if (imageInit.args !== undefined && imageInit.args !== null) { + normalized.args = imageInit.args; + } + if (imageInit.workingDir !== undefined && imageInit.workingDir !== null) { + normalized.workingDir = imageInit.workingDir; + } + return normalized; +}; + +/** + * Name an immutable input identity without packaging or uploading it. + * + * Fingerprints identify effective Dockerfile contexts or platform-specific + * Docker image digests. Builder resources and wait policies do not change the + * image contents and are excluded. Mutable external inputs (base tags, network + * downloads) require an explicit `forceBuild` to request another build. + */ +export const imageBuildName = (options: ImageBuildNameOptions): string => { + const platform = (options.platform ?? "linux/amd64").trim().toLowerCase(); + const namePrefix = options.namePrefix ?? "hb"; + if (!/^[a-z0-9]+\/[a-z0-9]+(?:\/[a-z0-9]+)?$/.test(platform)) { + throw new Error("platform must be an OCI platform such as 'linux/amd64'"); + } + if (!/^[A-Za-z0-9_-]{1,21}$/.test(namePrefix)) { + throw new Error("imageNamePrefix must be 1-21 letters, digits, '_' or '-'"); + } + let payload: string; + if (options.source === "prebuilt") { + const fingerprint = options.fingerprint.toLowerCase(); + if (!/^sha256:[0-9a-f]{64}$/.test(fingerprint)) { + throw new Error("expectedImageDigest must be a sha256: Docker digest"); + } + payload = `docker_image\0${fingerprint}\0platform\0${platform}`; + } else if (options.source === "dockerfile") { + if (!/^[0-9a-f]{64}$/.test(options.fingerprint)) { + throw new Error("expectedContextFingerprint must be a SHA-256 hex digest"); + } + payload = `dockerfile-context-v3\0${options.fingerprint}\0platform\0${platform}`; + } else { + throw new Error("source must be 'dockerfile' or 'prebuilt'"); + } + const identityOptions: Record = {}; + const initialization = normalizeImageInit(options.imageInit); + if (initialization !== undefined && Object.keys(initialization).length > 0) { + identityOptions.imageInit = initialization; + } + if (options.imageConfigUser !== undefined) { + identityOptions.imageConfigUser = options.imageConfigUser.trim(); + } + if (Object.keys(identityOptions).length > 0) { + payload += "\0options\0" + compactJson(identityOptions, { sortKeys: true }); + } + const digest = blake2b(Buffer.from(payload, "utf8"), 8).toString("hex"); + const name = `${namePrefix}__${options.source}__${digest}__${platform.replace(/\//g, "-")}`; + if (name.length > 64) { + throw new Error("imageNamePrefix and platform produce a name longer than 64 characters"); + } + return name; +}; + +const IMAGE_BUILD_STATUSES = new Set([ + "awaiting_upload", + "upload_verified", + "dispatching", + "building", + "verifying", + "completed", + "failed", + "canceled", +]); + +/** Return the compatible in-progress build described by a 409 conflict, if any. */ +export const matchingImageBuild = ( + error: HyperbrowserError, + imageName: string, + inputFormat: string +): SandboxImageBuild | null => { + if (error.statusCode !== 409 || error.code !== "image_build_in_progress") { + return null; + } + if (!isRecord(error.details)) { + return null; + } + const data = error.details.build; + if (!isRecord(data)) { + return null; + } + const metadata = data.metadata; + if (!isRecord(metadata) || metadata.inputFormat !== inputFormat) { + return null; + } + if ((metadata.sourcePlatform ?? "linux/amd64") !== "linux/amd64") { + return null; + } + if ( + typeof data.id !== "string" || + !data.id || + typeof data.imageName !== "string" || + typeof data.status !== "string" || + !IMAGE_BUILD_STATUSES.has(data.status) + ) { + return null; + } + const build = data as unknown as SandboxImageBuild; + if (build.imageName !== imageName) { + return null; + } + return build; +}; + +export const completedImageId = (build: SandboxImageBuild): string | undefined => { + if (build.status !== "completed") { + return undefined; + } + if (!build.imageId) { + throw new Error("Completed image build did not return an image ID"); + } + return build.imageId; +}; + +export const isTerminalImageBuildStatus = (status: SandboxImageBuild["status"]): boolean => + status === "completed" || status === "failed" || status === "canceled"; diff --git a/src/sandbox/image-build/tar.ts b/src/sandbox/image-build/tar.ts new file mode 100644 index 0000000..9f92c52 --- /dev/null +++ b/src/sandbox/image-build/tar.ts @@ -0,0 +1,426 @@ +/** + * Minimal tar support: a PAX-format writer that mirrors CPython's `tarfile` + * output for the normalized entries used in build contexts, and a streaming + * reader for `docker image save` archives. + */ + +import type { Readable } from "stream"; + +const BLOCK_SIZE = 512; +const RECORD_SIZE = BLOCK_SIZE * 20; +const POSIX_MAGIC = Buffer.from("ustar\x0000", "binary"); +const NUL = Buffer.alloc(1); + +export type TarEntryType = "file" | "directory" | "symlink"; + +export interface TarEntryInfo { + /** Archive path; directories carry a trailing slash. */ + name: string; + type: TarEntryType; + mode: number; + size: number; + linkname: string; +} + +const TYPE_FLAGS: Record = { + file: "0", + directory: "5", + symlink: "2", +}; + +// Tar header validation intentionally includes NUL. +// eslint-disable-next-line no-control-regex +const isAscii = (value: string): boolean => /^[\x00-\x7f]*$/.test(value); + +/** Python `stn`: encode with ASCII "replace" errors, then NUL-pad or truncate. */ +const stringField = (value: string, length: number): Buffer => { + // eslint-disable-next-line no-control-regex + const encoded = Buffer.from(value.replace(/[^\x00-\x7f]/g, "?"), "latin1"); + const field = Buffer.alloc(length); + encoded.copy(field, 0, 0, Math.min(encoded.length, length)); + return field; +}; + +/** Python `itn` for values that fit the octal field. */ +const numberField = (value: number, digits: number): Buffer => { + if (value < 0 || value >= 8 ** (digits - 1)) { + throw new RangeError(`tar header value ${value} does not fit in ${digits} digits`); + } + return Buffer.concat([Buffer.from(value.toString(8).padStart(digits - 1, "0"), "ascii"), NUL]); +}; + +interface HeaderFields { + name: string; + mode?: number; + size: number; + type: string; + linkname?: string; +} + +const createHeader = (fields: HeaderFields): Buffer => { + const parts = [ + stringField(fields.name, 100), + numberField((fields.mode ?? 0) & 0o7777, 8), + numberField(0, 8), + numberField(0, 8), + numberField(fields.size, 12), + numberField(0, 12), + Buffer.from(" ", "ascii"), + Buffer.from(fields.type, "ascii"), + stringField(fields.linkname ?? "", 100), + POSIX_MAGIC, + stringField("", 32), + stringField("", 32), + numberField(0, 8), + numberField(0, 8), + stringField("", 155), + ]; + const header = Buffer.alloc(BLOCK_SIZE); + Buffer.concat(parts).copy(header); + let checksum = 0; + for (const byte of header) { + checksum += byte; + } + Buffer.from(checksum.toString(8).padStart(6, "0") + "\0 ", "ascii").copy(header, 148); + return header; +}; + +const padToBlock = (size: number): Buffer => { + const remainder = size % BLOCK_SIZE; + return remainder === 0 ? Buffer.alloc(0) : Buffer.alloc(BLOCK_SIZE - remainder); +}; + +const createPaxHeader = (records: Array<[string, string]>): Buffer => { + const encodedRecords: Buffer[] = []; + for (const [keyword, value] of records) { + const key = Buffer.from(keyword, "utf8"); + const data = Buffer.from(value, "utf8"); + const base = key.length + data.length + 3; + let n = 0; + let p = 0; + while (true) { + n = base + String(p).length; + if (n === p) { + break; + } + p = n; + } + encodedRecords.push(Buffer.from(`${p} `, "ascii"), key, Buffer.from("=", "ascii"), data); + encodedRecords.push(Buffer.from("\n", "ascii")); + } + const payload = Buffer.concat(encodedRecords); + return Buffer.concat([ + createHeader({ name: "././@PaxHeader", size: payload.length, type: "x" }), + payload, + padToBlock(payload.length), + ]); +}; + +export const createTarEntryHeader = (info: TarEntryInfo): Buffer => { + const pax: Array<[string, string]> = []; + if (!isAscii(info.name) || info.name.length > 100) { + pax.push(["path", info.name]); + } + if (!isAscii(info.linkname) || info.linkname.length > 100) { + pax.push(["linkpath", info.linkname]); + } + let size = info.size; + if (size < 0 || size >= 8 ** 11) { + pax.push(["size", String(info.size)]); + size = 0; + } + const header = createHeader({ + name: info.name, + mode: info.mode, + size, + type: TYPE_FLAGS[info.type], + linkname: info.linkname, + }); + return pax.length > 0 ? Buffer.concat([createPaxHeader(pax), header]) : header; +}; + +export type TarSink = (chunk: Buffer) => Promise; + +/** Sequential PAX tar writer; `close` writes the end-of-archive blocks. */ +export class PaxTarWriter { + private written = 0; + + constructor(private readonly sink: TarSink) {} + + private async emit(chunk: Buffer): Promise { + if (chunk.length === 0) { + return; + } + this.written += chunk.length; + await this.sink(chunk); + } + + async addEntry(info: TarEntryInfo, content?: AsyncIterable): Promise { + await this.emit(createTarEntryHeader(info)); + if (info.type !== "file") { + return; + } + let copied = 0; + if (content) { + for await (const chunk of content) { + copied += chunk.length; + if (copied > info.size) { + throw new Error(`tar entry "${info.name}" produced more data than its declared size`); + } + await this.emit(chunk); + } + } + if (copied !== info.size) { + throw new Error(`tar entry "${info.name}" was truncated`); + } + await this.emit(padToBlock(info.size)); + } + + async close(): Promise { + await this.emit(Buffer.alloc(BLOCK_SIZE * 2)); + const remainder = this.written % RECORD_SIZE; + if (remainder !== 0) { + await this.emit(Buffer.alloc(RECORD_SIZE - remainder)); + } + } +} + +class ByteReader { + private readonly iterator: AsyncIterator; + private pending: Buffer = Buffer.alloc(0); + private done = false; + + constructor(source: AsyncIterable) { + const raw = source[Symbol.asyncIterator](); + this.iterator = { + next: async () => { + const result = await raw.next(); + return result.done + ? { done: true, value: undefined as unknown as Buffer } + : { + done: false, + value: Buffer.isBuffer(result.value) ? result.value : Buffer.from(result.value), + }; + }, + }; + } + + private async fill(): Promise { + if (this.done) { + return false; + } + const result = await this.iterator.next(); + if (result.done) { + this.done = true; + return false; + } + this.pending = this.pending.length ? Buffer.concat([this.pending, result.value]) : result.value; + return true; + } + + /** Read exactly `length` bytes, or return null at a clean end of stream. */ + async readExact(length: number): Promise { + while (this.pending.length < length) { + if (!(await this.fill())) { + if (this.pending.length === 0) { + return null; + } + throw new Error("unexpected end of tar stream"); + } + } + const chunk = this.pending.subarray(0, length); + this.pending = this.pending.subarray(length); + return chunk; + } + + async *readChunks(length: number): AsyncGenerator { + let remaining = length; + while (remaining > 0) { + if (this.pending.length === 0 && !(await this.fill())) { + throw new Error("unexpected end of tar stream"); + } + const take = Math.min(remaining, this.pending.length); + const chunk = this.pending.subarray(0, take); + this.pending = this.pending.subarray(take); + remaining -= take; + yield chunk; + } + } + + async skip(length: number): Promise { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _chunk of this.readChunks(length)) { + // discard + } + } +} + +export interface TarStreamEntry { + name: string; + size: number; + typeflag: string; + mode: number; + linkname: string; + isFile: boolean; + /** Stream the entry contents. Must be consumed before advancing. */ + content(): AsyncGenerator; +} + +const parseOctal = (field: Buffer): number => { + if (field.length > 0 && field[0] & 0x80) { + let value = 0n; + for (let index = 0; index < field.length; index += 1) { + value = (value << 8n) | BigInt(index === 0 ? field[0] & 0x7f : field[index]); + } + return Number(value); + } + const text = field.toString("ascii").replace(/\0.*$/s, "").trim(); + if (text === "") { + return 0; + } + if (!/^[0-7]+$/.test(text)) { + throw new Error("invalid tar header number field"); + } + return parseInt(text, 8); +}; + +const parseString = (field: Buffer): string => { + const end = field.indexOf(0); + return (end === -1 ? field : field.subarray(0, end)).toString("utf8"); +}; + +const parsePaxRecords = (payload: Buffer): Map => { + const records = new Map(); + let offset = 0; + while (offset < payload.length) { + const space = payload.indexOf(0x20, offset); + if (space === -1) { + throw new Error("invalid PAX header record"); + } + const length = parseInt(payload.subarray(offset, space).toString("ascii"), 10); + if (!Number.isInteger(length) || length <= 0 || offset + length > payload.length) { + throw new Error("invalid PAX header record"); + } + const record = payload.subarray(space + 1, offset + length - 1).toString("utf8"); + const separator = record.indexOf("="); + if (separator === -1) { + throw new Error("invalid PAX header record"); + } + records.set(record.slice(0, separator), record.slice(separator + 1)); + offset += length; + } + return records; +}; + +/** + * Iterate over tar entries from a stream. Callers must fully consume an + * entry's content before advancing; unread content is skipped. + */ +export async function* readTarEntries(source: Readable): AsyncGenerator { + const reader = new ByteReader(source); + const globalPax = new Map(); + let paxOverrides: Map | null = null; + let gnuLongName: string | null = null; + let gnuLongLink: string | null = null; + + while (true) { + const header = await reader.readExact(BLOCK_SIZE); + if (header === null || header.every((byte) => byte === 0)) { + return; + } + const typeflag = header.subarray(156, 157).toString("ascii"); + const size = parseOctal(header.subarray(124, 136)); + if (!Number.isSafeInteger(size) || size < 0) throw new Error("invalid tar entry size"); + if (["x", "g", "L", "K"].includes(typeflag) && size > 16 * 1024 * 1024) throw new Error("tar metadata exceeds the size limit"); + const padded = size + (BLOCK_SIZE - (size % BLOCK_SIZE || BLOCK_SIZE)); + + if (typeflag === "x" || typeflag === "g") { + const payload = await reader.readExact(padded); + if (payload === null) { + throw new Error("unexpected end of tar stream"); + } + if (typeflag === "x") { + paxOverrides = parsePaxRecords(payload.subarray(0, size)); + } else { + for (const [key, value] of parsePaxRecords(payload.subarray(0, size))) { + if (value) globalPax.set(key, value); else globalPax.delete(key); + } + } + continue; + } + if (typeflag === "L" || typeflag === "K") { + const payload = await reader.readExact(padded); + if (payload === null) { + throw new Error("unexpected end of tar stream"); + } + const value = parseString(payload.subarray(0, size)); + if (typeflag === "L") { + gnuLongName = value; + } else { + gnuLongLink = value; + } + continue; + } + + let name = parseString(header.subarray(0, 100)); + const magic = header.subarray(257, 263).toString("binary"); + const prefix = parseString(header.subarray(345, 500)); + if (magic.startsWith("ustar") && prefix && typeflag !== "L") { + name = `${prefix}/${name}`; + } + if (gnuLongName !== null) { + name = gnuLongName; + } + let linkname = parseString(header.subarray(157, 257)); + if (gnuLongLink !== null) { + linkname = gnuLongLink; + } + const mode = parseOctal(header.subarray(100, 108)) & 0o7777; + let entrySize = size; + paxOverrides = new Map([...globalPax, ...(paxOverrides ?? [])]); + if (paxOverrides) { + const path = paxOverrides.get("path"); + if (path !== undefined) { + name = path; + } + const linkpath = paxOverrides.get("linkpath"); + if (linkpath !== undefined) { + linkname = linkpath; + } + const paxSize = paxOverrides.get("size"); + if (paxSize !== undefined) { + entrySize = Number(paxSize); + if (!Number.isSafeInteger(entrySize) || entrySize < 0) { + throw new Error("invalid PAX size record"); + } + } + } + paxOverrides = null; + gnuLongName = null; + gnuLongLink = null; + + let remaining = entrySize; + const content = async function* (): AsyncGenerator { + const total = remaining; + remaining = 0; + yield* reader.readChunks(total); + }; + yield { + name, + size: entrySize, + typeflag, + mode, + linkname, + isFile: typeflag === "0" || typeflag === "\0" || typeflag === "7", + content, + }; + if (remaining > 0) { + await reader.skip(remaining); + remaining = 0; + } + const padding = entrySize % BLOCK_SIZE === 0 ? 0 : BLOCK_SIZE - (entrySize % BLOCK_SIZE); + if (padding > 0) { + await reader.skip(padding); + } + } +} diff --git a/src/sandbox/image-build/upload.ts b/src/sandbox/image-build/upload.ts new file mode 100644 index 0000000..0d1bff6 --- /dev/null +++ b/src/sandbox/image-build/upload.ts @@ -0,0 +1,165 @@ +import fetch from "node-fetch"; +import { createReadStream, statSync } from "fs"; +import { PassThrough } from "stream"; +import { SandboxImageBuildUpload } from "../../types/sandbox"; +import { DockerImageBuildArtifact } from "./artifacts"; +import { sleep } from "./common"; + +class UploadStatusError extends Error { + constructor( + readonly statusCode: number, + body: string + ) { + super(`image artifact upload failed: ${statusCode}: ${body}`.trimEnd()); + this.name = "UploadStatusError"; + } +} + +const uploadOnce = async ( + upload: SandboxImageBuildUpload, + artifactPath: string, + method: string, + timeoutSeconds: number | null | undefined +): Promise => { + const size = statSync(artifactPath).size; + const headers: Record = { ...(upload.headers || {}) }; + if (!Object.keys(headers).some((key) => key.toLowerCase() === "content-length")) { + headers["content-length"] = String(size); + } + + const controller = new AbortController(); + let timer: NodeJS.Timeout | undefined; + const resetTimer = () => { + if (timeoutSeconds === null || timeoutSeconds === undefined) { + return; + } + if (timer) { + clearTimeout(timer); + } + timer = setTimeout(() => controller.abort(), timeoutSeconds * 1000); + }; + + const source = createReadStream(artifactPath); + const body = new PassThrough(); + source.on("data", () => resetTimer()); + source.on("error", (error) => body.destroy(error)); + source.pipe(body); + resetTimer(); + + try { + const response = await fetch(upload.url, { + method, + headers, + body: size === 0 ? undefined : body, + signal: controller.signal, + }); + resetTimer(); + const text = await response.text(); + if (response.ok) { + return; + } + throw new UploadStatusError(response.status, text.trim()); + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + throw new Error(`image artifact upload timed out after ${timeoutSeconds}s of inactivity`); + } + throw error; + } finally { + if (timer) { + clearTimeout(timer); + } + source.destroy(); + } +}; + +export const uploadImageBuildArtifact = async ( + upload: SandboxImageBuildUpload, + artifactPath: string, + options: { timeout?: number | null } = {} +): Promise => { + const artifactSize = statSync(artifactPath).size; + if (upload.maxUploadBytes > 0 && artifactSize > upload.maxUploadBytes) { + throw new Error( + `image artifact exceeds the server upload limit (${artifactSize} > ${upload.maxUploadBytes})` + ); + } + const method = (upload.method || "PUT").trim().toUpperCase(); + const attempts = method === "PUT" ? 3 : 1; + let lastError: unknown; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + await uploadOnce(upload, artifactPath, method, options.timeout); + return; + } catch (error) { + lastError = error; + if (error instanceof UploadStatusError) { + if (error.statusCode !== 408 && error.statusCode !== 429 && error.statusCode < 500) { + throw error; + } + } + } + if (attempt < attempts) { + await sleep(attempt * 0.25); + } + } + throw lastError; +}; + +/** Upload only the artifacts the server requested, verifying each request. */ +export const uploadMissingImageBuildArtifacts = async ( + uploads: SandboxImageBuildUpload[] | undefined, + artifacts: Record, + options: { label: string; timeout?: number | null } +): Promise => { + const requested: Array<[SandboxImageBuildUpload, DockerImageBuildArtifact, string]> = []; + const seen = new Set(); + for (const upload of uploads ?? []) { + const digest = (upload.sha256 || "").trim().toLowerCase(); + const artifact = artifacts[digest]; + if (!digest || artifact === undefined) { + throw new Error(`server requested unknown ${options.label} "${upload.sha256}"`); + } + if (seen.has(digest)) { + throw new Error(`server requested duplicate ${options.label} ${digest}`); + } + seen.add(digest); + const method = (upload.method || "PUT").trim().toUpperCase(); + if (method !== "PUT") { + throw new Error( + `server requested unsupported ${options.label} upload method "${upload.method}"` + ); + } + if (upload.maxUploadBytes > 0 && artifact.sizeBytes > upload.maxUploadBytes) { + throw new Error( + `${options.label} ${digest} exceeds the server upload limit ` + + `(${artifact.sizeBytes} > ${upload.maxUploadBytes})` + ); + } + requested.push([upload, artifact, digest]); + } + if (requested.length === 0) { + return; + } + + let next = 0; + let failed = false; + const worker = async (): Promise => { + while (!failed && next < requested.length) { + const [upload, artifact, digest] = requested[next]; + next += 1; + try { + await uploadImageBuildArtifact(upload, artifact.path, { timeout: options.timeout }); + } catch (error) { + failed = true; + throw new Error( + `upload ${options.label} ${digest}: ${error instanceof Error ? error.message : error}` + ); + } + } + }; + const results = await Promise.allSettled( + Array.from({ length: Math.min(4, requested.length) }, () => worker()) + ); + const failure = results.find((result) => result.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; +}; diff --git a/src/sandbox/index.ts b/src/sandbox/index.ts index 695b70b..3e1957b 100644 --- a/src/sandbox/index.ts +++ b/src/sandbox/index.ts @@ -1,4 +1,5 @@ export { RuntimeTransport } from "./base"; export { SandboxProcessesApi, SandboxProcessHandle } from "./process"; -export { SandboxFilesApi, SandboxWatchDirHandle } from "./files"; +export { SandboxFilesApi, SandboxFileWatchHandle, SandboxWatchDirHandle } from "./files"; export { SandboxTerminalApi, SandboxTerminalConnection, SandboxTerminalHandle } from "./terminal"; +export { SandboxHandle } from "../services/sandboxes"; diff --git a/src/sandbox/process-output.ts b/src/sandbox/process-output.ts new file mode 100644 index 0000000..98508bf --- /dev/null +++ b/src/sandbox/process-output.ts @@ -0,0 +1,156 @@ +/** Collection and validation of the receiver's command event stream. */ + +import { StringDecoder } from "string_decoder"; +import { HyperbrowserError } from "../error"; +import { SandboxProcessOutputEvent, SandboxProcessResult } from "../types/sandbox"; +import { RuntimeSSEEvent } from "./base"; + +export const DEFAULT_MAX_PROCESS_OUTPUT_BYTES = 64 * 1024 * 1024; + +const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/; + +type OutputStream = "stdout" | "stderr" | "system"; + +interface RawOutputEvent { + seq: number; + stream: OutputStream; + data: string; + encoding?: string; + timestamp: number; +} + +export interface RawProcessResult extends Partial { + id: string; + status: SandboxProcessResult["status"]; + exit_code?: number | null; + stdout: string; + stderr: string; + started_at: number; + completed_at?: number | null; + error?: string | null; + output_truncated?: boolean; + last_seq?: number | null; +} + +export const normalizeProcessResult = (result: RawProcessResult): SandboxProcessResult => ({ + id: result.id, + status: result.status, + exitCode: result.exit_code !== undefined ? result.exit_code : result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + startedAt: result.started_at ?? result.startedAt!, + completedAt: result.completed_at !== undefined ? result.completed_at : result.completedAt, + error: result.error, + outputTruncated: result.output_truncated, + lastSeq: result.last_seq, +}); + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +export class ProcessOutput { + size = 0; + seq = 0; + readonly events: SandboxProcessOutputEvent[] = []; + result: SandboxProcessResult | null = null; + error: HyperbrowserError | null = null; + private readonly chunks: { stdout: string[]; stderr: string[] } = { stdout: [], stderr: [] }; + private readonly decoders: Record = { + stdout: new StringDecoder("utf8"), + stderr: new StringDecoder("utf8"), + system: new StringDecoder("utf8"), + }; + + constructor( + readonly processId: string, + readonly maxBytes: number + ) {} + + failure(message: string, code = "incomplete_output"): HyperbrowserError { + return new HyperbrowserError(message, { + code, + service: "runtime", + retryable: false, + details: { process_id: this.processId, last_seq: this.seq }, + }); + } + + consume(event: RuntimeSSEEvent): void { + if (event.event === "output") { + this.consumeOutput(event.data); + return; + } + if (event.event === "done") { + this.consumeDone(event.data); + return; + } + if (event.event === "error") { + const data = isRecord(event.data) ? event.data : {}; + throw this.failure( + String(data.error ?? "Command stream failed"), + typeof data.code === "string" ? data.code : "incomplete_output" + ); + } + } + + private decodePayload(data: RawOutputEvent): Buffer { + if (data.encoding === "base64") { + const text = data.data; + if (text.length % 4 !== 0 || !BASE64_PATTERN.test(text)) { + throw this.failure("Command output contains invalid base64 data"); + } + return Buffer.from(text, "base64"); + } + return Buffer.from(data.data, "utf8"); + } + + private consumeOutput(payload: unknown): void { + if (!isRecord(payload)) { + throw this.failure("Command output event is malformed"); + } + const data = payload as unknown as RawOutputEvent; + if (data.seq !== this.seq + 1) { + throw this.failure("Command output contains a sequence gap"); + } + const stream = data.stream; + if (!Object.prototype.hasOwnProperty.call(this.decoders, stream)) { + throw this.failure("Unknown command output stream"); + } + const raw = this.decodePayload(data); + this.size += raw.length; + if (this.size > this.maxBytes) { + throw this.failure( + "Command output exceeds maxOutputBytes; increase the collection limit or disconnect a detached process", + "output_limit_exceeded" + ); + } + this.seq = data.seq; + const text = this.decoders[stream].write(raw); + this.chunks[stream === "stdout" ? "stdout" : "stderr"].push(text); + this.events.push({ type: stream, seq: this.seq, data: text, timestamp: data.timestamp }); + } + + private consumeDone(payload: unknown): void { + if (!isRecord(payload)) { + throw this.failure("Command completion event is malformed"); + } + const data = payload as unknown as RawProcessResult; + if (data.last_seq !== this.seq || data.output_truncated) { + throw this.failure("Receiver reported incomplete command output"); + } + for (const stream of Object.keys(this.decoders) as OutputStream[]) { + this.chunks[stream === "stdout" ? "stdout" : "stderr"].push(this.decoders[stream].end()); + } + this.result = normalizeProcessResult({ + ...data, + stdout: this.chunks.stdout.join(""), + stderr: this.chunks.stderr.join(""), + }); + } +} + +export const validateOutputLimit = (value: unknown): void => { + if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) { + throw new HyperbrowserError("maxOutputBytes must be a positive integer"); + } +}; diff --git a/src/sandbox/process.ts b/src/sandbox/process.ts index 206d876..246802b 100644 --- a/src/sandbox/process.ts +++ b/src/sandbox/process.ts @@ -1,4 +1,12 @@ -import { RuntimeSSEEvent, RuntimeTransport } from "./base"; +import { HyperbrowserError } from "../error"; +import { RuntimeSSEEvent, RuntimeSSEStream, RuntimeTransport } from "./base"; +import { + DEFAULT_MAX_PROCESS_OUTPUT_BYTES, + normalizeProcessResult, + ProcessOutput, + RawProcessResult, + validateOutputLimit, +} from "./process-output"; import { SandboxExecParams, SandboxExecOptions, @@ -25,35 +33,16 @@ interface ProcessListWireResponse { next_cursor?: string; } -interface RawProcessSummary { +interface RawProcessSummary extends Partial { id: string; status: SandboxProcessSummary["status"]; command: string; - args?: string[]; + args?: string[] | null; cwd: string; - pid?: number; + pid?: number | null; exit_code?: number | null; started_at: number; - completed_at?: number; -} - -interface RawProcessResult { - id: string; - status: SandboxProcessResult["status"]; - exit_code?: number | null; - stdout: string; - stderr: string; - started_at: number; - completed_at?: number; - error?: string; -} - -interface ExecResponse { - result: RawProcessResult; -} - -interface StartProcessResponse { - process: RawProcessSummary; + completed_at?: number | null; } const DEFAULT_PROCESS_KILL_WAIT_MS = 5_000; @@ -66,20 +55,9 @@ const normalizeProcessSummary = (process: RawProcessSummary): SandboxProcessSumm args: process.args, cwd: process.cwd, pid: process.pid, - exitCode: process.exit_code, - startedAt: process.started_at, - completedAt: process.completed_at, -}); - -const normalizeProcessResult = (result: RawProcessResult): SandboxProcessResult => ({ - id: result.id, - status: result.status, - exitCode: result.exit_code, - stdout: result.stdout, - stderr: result.stderr, - startedAt: result.started_at, - completedAt: result.completed_at, - error: result.error, + exitCode: process.exit_code !== undefined ? process.exit_code : process.exitCode, + startedAt: process.started_at ?? process.startedAt!, + completedAt: process.completed_at !== undefined ? process.completed_at : process.completedAt, }); const normalizeResultToSummary = (result: SandboxProcessResult): SandboxProcessSummary => ({ @@ -142,6 +120,46 @@ const normalizeLegacyProcessParams = (input: SandboxExecParams): SandboxExecPara useShell: undefined, }); +class ChangeSignal { + private waiters: Array<() => void> = []; + + notify(): void { + const waiters = this.waiters; + this.waiters = []; + waiters.forEach((resolve) => resolve()); + } + + /** Resolve true on the next notification, or false once `timeoutMs` elapses. */ + wait(timeoutMs?: number): Promise { + return new Promise((resolve) => { + let timer: NodeJS.Timeout | undefined; + const onChange = () => { + if (timer) { + clearTimeout(timer); + } + resolve(true); + }; + this.waiters.push(onChange); + if (timeoutMs !== undefined) { + timer = setTimeout(() => { + this.waiters = this.waiters.filter((waiter) => waiter !== onChange); + resolve(false); + }, timeoutMs); + } + }); + } +} + +const localWaitTimeoutMs = (params: SandboxProcessWaitParams): number | undefined => { + if (params.timeoutSec !== undefined && params.timeoutSec > 0) { + return params.timeoutSec * 1000; + } + if (params.timeoutMs !== undefined && params.timeoutMs > 0) { + return params.timeoutMs; + } + return undefined; +}; + const buildProcessPayload = (input: SandboxExecParams) => ({ command: input.command, cwd: input.cwd, @@ -185,11 +203,86 @@ const encodeStdinPayload = (input: SandboxProcessStdinParams) => { }; export class SandboxProcessHandle { + private output: ProcessOutput | null = null; + private collector: Promise | null = null; + private closeStream: (() => void) | null = null; + private readonly changed = new ChangeSignal(); + constructor( private readonly transport: RuntimeTransport, private summary: SandboxProcessSummary ) {} + /** @internal Collect output from the start-stream that created this process. */ + attachCollector(stream: RuntimeSSEStream, maxOutputBytes: number): void { + this.output = new ProcessOutput(this.id, maxOutputBytes); + this.closeStream = stream.close; + this.collector = this.collect(stream.events); + } + + private async collect(events: AsyncGenerator): Promise { + const output = this.output as ProcessOutput; + try { + for await (const event of events) { + if (output.error !== null) { + return; + } + output.consume(event); + this.changed.notify(); + if (output.result !== null) { + return; + } + } + output.error = output.failure("Command stream ended before its completion event"); + } catch (error) { + if (output.error === null) { + output.error = + error instanceof HyperbrowserError + ? error + : output.failure(error instanceof Error ? error.message : String(error)); + } + } finally { + try { + await events.return(undefined); + } catch (error) { + if (output.result === null && output.error === null) { + output.error = output.failure(error instanceof Error ? error.message : String(error)); + } + } + this.closeStream?.(); + this.changed.notify(); + } + } + + private collectedResult(): SandboxProcessResult { + const output = this.output as ProcessOutput; + if (output.error !== null) { + throw output.error; + } + if (output.result === null) { + throw output.failure("Command stream ended before its completion event"); + } + const result = output.result; + this.summary = { + ...this.summary, + status: result.status, + exitCode: result.exitCode, + completedAt: result.completedAt, + }; + return result; + } + + /** Stop collecting output; the detached command continues running. */ + disconnect(): void { + if (this.output !== null) { + if (this.output.result === null && this.output.error === null) { + this.output.error = this.output.failure("Command output collection disconnected"); + } + this.changed.notify(); + this.closeStream?.(); + } + } + get id(): string { return this.summary.id; } @@ -210,7 +303,29 @@ export class SandboxProcessHandle { return this; } + /** + * Wait for completion. Processes started by this client resolve from the + * collected stream; a local timeout rejects without stopping collection. + */ async wait(params: SandboxProcessWaitParams = {}): Promise { + if (this.output !== null && this.collector !== null) { + const output = this.output; + const timeoutMs = localWaitTimeoutMs(params); + const deadline = timeoutMs === undefined ? undefined : Date.now() + timeoutMs; + while (output.result === null && output.error === null) { + const remaining = deadline === undefined ? undefined : deadline - Date.now(); + const ready = + remaining !== undefined && remaining <= 0 ? false : await this.changed.wait(remaining); + if (!ready) { + throw new HyperbrowserError("Timed out waiting for command output", { + code: "wait_timeout", + service: "runtime", + retryable: false, + }); + } + } + return this.collectedResult(); + } const response = await this.transport.requestJSON( `/sandbox/processes/${this.id}/wait`, { @@ -225,6 +340,11 @@ export class SandboxProcessHandle { } ); const result = normalizeProcessResult(response.result); + if (result.outputTruncated) { + throw new ProcessOutput(this.id, 0).failure( + "Retained process output is incomplete; collect output from process start" + ); + } this.summary = { ...this.summary, ...normalizeResultToSummary(result), @@ -291,7 +411,33 @@ export class SandboxProcessHandle { }); } + /** + * Replay and follow output. Processes started by this client stream from the + * collected events; other handles attach to the receiver's stream endpoint. + */ async *stream(fromSeq?: number): AsyncGenerator { + if (this.output !== null) { + const output = this.output; + let index = 0; + for (;;) { + const events = output.events.slice(index); + index += events.length; + const done = output.result !== null || output.error !== null; + if (events.length === 0 && !done) { + await this.changed.wait(); + continue; + } + for (const event of events) { + if (fromSeq === undefined || event.seq >= fromSeq) { + yield event; + } + } + if (done) { + yield { type: "exit", result: this.collectedResult() }; + return; + } + } + } const params = fromSeq && fromSeq > 0 ? { @@ -318,24 +464,27 @@ export class SandboxProcessHandle { export class SandboxProcessesApi { constructor(private readonly transport: RuntimeTransport) {} + /** Run a command to completion, collecting its full output from process start. */ async exec(command: string, options?: SandboxExecOptions): Promise; async exec(input: SandboxExecParams): Promise; async exec( input: string | SandboxExecParams, options?: SandboxExecOptions ): Promise { - const params = normalizeExecParams(input, options); - const response = await this.transport.requestJSON("/sandbox/exec", { - method: "POST", - body: JSON.stringify(buildProcessPayload(params)), - headers: { - "content-type": "application/json", - }, - }); - - return normalizeProcessResult(response.result); + const handle = + typeof input === "string" ? await this.start(input, options) : await this.start(input); + try { + return await handle.wait(); + } finally { + handle.disconnect(); + } } + /** + * Start a process and collect its output in the background from the same + * streamed request, so output is complete even beyond the receiver's replay + * limit. Call `disconnect()` to stop collecting without killing the process. + */ async start(command: string, options?: SandboxExecOptions): Promise; async start(input: SandboxExecParams): Promise; async start( @@ -343,15 +492,33 @@ export class SandboxProcessesApi { options?: SandboxExecOptions ): Promise { const params = normalizeExecParams(input, options); - const response = await this.transport.requestJSON("/sandbox/processes", { + const maxOutputBytes = params.maxOutputBytes ?? DEFAULT_MAX_PROCESS_OUTPUT_BYTES; + validateOutputLimit(maxOutputBytes); + const stream = await this.transport.openSSE("/sandbox/processes", undefined, { method: "POST", body: JSON.stringify(buildProcessPayload(params)), - headers: { - "content-type": "application/json", - }, + signal: params.signal, }); - - return new SandboxProcessHandle(this.transport, normalizeProcessSummary(response.process)); + let handle: SandboxProcessHandle; + try { + const started = await stream.events.next(); + if (started.done || started.value.event !== "started") { + throw new HyperbrowserError("Expected process start event", { + service: "runtime", + retryable: false, + }); + } + handle = new SandboxProcessHandle( + this.transport, + normalizeProcessSummary(started.value.data as RawProcessSummary) + ); + } catch (error) { + stream.close(); + await stream.events.return(undefined).catch(() => undefined); + throw error; + } + handle.attachCollector(stream, maxOutputBytes); + return handle; } async get(processId: string): Promise { diff --git a/src/sandbox/terminal.ts b/src/sandbox/terminal.ts index 1446679..3750ed7 100644 --- a/src/sandbox/terminal.ts +++ b/src/sandbox/terminal.ts @@ -17,17 +17,17 @@ interface PTYStatusResponse { interface RawPTYStatus { id: string; command: string; - args?: string[]; + args?: string[] | null; cwd: string; - pid?: number; + pid?: number | null; running: boolean; exitCode?: number | null; - error?: string; + error?: string | null; timedOut?: boolean; rows: number; cols: number; startedAt: number; - finishedAt?: number; + finishedAt?: number | null; output?: RawPTYOutput[]; } @@ -107,10 +107,12 @@ export class SandboxTerminalConnection { return; } - this.eventsQueue.push({ - type: "exit", - status: normalizeTerminalStatus(parsed.status), - }); + if (parsed.type === "exit") { + this.eventsQueue.push({ + type: "exit", + status: normalizeTerminalStatus(parsed.status), + }); + } } catch (error) { this.eventsQueue.fail(error); } @@ -290,7 +292,9 @@ export class SandboxTerminalHandle { const ws = await openRuntimeWebSocket(target, headers); - return new SandboxTerminalConnection(ws); + const connection = new SandboxTerminalConnection(ws); + ws.resume?.(); + return connection; } } diff --git a/src/sandbox/ws.ts b/src/sandbox/ws.ts index 61f83d2..241ec22 100644 --- a/src/sandbox/ws.ts +++ b/src/sandbox/ws.ts @@ -1,6 +1,6 @@ import type { IncomingMessage } from "http"; import WebSocket from "ws"; -import { HyperbrowserError } from "../client"; +import { HyperbrowserError } from "../error"; import { runtimeBaseUrlSessionId } from "./runtime-path"; export class AsyncEventQueue implements AsyncIterable { @@ -252,31 +252,56 @@ const buildHandshakeError = async (response: IncomingMessage): Promise + headers: Record, + options: { signal?: AbortSignal; timeoutMs?: number } = {} ): Promise => new Promise((resolve, reject) => { let settled = false; - + let response: IncomingMessage | undefined; + if (options.signal?.aborted) { + reject(new HyperbrowserError("Runtime websocket request canceled", { + code: "request_aborted", service: "runtime", retryable: false, + })); + return; + } const socket = new WebSocket(target.url, { headers }); - + const cleanup = () => { + clearTimeout(timer); + options.signal?.removeEventListener("abort", abort); + }; const rejectOnce = (error: unknown) => { if (settled) { return; } settled = true; + cleanup(); + response?.destroy(); + socket.terminate(); reject(normalizeWebSocketError(error)); }; + const abort = () => rejectOnce(new HyperbrowserError("Runtime websocket request canceled", { + code: "request_aborted", service: "runtime", retryable: false, + })); + const timer = setTimeout(() => rejectOnce(new HyperbrowserError("Runtime websocket handshake timed out", { + code: "request_timeout", service: "runtime", retryable: true, + })), options.timeoutMs ?? 30_000); + timer.unref?.(); + options.signal?.addEventListener("abort", abort, { once: true }); socket.once("open", () => { if (settled) { return; } settled = true; + cleanup(); + // Frames can arrive with the upgrade response, before an awaiting caller resumes. + socket.pause(); resolve(socket); }); - socket.once("unexpected-response", (_request, response) => { - void buildHandshakeError(response).then(rejectOnce).catch(rejectOnce); + socket.once("unexpected-response", (_request, incoming) => { + response = incoming; + void buildHandshakeError(incoming).then(rejectOnce).catch(rejectOnce); }); socket.once("error", rejectOnce); diff --git a/src/services/base.ts b/src/services/base.ts index 95c1aad..0c59370 100644 --- a/src/services/base.ts +++ b/src/services/base.ts @@ -1,32 +1,17 @@ import fetch, { HeadersInit, RequestInit, Response } from "node-fetch"; -import { HyperbrowserError } from "../client"; - -const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504]); -const RETRYABLE_NETWORK_CODES = new Set([ - "ECONNRESET", - "ECONNREFUSED", - "EAI_AGAIN", - "ETIMEDOUT", - "ESOCKETTIMEDOUT", -]); +import { errorRequestPath, HyperbrowserError, networkErrorMessage } from "../error"; +import { + getRetryDelayMs, + isRetryableNetworkError, + RETRYABLE_STATUS_CODES, + retryDelay, + shouldRetryGet, +} from "../retry"; const getRequestId = (response: Response): string | undefined => { return response.headers.get("x-request-id") || response.headers.get("request-id") || undefined; }; -const isRetryableNetworkError = (error: unknown): boolean => { - if (!(error instanceof Error)) { - return false; - } - - const networkError = error as Error & { code?: string; type?: string }; - return ( - networkError.name === "AbortError" || - networkError.type === "request-timeout" || - (networkError.code ? RETRYABLE_NETWORK_CODES.has(networkError.code) : false) - ); -}; - export class BaseService { constructor( protected readonly apiKey: string, @@ -34,12 +19,39 @@ export class BaseService { protected readonly timeout: number = 30000 ) {} + /** Transient GET failures (429/502/503/504, network errors) retry up to three attempts. */ protected async request( path: string, init?: RequestInit, params?: Record, fullUrl: boolean = false ): Promise { + const method = (init?.method ?? "GET").toUpperCase(); + let failedAttempt = 1; + for (;;) { + try { + return await this.requestOnce(path, init, params, fullUrl); + } catch (error) { + if ( + init?.signal?.aborted || + !(error instanceof HyperbrowserError) || + !shouldRetryGet(method, error, failedAttempt) + ) { + throw error; + } + await retryDelay(getRetryDelayMs(failedAttempt), init?.signal ?? undefined); + failedAttempt += 1; + } + } + } + + private async requestOnce( + path: string, + init?: RequestInit, + params?: Record, + fullUrl: boolean = false + ): Promise { + let response: Response | undefined; try { const url = new URL(fullUrl ? path : `${this.baseUrl}/api${path}`); @@ -64,7 +76,7 @@ export class BaseService { const requestTimeout = init?.timeout ?? this.timeout; - const response = await fetch(url.toString(), { + response = await fetch(url.toString(), { ...init, timeout: requestTimeout, headers: { @@ -86,7 +98,8 @@ export class BaseService { errorCode = typeof errorData?.code === "string" ? errorData.code : undefined; errorMessage = errorData.message || errorData.error || `HTTP error! status: ${response.status}`; - } catch { + } catch (error) { + if (init?.signal?.aborted) throw error; errorMessage = `HTTP error! status: ${response.status}`; } throw new HyperbrowserError(errorMessage, { @@ -96,21 +109,24 @@ export class BaseService { retryable: RETRYABLE_STATUS_CODES.has(response.status), service: "control", details: errorDetails, + method: init?.method ?? "GET", + path: errorRequestPath(path), }); } - if (response.headers.get("content-length") === "0") { - return {} as T; - } - + const text = await response.text(); + if (!text) return {} as T; try { - return (await response.json()) as T; - } catch { + return JSON.parse(text) as T; + } catch (cause) { throw new HyperbrowserError("Failed to parse JSON response", { statusCode: response.status, requestId: getRequestId(response), retryable: false, service: "control", + cause, + method: init?.method ?? "GET", + path: errorRequestPath(path), }); } } catch (error) { @@ -119,11 +135,16 @@ export class BaseService { } throw new HyperbrowserError( - error instanceof Error ? error.message : "Unknown error occurred", + networkErrorMessage(error, "Unknown error occurred"), { - retryable: isRetryableNetworkError(error), + code: init?.signal?.aborted ? "request_aborted" : undefined, + retryable: !init?.signal?.aborted && isRetryableNetworkError(error), + method: init?.method ?? "GET", + path: errorRequestPath(path), service: "control", cause: error, + statusCode: response?.status, + requestId: response ? getRequestId(response) : undefined, } ); } diff --git a/src/services/normalize.ts b/src/services/normalize.ts new file mode 100644 index 0000000..339ea77 --- /dev/null +++ b/src/services/normalize.ts @@ -0,0 +1,18 @@ +import { HyperbrowserError } from "../error"; + +/** Nullable numeric wire values used by sandbox and volume endpoints. */ +export const optionalNumber = (value: unknown, integer = false): number | null | undefined => { + if (value === undefined || value === null) return value; + if (typeof value === "string" && value.trim() === "") return null; + const parsed = typeof value === "string" ? Number(value) : value; + if ( + typeof parsed !== "number" || + !Number.isFinite(parsed) || + (integer && !Number.isSafeInteger(parsed)) + ) { + throw new HyperbrowserError("Invalid numeric value in API response", { service: "control" }); + } + return parsed; +}; +export const optionalInteger = (value: unknown): number | null | undefined => + optionalNumber(value, true); diff --git a/src/services/sandboxes.ts b/src/services/sandboxes.ts index fe76b4a..b211db5 100644 --- a/src/services/sandboxes.ts +++ b/src/services/sandboxes.ts @@ -1,14 +1,42 @@ -import { HyperbrowserError } from "../client"; +import { HyperbrowserError } from "../error"; import { SandboxFilesApi } from "../sandbox/files"; import { RuntimeConnection, RuntimeTransport } from "../sandbox/base"; import { runtimeSessionIdFromPath } from "../sandbox/runtime-path"; import { SandboxProcessHandle, SandboxProcessesApi } from "../sandbox/process"; import { SandboxTerminalApi } from "../sandbox/terminal"; +import { + buildDockerImageFromDockerfile, + completedImageId, + dockerBuildContextFingerprint, + dockerImageDigest, + DockerImageBuildArtifact, + IMAGE_BUILD_SOURCE_PLATFORM, + imageBuildName, + makeTempDockerTag, + matchingImageBuild, + mergeImageInit, + packageDockerBuildContextManifest, + packageDockerImageManifest, + prepareDockerImageManifestSource, + removeDockerImage, + uploadMissingImageBuildArtifacts, +} from "../sandbox/image-build"; import { BasicResponse } from "../types/session"; import { + BuildSandboxImageFromDockerImageOptions, + BuildSandboxImageFromDockerfileOptions, CompleteSandboxImageBuildParams, CreateSandboxImageBuildParams, CreateSandboxParams, + StartSandboxFromSnapshotParams, + SandboxRuntimeSession, + SandboxRuntimeTarget, + GetOrBuildSandboxImageOptions, + ReuseSandboxDockerImageParams, + SandboxDockerImageReuseResult, + SandboxImageBuildResolution, + SandboxImageBuildWaitOptions, + SandboxImageSummary, Sandbox, SandboxDetail, SandboxExposeParams, @@ -35,6 +63,8 @@ import { SandboxSnapshotSummary, SandboxUnexposeResult, } from "../types/sandbox"; +import { retryDelay } from "../retry"; +import { optionalInteger, optionalNumber } from "./normalize"; import { BaseService } from "./base"; const RUNTIME_SESSION_REFRESH_BUFFER_MS = 60_000; @@ -59,9 +89,18 @@ const normalizeSandbox = (sandbox: WireSandbox): Sandbox => { const { vcpus, memMiB, diskSizeMiB, ...rest } = sandbox; return { ...rest, - cpu: vcpus, - memoryMiB: memMiB, - diskMiB: diskSizeMiB, + cpu: optionalInteger(vcpus), + memoryMiB: optionalInteger(memMiB), + diskMiB: optionalInteger(diskSizeMiB), + endTime: optionalInteger(rest.endTime), + startTime: optionalInteger(rest.startTime), + dataConsumed: optionalInteger(rest.dataConsumed), + proxyDataConsumed: optionalInteger(rest.proxyDataConsumed), + proxyBytesUsed: optionalInteger(rest.proxyBytesUsed), + timeoutMinutes: optionalInteger(rest.timeoutMinutes), + creditsUsed: optionalNumber(rest.creditsUsed) ?? null, + exposedPorts: rest.exposedPorts ?? [], + network: rest.network ? { ...rest.network, allowOut: rest.network.allowOut ?? [], denyOut: rest.network.denyOut ?? [] } : rest.network, }; }; @@ -69,8 +108,8 @@ const normalizeSandboxDetail = (detail: WireSandboxDetail): SandboxDetail => { const { token, tokenExpiresAt, ...sandbox } = detail; return { ...normalizeSandbox(sandbox), - token, - tokenExpiresAt, + token: token || null, + tokenExpiresAt: tokenExpiresAt || null, }; }; @@ -80,8 +119,33 @@ const normalizeSandboxListResponse = (response: WireSandboxListResponse): Sandbo }); const serializeCreateSandboxParams = (params: CreateSandboxParams): Record => { + if (!params || typeof params !== "object") + throw new HyperbrowserError("Sandbox launch parameters are required"); + for (const name of ["imageName", "snapshotName", "imageId", "snapshotId"] as const) { + const value = params[name]; + if (value !== undefined && (typeof value !== "string" || !value.trim())) + throw new HyperbrowserError(`${name} must be a nonempty string`); + } + if (params.imageId !== undefined && !params.imageName) + throw new HyperbrowserError("imageId requires imageName"); + if (params.snapshotId !== undefined && !params.snapshotName) + throw new HyperbrowserError("snapshotId requires snapshotName"); + if (Boolean(params.imageName) === Boolean(params.snapshotName)) + throw new HyperbrowserError("Provide exactly one start source: snapshotName or imageName"); + for (const name of ["cpu", "memoryMiB", "diskMiB"] as const) { + const value = params[name]; + if (value !== undefined && (!Number.isSafeInteger(value) || value < 1)) + throw new HyperbrowserError(`${name} must be a positive integer`); + } + if ( + params.runtimeClass !== undefined && + params.runtimeClass !== "firecracker" && + params.runtimeClass !== "gvisor-cpu" + ) + throw new HyperbrowserError("Unsupported sandbox runtimeClass"); if (typeof params.imageName === "string") { return { + runtimeClass: params.runtimeClass, imageName: params.imageName, imageId: params.imageId, region: params.region, @@ -116,6 +180,7 @@ const serializeCreateSandboxParams = (params: CreateSandboxParams): Record => new Promise((resolve) => setTimeout(resolve, ms)); + +const validatePagination = (params: { page?: number; limit?: number }, maxLimit?: number): void => { + for (const key of ["page", "limit"] as const) { + const value = params[key]; + if (value !== undefined && (!Number.isSafeInteger(value) || value < 1)) + throw new HyperbrowserError(`${key} must be a positive integer`); + } + if (maxLimit !== undefined && params.limit !== undefined && params.limit > maxLimit) + throw new HyperbrowserError(`limit must be at most ${maxLimit}`); +}; + +const validateImageBuildFormat = (params: { inputFormat?: string; sourcePlatform?: string }): void => { + if (params.sourcePlatform !== undefined && params.sourcePlatform !== "linux/amd64") + throw new HyperbrowserError("sourcePlatform must be linux/amd64"); + if (params.inputFormat !== undefined && ![ + "rootfs_export_tar_gz", "dockerfile_context_tar_gz", + "dockerfile_context_manifest_v1", "docker_image_manifest_v1", + ].includes(params.inputFormat)) throw new HyperbrowserError("Unsupported image build inputFormat"); }; +const serializeCreateImageBuildParams = ( + params: CreateSandboxImageBuildParams +): Record => { + validateImageBuildFormat(params); + for (const key of ["builderCpus", "builderMemoryMiB", "builderScratchMiB"] as const) { + const value = params[key]; + if (value !== undefined && (!Number.isSafeInteger(value) || value < 1)) throw new HyperbrowserError(`${key} must be a positive integer`); + } + return ({ + imageName: params.imageName, + inputSha256: params.inputSha256, + inputSizeBytes: params.inputSizeBytes, + vcpus: params.builderCpus, + memMiB: params.builderMemoryMiB, + scratchMiB: params.builderScratchMiB, + inputFormat: params.inputFormat, + sourcePlatform: params.sourcePlatform, + imageConfigUser: params.imageConfigUser, + imageInit: params.imageInit, + dockerfilePath: params.dockerfilePath, + contextManifest: params.contextManifest, + dockerImageManifest: params.dockerImageManifest, +}); +}; + +const normalizeImageBuildPlatform = (platform: string | undefined): string => { + const normalized = (platform ?? IMAGE_BUILD_SOURCE_PLATFORM).trim().toLowerCase(); + if (normalized !== IMAGE_BUILD_SOURCE_PLATFORM) { + throw new HyperbrowserError(`Image builds require platform '${IMAGE_BUILD_SOURCE_PLATFORM}'`); + } + return normalized; +}; + +type SandboxRuntimeState = SandboxRuntimeSession; + const resolveSandboxRuntimeSessionHost = ( runtime: SandboxDetail["runtime"], baseUrl: URL @@ -294,7 +411,7 @@ export class SandboxHandle { } async expose(params: SandboxExposeParams): Promise { - const exposure = await this.service.expose(this.id, params); + const exposure = await this.service.expose(this.id, params, this.runtime); this.detail = { ...this.detail, exposedPorts: upsertExposedPort(this.detail.exposedPorts ?? [], exposure), @@ -395,6 +512,13 @@ export class SandboxHandle { return expiresAt - Date.now() <= RUNTIME_SESSION_REFRESH_BUFFER_MS; } + async createRuntimeSession( + options: { forceRefresh?: boolean } = {} + ): Promise { + const session = await this.ensureRuntimeSession(options.forceRefresh); + return { ...session, runtime: { ...session.runtime } }; + } + private async ensureRuntimeSession(forceRefresh: boolean = false): Promise { this.assertRuntimeAvailable(); @@ -485,6 +609,30 @@ export class SandboxesService extends BaseService { return this.attach(detail); } + async startFromSnapshot(params: StartSandboxFromSnapshotParams): Promise { + if (!params?.snapshotName) throw new HyperbrowserError("snapshotName is required"); + return this.create(params); + } + + async getRuntimeSession(id: string): Promise { + const detail = await this.getDetail(id); + if (!detail.token || ["closed", "close-error", "error"].includes(detail.status)) { + throw new HyperbrowserError(`Sandbox ${id} is not running`, { + statusCode: 409, + code: "sandbox_not_running", + service: "runtime", + }); + } + return { + sandboxId: id, + status: detail.status, + region: detail.region, + token: detail.token, + tokenExpiresAt: detail.tokenExpiresAt, + runtime: { ...detail.runtime }, + }; + } + async get(id: string): Promise { const detail = await this.getDetail(id); return this.attach(detail); @@ -497,14 +645,15 @@ export class SandboxesService extends BaseService { } async list(params: SandboxListParams = {}): Promise { + validatePagination(params); try { const response = await this.request("/sandboxes", undefined, { status: params.status, start: params.start, end: params.end, search: params.search, - page: params.page, - limit: params.limit, + page: params.page ?? 1, + limit: params.limit ?? 10, }); return normalizeSandboxListResponse(response); } catch (error) { @@ -529,6 +678,7 @@ export class SandboxesService extends BaseService { } async listImages(params: SandboxImageListParams = {}): Promise { + validatePagination(params, 100); try { return await this.request("/images", undefined, { source: params.source, @@ -547,6 +697,7 @@ export class SandboxesService extends BaseService { async listSnapshots( params: SandboxSnapshotListParams = {} ): Promise { + validatePagination(params, 100); try { return await this.request("/snapshots", undefined, { status: params.status, @@ -609,12 +760,22 @@ export class SandboxesService extends BaseService { } } - async expose(id: string, params: SandboxExposeParams): Promise { + async expose( + id: string, + params: SandboxExposeParams, + runtime?: SandboxRuntimeTarget + ): Promise { try { - return await this.request(`/sandbox/${id}/expose`, { + const exposure = await this.request(`/sandbox/${id}/expose`, { method: "POST", body: JSON.stringify(params), }); + if (!exposure.url) + exposure.url = buildSandboxExposedUrl( + runtime ?? (await this.getDetail(id)).runtime, + exposure.port + ); + return exposure; } catch (error) { if (error instanceof HyperbrowserError) { throw error; @@ -657,7 +818,7 @@ export class SandboxesService extends BaseService { try { return await this.request("/images/builds", { method: "POST", - body: JSON.stringify(params), + body: JSON.stringify(serializeCreateImageBuildParams(params)), }); } catch (error) { if (error instanceof HyperbrowserError) { @@ -667,10 +828,14 @@ export class SandboxesService extends BaseService { } } - async getImageBuild(buildId: string): Promise { + async getImageBuild( + buildId: string, + options: { signal?: AbortSignal } = {} + ): Promise { try { const response = await this.request<{ build: SandboxImageBuild }>( - `/images/builds/${encodeURIComponent(buildId)}` + `/images/builds/${encodeURIComponent(buildId)}`, + { signal: options.signal } ); return response.build; } catch (error) { @@ -701,6 +866,7 @@ export class SandboxesService extends BaseService { buildId: string, params: CompleteSandboxImageBuildParams ): Promise { + validateImageBuildFormat(params); try { const response = await this.request<{ build: SandboxImageBuild }>( `/images/builds/${encodeURIComponent(buildId)}/complete`, @@ -733,6 +899,461 @@ export class SandboxesService extends BaseService { } } + async reuseDockerImage( + params: ReuseSandboxDockerImageParams + ): Promise { + validateImageBuildFormat(params); + try { + return await this.request("/images/builds/reuse", { + method: "POST", + body: JSON.stringify(params), + }); + } catch (error) { + if (error instanceof HyperbrowserError) { + throw error; + } + throw new HyperbrowserError("Failed to reuse Docker image"); + } + } + + /** Find an exact ready team image, including revisions awaiting backup. */ + async findReadyImage(imageName: string): Promise { + let page = 1; + for (;;) { + const response = await this.listImages({ + search: imageName, + source: ["team"], + page, + limit: READY_IMAGE_PAGE_SIZE, + }); + for (const image of response.images) { + if (image.imageName === imageName && (image.uploaded || image.ready === true)) { + return image; + } + } + if (response.images.length < READY_IMAGE_PAGE_SIZE) { + return null; + } + if ( + typeof response.totalCount === "number" && + page * READY_IMAGE_PAGE_SIZE >= response.totalCount + ) { + return null; + } + page += 1; + } + } + + /** Poll an image build until it completes, rejecting on failure or timeout. */ + async waitForImageBuild( + buildId: string, + options: SandboxImageBuildWaitOptions = {} + ): Promise { + const pollInterval = options.pollInterval ?? IMAGE_BUILD_DEFAULT_POLL_INTERVAL_SECONDS; + const timeout = + options.timeout === undefined ? IMAGE_BUILD_DEFAULT_WAIT_TIMEOUT_SECONDS : options.timeout; + if ( + !Number.isFinite(pollInterval) || + pollInterval < 0 || + (timeout !== null && (!Number.isFinite(timeout) || timeout < 0)) + ) { + throw new HyperbrowserError( + "Image build wait timeout and interval must be nonnegative finite seconds" + ); + } + const controller = new AbortController(); + let timedOut = false; + const deadlineTimer = + timeout === null + ? undefined + : setTimeout(() => { + timedOut = true; + controller.abort(); + }, timeout * 1000); + const cancel = () => controller.abort(); + options.signal?.addEventListener("abort", cancel, { once: true }); + if (options.signal?.aborted) cancel(); + const waitFailure = (error: unknown): never => { + if (timedOut) + throw new HyperbrowserError(`Timed out waiting for image build ${buildId}`, { + code: "wait_timeout", + service: "control", + cause: error, + }); + throw error; + }; + try { + for (;;) { + const build = await this.getImageBuild(buildId, { signal: controller.signal }).catch(waitFailure); + if (build.status === "completed") return build; + if (build.status === "failed" || build.status === "canceled") { + throw new HyperbrowserError( + build.errorMessage || `Image build ${buildId} ${build.status}`, + { code: build.errorCode || "image_build_failed", service: "control", details: build } + ); + } + await retryDelay(pollInterval * 1000, controller.signal).catch(waitFailure); + } + } finally { + clearTimeout(deadlineTimer); + options.signal?.removeEventListener("abort", cancel); + } + } + + /** Import a local Docker image as reusable layers, reusing exact cached imports. */ + async buildImageFromDockerImage( + options: BuildSandboxImageFromDockerImageOptions + ): Promise { + const platform = normalizeImageBuildPlatform(options.platform); + const source = await prepareDockerImageManifestSource(options.dockerImage, { platform }); + try { + if ( + options.expectedImageDigest !== undefined && + source.imageDigest !== options.expectedImageDigest.toLowerCase() + ) { + throw new HyperbrowserError( + "Docker image changed after its cache identity was computed. " + + "Retry with a fresh image digest." + ); + } + const imageInit = mergeImageInit(source.imageInit, options.imageInit); + const imageConfigUser = options.imageConfigUser ?? source.imageConfigUser; + let reused: SandboxDockerImageReuseResult | null = null; + try { + reused = await this.reuseDockerImage({ + imageName: options.imageName, + sourceImageDigest: source.imageDigest, + sourcePlatform: "linux/amd64", + imageConfigUser, + imageInit, + }); + } catch (error) { + if (!(error instanceof HyperbrowserError) || error.statusCode !== 404) { + throw error; + } + } + if (reused?.hit) { + if (!reused.build) { + throw new HyperbrowserError("exact image cache response is missing its completed build"); + } + return reused.build; + } + + const packaged = await packageDockerImageManifest( + options.dockerImage, + source.imageDigest, + source.config, + { platform, tempDir: options.tempDir } + ); + try { + return await this.submitImageBuild( + { + imageName: options.imageName, + inputSha256: packaged.artifact.sha256Hex, + inputSizeBytes: packaged.artifact.sizeBytes, + inputFormat: packaged.artifact.inputFormat, + sourcePlatform: "linux/amd64", + imageConfigUser, + imageInit, + dockerImageManifest: packaged.manifest, + builderCpus: options.builderCpus, + builderMemoryMiB: options.builderMemoryMiB, + builderScratchMiB: options.builderScratchMiB, + }, + packaged.artifact, + packaged.layers, + "Docker image layer", + options + ); + } finally { + packaged.cleanup(); + } + } finally { + await source.cleanup(); + } + } + + /** + * Build a Dockerfile into a sandbox image. + * + * Remote builds (default) upload only the effective build context and build + * on Hyperbrowser. `remote: false` builds with local Docker and imports the + * resulting image instead. + */ + async buildImageFromDockerfile( + options: BuildSandboxImageFromDockerfileOptions + ): Promise { + options = { ...options, platform: normalizeImageBuildPlatform(options.platform) }; + const remote = options.remote ?? true; + if (remote) { + if ( + options.dockerTag !== undefined || + (options.buildArgs && Object.keys(options.buildArgs).length > 0) + ) { + throw new HyperbrowserError( + "dockerTag and buildArgs require remote: false; remote Dockerfile builds " + + "send the build context to Hyperbrowser" + ); + } + return this.buildImageFromRemoteDockerfile(options); + } + if (options.expectedContextFingerprint !== undefined) { + throw new HyperbrowserError("expectedContextFingerprint requires remote: true"); + } + const tag = options.dockerTag ?? makeTempDockerTag(); + try { + await buildDockerImageFromDockerfile({ + contextPath: options.contextPath, + dockerfile: options.dockerfile, + tag, + platform: options.platform, + buildArgs: options.buildArgs, + }); + return await this.buildImageFromDockerImage({ + dockerImage: tag, + imageName: options.imageName, + platform: options.platform, + imageInit: options.imageInit, + imageConfigUser: options.imageConfigUser, + builderCpus: options.builderCpus, + builderMemoryMiB: options.builderMemoryMiB, + builderScratchMiB: options.builderScratchMiB, + wait: options.wait, + pollInterval: options.pollInterval, + waitTimeout: options.waitTimeout, + signal: options.signal, + tempDir: options.tempDir, + uploadTimeout: options.uploadTimeout, + }); + } finally { + if (options.dockerTag === undefined) { + await removeDockerImage(tag); + } + } + } + + /** + * Reuse, join, or build content-derived remote Dockerfile/image inputs. + * + * Supply exactly one of `contextPath` or `dockerImage`. Names include source + * contents, platform and image initialization overrides. `forceBuild` skips + * ready-image lookup, but joins matching active builds and retains builder + * layer/artifact caches. Canceling polling never cancels the backend build. + * `waitTimeout` applies to this caller's polling, independently of uploads. + * This composes existing APIs; lookup plus creation is not server-atomic. + */ + async getOrBuildImage( + options: GetOrBuildSandboxImageOptions + ): Promise { + const platform = normalizeImageBuildPlatform(options.platform); + const dockerfile = options.dockerfile ?? "Dockerfile"; + const remoteFullContext = options.remoteFullContext ?? false; + const { contextPath, dockerImage } = options; + if ((contextPath === undefined) === (dockerImage === undefined)) { + throw new HyperbrowserError("Supply exactly one of contextPath or dockerImage"); + } + let fingerprint: string; + let source: "dockerfile" | "prebuilt"; + let inputFormat: string; + if (contextPath !== undefined) { + if (options.expectedImageDigest !== undefined) { + throw new HyperbrowserError("expectedImageDigest requires dockerImage"); + } + fingerprint = + options.expectedContextFingerprint ?? + (await dockerBuildContextFingerprint(contextPath, { + dockerfile, + forceFullContext: remoteFullContext, + })); + source = "dockerfile"; + inputFormat = "dockerfile_context_manifest_v1"; + } else { + if ( + options.expectedContextFingerprint !== undefined || + remoteFullContext || + dockerfile !== "Dockerfile" + ) { + throw new HyperbrowserError("Dockerfile context options require contextPath"); + } + fingerprint = + options.expectedImageDigest ?? + (await dockerImageDigest(dockerImage as string, { platform })); + source = "prebuilt"; + inputFormat = "docker_image_manifest_v1"; + } + const imageName = imageBuildName({ + source, + fingerprint, + namePrefix: options.imageNamePrefix, + platform, + imageInit: options.imageInit, + imageConfigUser: options.imageConfigUser, + }); + if (!options.forceBuild) { + const image = await this.findReadyImage(imageName); + if (image !== null) { + return { outcome: "reused", imageName, imageId: image.id }; + } + } + const common = { + imageName, + platform, + imageInit: options.imageInit, + imageConfigUser: options.imageConfigUser, + builderCpus: options.builderCpus, + builderMemoryMiB: options.builderMemoryMiB, + builderScratchMiB: options.builderScratchMiB, + wait: false, + uploadTimeout: options.uploadTimeout === undefined ? 600 : options.uploadTimeout, + tempDir: options.tempDir, + }; + let outcome: SandboxImageBuildResolution["outcome"] = "created"; + let build: SandboxImageBuild; + try { + build = + contextPath !== undefined + ? await this.buildImageFromDockerfile({ + ...common, + contextPath, + dockerfile, + remote: true, + remoteFullContext, + expectedContextFingerprint: fingerprint, + }) + : await this.buildImageFromDockerImage({ + ...common, + dockerImage: dockerImage as string, + expectedImageDigest: fingerprint, + }); + } catch (error) { + if (!(error instanceof HyperbrowserError)) { + throw error; + } + const existing = matchingImageBuild(error, imageName, inputFormat); + if (existing === null) { + throw error; + } + build = existing; + outcome = "joined"; + } + const wait = options.wait ?? true; + if (wait && build.status !== "completed") { + build = await this.waitForImageBuild(build.id, { + pollInterval: options.pollInterval, + timeout: options.waitTimeout, + signal: options.signal, + }); + } + return { outcome, imageName, imageId: completedImageId(build), build }; + } + + private async buildImageFromRemoteDockerfile( + options: BuildSandboxImageFromDockerfileOptions + ): Promise { + const packaged = await packageDockerBuildContextManifest(options.contextPath, { + dockerfile: options.dockerfile, + forceFullContext: options.remoteFullContext, + expectedContextFingerprint: options.expectedContextFingerprint, + tempDir: options.tempDir, + }); + try { + return await this.submitImageBuild( + { + imageName: options.imageName, + inputSha256: packaged.artifact.sha256Hex, + inputSizeBytes: packaged.artifact.sizeBytes, + inputFormat: packaged.artifact.inputFormat, + sourcePlatform: "linux/amd64", + dockerfilePath: packaged.manifest.dockerfilePath, + imageConfigUser: options.imageConfigUser, + imageInit: options.imageInit, + contextManifest: packaged.manifest, + builderCpus: options.builderCpus, + builderMemoryMiB: options.builderMemoryMiB, + builderScratchMiB: options.builderScratchMiB, + }, + packaged.artifact, + packaged.bundles, + "build context bundle", + options + ); + } finally { + packaged.cleanup(); + } + } + + /** Create a build, upload requested artifacts, complete it, and optionally wait. */ + private async submitImageBuild( + params: CreateSandboxImageBuildParams, + artifact: DockerImageBuildArtifact, + artifacts: Record, + label: string, + options: { + wait?: boolean; + pollInterval?: number; + waitTimeout?: number | null; + signal?: AbortSignal; + uploadTimeout?: number | null; + } + ): Promise { + let buildId: string | null = null; + let buildStarted = false; + try { + const createResult = await this.createImageBuild(params); + buildId = createResult.build.id; + await uploadMissingImageBuildArtifacts(createResult.uploads, artifacts, { + label, + timeout: options.uploadTimeout, + }); + const build = await this.completeImageBuildResilient(buildId, artifact); + buildStarted = true; + if (options.wait ?? true) { + return await this.waitForImageBuild(build.id, { + pollInterval: options.pollInterval, + timeout: options.waitTimeout, + signal: options.signal, + }); + } + return build; + } catch (error) { + if (buildId !== null && !buildStarted) { + try { + await this.cancelImageBuild(buildId); + } catch { + // Best-effort cancellation; surface the original error. + } + } + throw error; + } + } + + private async completeImageBuildResilient( + buildId: string, + artifact: DockerImageBuildArtifact + ): Promise { + const params: CompleteSandboxImageBuildParams = { + inputSha256: artifact.sha256Hex, + inputSizeBytes: artifact.sizeBytes, + inputFormat: artifact.inputFormat, + }; + try { + return await this.completeImageBuild(buildId, params); + } catch (error) { + if (!(error instanceof HyperbrowserError) || error.statusCode !== 409) { + throw error; + } + if (error.message.toLowerCase().includes("already in progress")) { + return this.getImageBuild(buildId); + } + const current = await this.getImageBuild(buildId); + if (current.status !== "awaiting_upload" && current.status !== "upload_verified") { + throw error; + } + await sleep(IMAGE_BUILD_COMPLETE_RETRY_DELAY_MS); + return this.completeImageBuild(buildId, params); + } + } + async updateNetwork( id: string, policy: SandboxNetworkPolicyPatch diff --git a/src/services/volumes.ts b/src/services/volumes.ts index 8bc9ee3..a551679 100644 --- a/src/services/volumes.ts +++ b/src/services/volumes.ts @@ -1,4 +1,4 @@ -import { HyperbrowserError } from "../client"; +import { HyperbrowserError } from "../error"; import { CreateVolumeParams, Volume, @@ -6,18 +6,27 @@ import { VolumeListParams, VolumeListResponse, } from "../types/volume"; +import { optionalInteger } from "./normalize"; import { BaseService } from "./base"; +const normalizeVolume = (volume: Volume): Volume => ({ + ...volume, + size: optionalInteger(volume.size), + transferAmount: optionalInteger(volume.transferAmount), +}); + export class VolumesService extends BaseService { /** * Create a new sandbox volume. */ async create(params: CreateVolumeParams): Promise { try { - return await this.request("/volume", { - method: "POST", - body: JSON.stringify(params), - }); + return normalizeVolume( + await this.request("/volume", { + method: "POST", + body: JSON.stringify(params), + }) + ); } catch (error) { if (error instanceof HyperbrowserError) { throw error; @@ -31,11 +40,18 @@ export class VolumesService extends BaseService { */ async list(params: VolumeListParams = {}): Promise { try { - return await this.request("/volume", undefined, { + const response = await this.request("/volume", undefined, { search: params.search, page: params.page, limit: params.limit, }); + return { + ...response, + volumes: response.volumes.map(normalizeVolume), + totalCount: optionalInteger(response.totalCount), + page: optionalInteger(response.page), + perPage: optionalInteger(response.perPage), + }; } catch (error) { if (error instanceof HyperbrowserError) { throw error; @@ -49,7 +65,7 @@ export class VolumesService extends BaseService { */ async get(id: string): Promise { try { - return await this.request(`/volume/${id}`); + return normalizeVolume(await this.request(`/volume/${id}`)); } catch (error) { if (error instanceof HyperbrowserError) { throw error; @@ -65,10 +81,9 @@ export class VolumesService extends BaseService { */ async delete(key: string): Promise { try { - return await this.request( - `/volume/${encodeURIComponent(key)}`, - { method: "DELETE" } - ); + return await this.request(`/volume/${encodeURIComponent(key)}`, { + method: "DELETE", + }); } catch (error) { if (error instanceof HyperbrowserError) { throw error; diff --git a/src/types/index.ts b/src/types/index.ts index 42b45b2..c121918 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -186,12 +186,29 @@ export { SandboxSnapshotListResponse, SandboxSnapshotDeleteResult, SandboxImageDeleteResult, + SandboxImageInit, SandboxImageBuildStatus, + SandboxImageBuildInputFormat, + SandboxImageBuildSourcePlatform, SandboxImageBuildUpload, SandboxImageBuild, + SandboxBuildContextBundle, + SandboxBuildContextMode, + SandboxBuildContextManifest, + SandboxDockerImageConfig, + SandboxDockerImageLayer, + SandboxDockerImageManifest, CreateSandboxImageBuildParams, + ReuseSandboxDockerImageParams, CompleteSandboxImageBuildParams, SandboxImageBuildCreateResult, + SandboxDockerImageReuseResult, + SandboxImageBuildResolutionOutcome, + SandboxImageBuildResolution, + SandboxImageBuildWaitOptions, + BuildSandboxImageFromDockerImageOptions, + BuildSandboxImageFromDockerfileOptions, + GetOrBuildSandboxImageOptions, SandboxImageBuildListParams, SandboxImageBuildListResponse, CreateSandboxParams, @@ -210,6 +227,8 @@ export { SandboxProcessWaitParams, SandboxProcessSignal, SandboxProcessStdinParams, + SandboxProcessOutputEvent, + SandboxProcessExitEvent, SandboxProcessStreamEvent, SandboxFileType, SandboxFileInfo, @@ -390,3 +409,18 @@ export { BrandingConfidence, BrandingColorScheme, } from "./web/branding"; + +export type { + StartSandboxFromSnapshotParams, + SandboxRuntimeSession, + SandboxFileUploadStream, + SandboxFileUploadStreamOptions, + SandboxFileDownloadStreamOptions, + SandboxFileMoveParams, + SandboxFileRenameOptions, + SandboxFileWatchParams, + SandboxFileWatchEvent, + SandboxFileWatchStatus, + SandboxFileWatchEventsParams, + SandboxFileWatchStreamEvent, +} from "./sandbox"; diff --git a/src/types/sandbox.ts b/src/types/sandbox.ts index d3c3f9a..7b20be9 100644 --- a/src/types/sandbox.ts +++ b/src/types/sandbox.ts @@ -6,7 +6,7 @@ import { SessionLaunchState, SessionStatus } from "./session"; export type SandboxStatus = SessionStatus; export interface SandboxNetworkPolicy { - allowInternetAccess: boolean; + allowInternetAccess?: boolean | null; allowOut: string[]; denyOut: string[]; } @@ -28,6 +28,18 @@ export interface SandboxRuntimeTarget { } export interface Sandbox { + runtimeClass?: "firecracker" | "gvisor-cpu"; + capabilities?: { + commands: boolean; + files: boolean; + pty: boolean; + gpu: boolean; + snapshots: boolean; + volumes: boolean; + exposedPorts: boolean; + internetAccess: boolean; + writableStorage: "memory"; + }; id: string; teamId: string; status: SandboxStatus; @@ -36,21 +48,21 @@ export interface Sandbox { createdAt: string; updatedAt: string; closeReason?: string | null; - dataConsumed?: number; - proxyDataConsumed?: number; - usageType?: string; + dataConsumed?: number | null; + proxyDataConsumed?: number | null; + usageType?: string | null; jobId?: string | null; launchState?: SessionLaunchState | null; creditsUsed: number | null; region: SessionRegion; sessionUrl: string; duration: number; - proxyBytesUsed: number; + proxyBytesUsed?: number | null; cpu?: number | null; memoryMiB?: number | null; diskMiB?: number | null; timeoutMinutes?: number | null; - network?: SandboxNetworkPolicy; + network?: SandboxNetworkPolicy | null; runtime: SandboxRuntimeTarget; exposedPorts: SandboxExposeResult[]; } @@ -69,6 +81,7 @@ export interface SandboxVolumeMount { } interface SandboxCreateCommonParams { + runtimeClass?: "firecracker" | "gvisor-cpu"; region?: SessionRegion; enableRecording?: boolean; exposedPorts?: SandboxExposeParams[]; @@ -117,13 +130,21 @@ export interface SandboxListResponse { export type SandboxImageSource = "public" | "team"; +export interface SandboxImageInit { + env?: Record; + command?: string; + args?: string[]; + workingDir?: string; +} + export interface SandboxImageSummary { id: string; imageName: string; namespace: string; source?: SandboxImageSource; - imageInit?: Record | null; + imageInit?: SandboxImageInit | Record | null; uploaded: boolean; + ready?: boolean | null; createdAt: string; updatedAt: string; } @@ -155,7 +176,7 @@ export interface SandboxSnapshotSummary { vcpus?: number | null; memMiB?: number | null; diskSizeMiB?: number | null; - compatibilityTag: string; + compatibilityTag?: string | null; metadata: Record; uploaded: boolean; createdAt: string; @@ -198,7 +219,16 @@ export type SandboxImageBuildStatus = | "failed" | "canceled"; +export type SandboxImageBuildInputFormat = + | "rootfs_export_tar_gz" + | "dockerfile_context_tar_gz" + | "dockerfile_context_manifest_v1" + | "docker_image_manifest_v1"; + +export type SandboxImageBuildSourcePlatform = "linux/amd64"; + export interface SandboxImageBuildUpload { + sha256?: string | null; url: string; method: string; headers: Record; @@ -230,29 +260,167 @@ export interface SandboxImageBuild { updatedAt?: string | null; } +export interface SandboxBuildContextBundle { + sha256: string; + sizeBytes: number; + uncompressedSizeBytes: number; + entryCount: number; +} + +export type SandboxBuildContextMode = "sparse" | "full"; + +export interface SandboxBuildContextManifest { + version: 1; + dockerfilePath: string; + contextMode: SandboxBuildContextMode; + fallbackReason?: string; + bundles: SandboxBuildContextBundle[]; +} + +export interface SandboxDockerImageConfig { + sha256: string; + sizeBytes: number; + dataBase64: string; +} + +export interface SandboxDockerImageLayer { + sha256: string; + sizeBytes: number; +} + +export interface SandboxDockerImageManifest { + version: 1; + imageDigest: string; + descriptor?: SandboxDockerImageConfig; + config: SandboxDockerImageConfig; + layers: SandboxDockerImageLayer[]; +} + export interface CreateSandboxImageBuildParams { imageName: string; inputSha256: string; inputSizeBytes: number; - inputFormat?: "rootfs_export_tar_gz"; - sourcePlatform?: "linux/amd64"; + /** Builder vCPUs (sent as `vcpus`). */ + builderCpus?: number; + /** Builder memory in MiB (sent as `memMiB`). */ + builderMemoryMiB?: number; + /** Builder scratch disk in MiB (sent as `scratchMiB`). */ + builderScratchMiB?: number; + inputFormat?: SandboxImageBuildInputFormat; + sourcePlatform?: SandboxImageBuildSourcePlatform; imageConfigUser?: string; - imageInit?: { - env?: Record; - command?: string; - args?: string[]; - }; + imageInit?: SandboxImageInit; + dockerfilePath?: string; + contextManifest?: SandboxBuildContextManifest; + dockerImageManifest?: SandboxDockerImageManifest; +} + +export interface ReuseSandboxDockerImageParams { + imageName: string; + sourceImageDigest: string; + sourcePlatform?: SandboxImageBuildSourcePlatform; + imageConfigUser?: string; + imageInit?: SandboxImageInit; } export interface CompleteSandboxImageBuildParams { inputSha256: string; inputSizeBytes: number; - inputFormat?: "rootfs_export_tar_gz"; + inputFormat?: SandboxImageBuildInputFormat; } export interface SandboxImageBuildCreateResult { build: SandboxImageBuild; - upload: SandboxImageBuildUpload; + upload?: SandboxImageBuildUpload | null; + uploads?: SandboxImageBuildUpload[]; +} + +export interface SandboxDockerImageReuseResult { + hit: boolean; + build?: SandboxImageBuild | null; +} + +export type SandboxImageBuildResolutionOutcome = "reused" | "joined" | "created"; + +/** + * The result of resolving content-derived image inputs. + * + * `imageId` is populated only for a ready image. With `wait: false`, `build` + * identifies the submitted or joined build, which the caller can poll later. + */ +export interface SandboxImageBuildResolution { + outcome: SandboxImageBuildResolutionOutcome; + imageName: string; + imageId?: string; + build?: SandboxImageBuild; +} + +export interface SandboxImageBuildWaitOptions { + signal?: AbortSignal; + /** Seconds between status polls. Defaults to 3. */ + pollInterval?: number; + /** Seconds to wait before giving up. `null` waits forever. Defaults to 35 minutes. */ + timeout?: number | null; +} + +interface SandboxImageBuildCommonOptions { + /** Cancel this caller's polling; accepted builds and uploads continue independently. */ + signal?: AbortSignal; + imageName: string; + platform?: string; + imageInit?: SandboxImageInit; + imageConfigUser?: string; + builderCpus?: number; + builderMemoryMiB?: number; + builderScratchMiB?: number; + /** Wait for the build to complete. Defaults to true. */ + wait?: boolean; + pollInterval?: number; + waitTimeout?: number | null; + /** Directory for temporary packaging artifacts. */ + tempDir?: string; + /** Per-upload inactivity timeout in seconds. */ + uploadTimeout?: number | null; +} + +export interface BuildSandboxImageFromDockerImageOptions extends SandboxImageBuildCommonOptions { + dockerImage: string; + expectedImageDigest?: string; +} + +export interface BuildSandboxImageFromDockerfileOptions extends SandboxImageBuildCommonOptions { + contextPath: string; + dockerfile?: string; + /** Send the build context to Hyperbrowser (default) instead of building with local Docker. */ + remote?: boolean; + remoteFullContext?: boolean; + expectedContextFingerprint?: string; + dockerTag?: string; + buildArgs?: Record; +} + +export interface GetOrBuildSandboxImageOptions { + /** Cancel this caller's polling; accepted builds and uploads continue independently. */ + signal?: AbortSignal; + contextPath?: string; + dockerImage?: string; + imageNamePrefix?: string; + dockerfile?: string; + platform?: string; + remoteFullContext?: boolean; + expectedContextFingerprint?: string; + expectedImageDigest?: string; + imageInit?: SandboxImageInit; + imageConfigUser?: string; + builderCpus?: number; + builderMemoryMiB?: number; + builderScratchMiB?: number; + forceBuild?: boolean; + wait?: boolean; + pollInterval?: number; + waitTimeout?: number | null; + uploadTimeout?: number | null; + tempDir?: string; } export interface SandboxImageBuildListParams { @@ -287,7 +455,7 @@ export interface SandboxExposeResult { port: number; auth: boolean; url: string; - browserUrl?: string; + browserUrl?: string | null; browserUrlExpiresAt?: string | null; } @@ -305,7 +473,11 @@ export type SandboxProcessStatus = | "timed_out"; export interface SandboxExecParams { + /** Cancel local collection without killing the detached command. */ + signal?: AbortSignal; command: string; + /** Maximum combined stdout/stderr bytes collected locally. Defaults to 64 MiB. */ + maxOutputBytes?: number; /** @deprecated Legacy compatibility only. Converted into a single shell command string. */ args?: string[]; cwd?: string; @@ -323,12 +495,12 @@ export interface SandboxProcessSummary { id: string; status: SandboxProcessStatus; command: string; - args?: string[]; + args?: string[] | null; cwd: string; - pid?: number; + pid?: number | null; exitCode?: number | null; startedAt: number; - completedAt?: number; + completedAt?: number | null; } export interface SandboxProcessResult { @@ -338,8 +510,10 @@ export interface SandboxProcessResult { stdout: string; stderr: string; startedAt: number; - completedAt?: number; - error?: string; + completedAt?: number | null; + error?: string | null; + outputTruncated?: boolean; + lastSeq?: number | null; } export interface SandboxProcessListParams { @@ -368,17 +542,19 @@ export interface SandboxProcessStdinParams { eof?: boolean; } -export type SandboxProcessStreamEvent = - | { - type: "stdout" | "stderr" | "system"; - seq: number; - data: string; - timestamp: number; - } - | { - type: "exit"; - result: SandboxProcessResult; - }; +export interface SandboxProcessOutputEvent { + type: "stdout" | "stderr" | "system"; + seq: number; + data: string; + timestamp: number; +} + +export interface SandboxProcessExitEvent { + type: "exit"; + result: SandboxProcessResult; +} + +export type SandboxProcessStreamEvent = SandboxProcessOutputEvent | SandboxProcessExitEvent; export type SandboxFileType = "file" | "dir"; @@ -522,17 +698,17 @@ export interface SandboxTerminalOutputChunk { export interface SandboxTerminalStatus { id: string; command: string; - args?: string[]; + args?: string[] | null; cwd: string; - pid?: number; + pid?: number | null; running: boolean; exitCode?: number | null; - error?: string; + error?: string | null; timedOut?: boolean; rows: number; cols: number; startedAt: number; - finishedAt?: number; + finishedAt?: number | null; output?: SandboxTerminalOutputChunk[]; } @@ -554,3 +730,63 @@ export type SandboxTerminalEvent = type: "exit"; status: SandboxTerminalStatus; }; + +/** True streaming transfer inputs; strings in iterable chunks are UTF-8. */ +export type SandboxFileUploadStream = + | AsyncIterable + | Iterable; +export interface SandboxFileUploadStreamOptions { + contentLength?: number; + signal?: AbortSignal; +} +export interface SandboxFileDownloadStreamOptions { + signal?: AbortSignal; +} +export interface SandboxFileMoveParams { + source: string; + destination: string; + overwrite?: boolean; +} +export interface SandboxFileRenameOptions { + overwrite?: boolean; +} +export interface SandboxFileWatchParams { + recursive?: boolean; +} +export interface SandboxFileWatchEvent { + seq: number; + path: string; + op: string; + timestamp: number; +} +export interface SandboxFileWatchStatus { + id: string; + path: string; + recursive: boolean; + active: boolean; + error?: string | null; + createdAt: number; + stoppedAt?: number | null; + oldestSeq?: number; + lastSeq?: number; + eventCount?: number; + events?: SandboxFileWatchEvent[] | null; +} +export interface SandboxFileWatchEventsParams { + cursor?: number; + route?: "ws" | "stream"; + signal?: AbortSignal; +} +export type SandboxFileWatchStreamEvent = + | { type: "event"; event: SandboxFileWatchEvent } + | { type: "done"; status: SandboxFileWatchStatus }; + +export type StartSandboxFromSnapshotParams = Extract; +export interface SandboxRuntimeSession { + sandboxId: string; + status: SandboxStatus; + region: SessionRegion; + token: string; + tokenExpiresAt: string | null; + runtime: SandboxRuntimeTarget; +} diff --git a/src/types/volume.ts b/src/types/volume.ts index c7da276..0c61ce8 100644 --- a/src/types/volume.ts +++ b/src/types/volume.ts @@ -5,8 +5,8 @@ export interface CreateVolumeParams { export interface Volume { id: string; name: string; - size?: number; - transferAmount?: number; + size?: number | null; + transferAmount?: number | null; } export interface VolumeListParams { @@ -17,13 +17,13 @@ export interface VolumeListParams { export interface VolumeListResponse { volumes: Volume[]; - totalCount?: number; - page?: number; - perPage?: number; + totalCount?: number | null; + page?: number | null; + perPage?: number | null; } export interface VolumeDeleteResult { deleted: boolean; - id?: string; - name?: string; + id?: string | null; + name?: string | null; } diff --git a/tests/sandbox/e2e/expose.test.ts b/tests/e2e/expose.test.ts similarity index 94% rename from tests/sandbox/e2e/expose.test.ts rename to tests/e2e/expose.test.ts index 10be581..cd83514 100644 --- a/tests/sandbox/e2e/expose.test.ts +++ b/tests/e2e/expose.test.ts @@ -3,11 +3,11 @@ */ import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import type { SandboxHandle } from "../../../src/services/sandboxes"; -import { createClient } from "../../helpers/config"; -import { expectHyperbrowserError } from "../../helpers/errors"; -import { fetchRuntimeUrl } from "../../helpers/http"; -import { defaultSandboxParams, stopSandboxIfRunning } from "../../helpers/sandbox"; +import type { SandboxHandle } from "../../src/services/sandboxes"; +import { createClient } from "../helpers/config"; +import { expectHyperbrowserError } from "../helpers/errors"; +import { fetchRuntimeUrl } from "../helpers/http"; +import { defaultSandboxParams, stopSandboxIfRunning } from "../helpers/sandbox"; const client = createClient(); const HTTP_PORT = 3210; diff --git a/tests/sandbox/e2e/files.test.ts b/tests/e2e/files.test.ts similarity index 99% rename from tests/sandbox/e2e/files.test.ts rename to tests/e2e/files.test.ts index a033b9f..46c375a 100644 --- a/tests/sandbox/e2e/files.test.ts +++ b/tests/e2e/files.test.ts @@ -6,15 +6,15 @@ import { Blob } from "buffer"; import { ReadableStream } from "node:stream/web"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import type { SandboxHandle } from "../../../src/services/sandboxes"; -import { createClient, testName } from "../../helpers/config"; -import { expectHyperbrowserError } from "../../helpers/errors"; -import { fetchSignedUrl } from "../../helpers/http"; +import type { SandboxHandle } from "../../src/services/sandboxes"; +import { createClient, testName } from "../helpers/config"; +import { expectHyperbrowserError } from "../helpers/errors"; +import { fetchSignedUrl } from "../helpers/http"; import { defaultSandboxParams, stopSandboxIfRunning, waitForRuntimeReady, -} from "../../helpers/sandbox"; +} from "../helpers/sandbox"; const client = createClient(); diff --git a/tests/sandbox/e2e/lifecycle.test.ts b/tests/e2e/lifecycle.test.ts similarity index 94% rename from tests/sandbox/e2e/lifecycle.test.ts rename to tests/e2e/lifecycle.test.ts index 302c30e..105aca4 100644 --- a/tests/sandbox/e2e/lifecycle.test.ts +++ b/tests/e2e/lifecycle.test.ts @@ -6,21 +6,21 @@ import { randomUUID } from "crypto"; import fetch from "node-fetch"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import { HyperbrowserError } from "../../../src/client"; -import type { SandboxHandle } from "../../../src/services/sandboxes"; +import { HyperbrowserError } from "../../src/client"; +import type { SandboxHandle } from "../../src/services/sandboxes"; import { API_KEY, BASE_URL, createClient, DEFAULT_IMAGE_NAME, -} from "../../helpers/config"; -import { expectHyperbrowserError } from "../../helpers/errors"; +} from "../helpers/config"; +import { expectHyperbrowserError } from "../helpers/errors"; import { defaultSandboxParams, stopSandboxIfRunning, waitForCreatedSnapshot, waitForRuntimeReady, -} from "../../helpers/sandbox"; +} from "../helpers/sandbox"; const client = createClient(); const CUSTOM_IMAGE_NAME = "node"; @@ -133,12 +133,17 @@ describe.sequential("sandbox lifecycle e2e", () => { }); afterAll(async () => { - await stopSandboxIfRunning(sandbox); - await stopSandboxIfRunning(staleHandle); - await stopSandboxIfRunning(secondary); - await stopSandboxIfRunning(imageSandbox); - await stopSandboxIfRunning(customImageSandbox); - await stopSandboxIfRunning(customSnapshotSandbox); + const stopped = await Promise.allSettled( + [sandbox, staleHandle, secondary, imageSandbox, customImageSandbox, customSnapshotSandbox] + .map(stopSandboxIfRunning) + ); + const snapshots = await Promise.allSettled( + [memorySnapshot, customImageMemorySnapshot] + .filter((snapshot) => snapshot !== null) + .map((snapshot) => client.sandboxes.deleteSnapshot(snapshot!.snapshotId)) + ); + const failure = [...stopped, ...snapshots].find((result) => result.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; }); test("create response contains runtime auth", async () => { diff --git a/tests/sandbox/e2e/list.test.ts b/tests/e2e/list.test.ts similarity index 91% rename from tests/sandbox/e2e/list.test.ts rename to tests/e2e/list.test.ts index 8f79e29..bf9edd8 100644 --- a/tests/sandbox/e2e/list.test.ts +++ b/tests/e2e/list.test.ts @@ -3,15 +3,15 @@ */ import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import type { SandboxHandle } from "../../../src/services/sandboxes"; -import type { Sandbox } from "../../../src/types"; -import { createClient, testName } from "../../helpers/config"; +import type { SandboxHandle } from "../../src/services/sandboxes"; +import type { Sandbox } from "../../src/types"; +import { createClient, testName } from "../helpers/config"; import { defaultSandboxParams, stopSandboxIfRunning, waitForCreatedSnapshot, waitForRuntimeReady, -} from "../../helpers/sandbox"; +} from "../helpers/sandbox"; const client = createClient(); const SANDBOX_PAGE_LIMIT = 50; @@ -71,7 +71,11 @@ describe.sequential("sandbox list e2e", () => { }); afterAll(async () => { - await stopSandboxIfRunning(sandbox); + try { + await stopSandboxIfRunning(sandbox); + } finally { + if (memorySnapshot) await client.sandboxes.deleteSnapshot(memorySnapshot.snapshotId); + } }); test("list returns the created sandbox in the active set", async () => { @@ -129,7 +133,7 @@ describe.sequential("sandbox list e2e", () => { const createdSnapshots = await client.sandboxes.listSnapshots({ status: "created", imageName: memorySnapshot!.imageName, - limit: 200, + limit: 100, }); expect(createdSnapshots.snapshots.some((entry) => entry.id === listedSnapshot.id)).toBe( diff --git a/tests/sandbox/e2e/process.test.ts b/tests/e2e/process.test.ts similarity index 87% rename from tests/sandbox/e2e/process.test.ts rename to tests/e2e/process.test.ts index aedef7d..dc5056b 100644 --- a/tests/sandbox/e2e/process.test.ts +++ b/tests/e2e/process.test.ts @@ -4,15 +4,15 @@ */ import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import type { SandboxHandle } from "../../../src/services/sandboxes"; -import type { SandboxProcessStreamEvent } from "../../../src/types"; -import { createClient } from "../../helpers/config"; -import { expectHyperbrowserError } from "../../helpers/errors"; +import type { SandboxHandle } from "../../src/services/sandboxes"; +import type { SandboxProcessStreamEvent } from "../../src/types"; +import { createClient } from "../helpers/config"; +import { expectHyperbrowserError } from "../helpers/errors"; import { defaultSandboxParams, stopSandboxIfRunning, waitForRuntimeReady, -} from "../../helpers/sandbox"; +} from "../helpers/sandbox"; async function collectProcessStream( events: AsyncIterable @@ -139,9 +139,19 @@ describe.sequential("sandbox process e2e", () => { const result = await noisyProcess.result(); expect(result.stdout.length).toBeGreaterThan(3 * 1024 * 1024); + // Collected output survives the runtime replay window on the original handle. + const replayed = await collectProcessStream(noisyProcess.stream(1)); + expect( + replayed + .filter((event) => event.type === "stdout") + .map((event) => (event.type === "stdout" ? event.data : "")) + .join("") + ).toBe(result.stdout); + + const reattached = await sandbox!.processes.get(noisyProcess.id); await expectHyperbrowserError( "process replay window expired", - () => collectProcessStream(noisyProcess.stream(1)), + () => collectProcessStream(reattached.stream(1)), { statusCode: 410, code: "replay_window_expired", @@ -162,10 +172,10 @@ describe.sequential("sandbox process e2e", () => { "process wait timeout", () => timeoutProcess.wait({ timeoutMs: 100 }), { - statusCode: 408, + code: "wait_timeout", service: "runtime", retryable: false, - messageIncludes: "timed out", + messageIncludes: "Timed out", } ); diff --git a/tests/sandbox/e2e/resource-config.test.ts b/tests/e2e/resource-config.test.ts similarity index 91% rename from tests/sandbox/e2e/resource-config.test.ts rename to tests/e2e/resource-config.test.ts index 365669c..cbd6daf 100644 --- a/tests/sandbox/e2e/resource-config.test.ts +++ b/tests/e2e/resource-config.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "vitest"; -import type { SandboxHandle } from "../../../src/services/sandboxes"; -import { createClient, DEFAULT_IMAGE_NAME } from "../../helpers/config"; -import { stopSandboxIfRunning, waitForRuntimeReady } from "../../helpers/sandbox"; +import type { SandboxHandle } from "../../src/services/sandboxes"; +import { createClient, DEFAULT_IMAGE_NAME } from "../helpers/config"; +import { stopSandboxIfRunning, waitForRuntimeReady } from "../helpers/sandbox"; const client = createClient(); diff --git a/tests/sandbox/e2e/sudo.test.ts b/tests/e2e/sudo.test.ts similarity index 92% rename from tests/sandbox/e2e/sudo.test.ts rename to tests/e2e/sudo.test.ts index 45a0534..dc95f78 100644 --- a/tests/sandbox/e2e/sudo.test.ts +++ b/tests/e2e/sudo.test.ts @@ -4,13 +4,13 @@ */ import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import type { SandboxHandle } from "../../../src/services/sandboxes"; -import { createClient } from "../../helpers/config"; +import type { SandboxHandle } from "../../src/services/sandboxes"; +import { createClient } from "../helpers/config"; import { defaultSandboxParams, stopSandboxIfRunning, waitForRuntimeReady, -} from "../../helpers/sandbox"; +} from "../helpers/sandbox"; const client = createClient(); diff --git a/tests/sandbox/e2e/terminal-smoke.test.ts b/tests/e2e/terminal-smoke.test.ts similarity index 91% rename from tests/sandbox/e2e/terminal-smoke.test.ts rename to tests/e2e/terminal-smoke.test.ts index 67d3d54..a11115a 100644 --- a/tests/sandbox/e2e/terminal-smoke.test.ts +++ b/tests/e2e/terminal-smoke.test.ts @@ -4,16 +4,16 @@ */ import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import type { SandboxHandle } from "../../../src/services/sandboxes"; -import type { SandboxTerminalConnection } from "../../../src/sandbox/terminal"; -import type { SandboxTerminalStatus } from "../../../src/types/sandbox"; -import { createClient } from "../../helpers/config"; -import { expectHyperbrowserError } from "../../helpers/errors"; +import type { SandboxHandle } from "../../src/services/sandboxes"; +import type { SandboxTerminalConnection } from "../../src/sandbox/terminal"; +import type { SandboxTerminalStatus } from "../../src/types/sandbox"; +import { createClient } from "../helpers/config"; +import { expectHyperbrowserError } from "../helpers/errors"; import { defaultSandboxParams, stopSandboxIfRunning, waitForRuntimeReady, -} from "../../helpers/sandbox"; +} from "../helpers/sandbox"; async function collectTerminalSession( connection: SandboxTerminalConnection @@ -131,9 +131,11 @@ describe.sequential("sandbox terminal e2e", () => { const connection = await terminal.attach(); try { await connection.resize(32, 110); - const refreshed = await terminal.refresh(); - expect(refreshed.current.rows).toBe(32); - expect(refreshed.current.cols).toBe(110); + // A WebSocket send completes locally; the HTTP read can overtake it. + await expect.poll(async () => { + const refreshed = await terminal.refresh(); + return { rows: refreshed.current.rows, cols: refreshed.current.cols }; + }, { timeout: 5_000, interval: 50 }).toEqual({ rows: 32, cols: 110 }); await connection.write("exit\n"); const result = await collectTerminalSession(connection); diff --git a/tests/fixtures/docker_context_parity.json b/tests/fixtures/docker_context_parity.json new file mode 100644 index 0000000..0b624b5 --- /dev/null +++ b/tests/fixtures/docker_context_parity.json @@ -0,0 +1,434 @@ +{ + "metadata": { + "buildkit": "v0.30.0", + "patternmatcher": "v0.6.1", + "fsutil": "a2aa163d723f" + }, + "dockerfiles": [ + { + "name": "no-context-sources", + "dockerfile": "FROM scratch\nCMD [\"true\"]\n", + "goSourceGroups": [], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "shell-copy-single", + "dockerfile": "FROM scratch\nCOPY app.py /app/\n", + "goSourceGroups": [["app.py"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "shell-copy-multiple", + "dockerfile": "FROM scratch\nCOPY pyproject.toml poetry.lock src /app/\n", + "goSourceGroups": [["pyproject.toml", "poetry.lock", "src"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "json-copy-spaces", + "dockerfile": "FROM scratch\nCOPY [\"one file\", \"two\", \"/dest/\"]\n", + "goSourceGroups": [["one file", "two"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "copy-modern-flags", + "dockerfile": "# syntax=docker/dockerfile:1-labs\nFROM scratch\nCOPY --chown=1:1 --chmod=0755 --link --parents --exclude=*.tmp src /app/\n", + "goSourceGroups": [["src"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "copy-from-stage", + "dockerfile": "FROM scratch AS generated\nFROM scratch\nCOPY --from=generated /out /out\n", + "goSourceGroups": [], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "copy-from-named-context", + "dockerfile": "FROM scratch\nCOPY --from=config /settings.json /settings.json\n", + "goSourceGroups": [], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "copy-continuation", + "dockerfile": "FROM scratch\nCOPY --link \\\n first \\\n second /dest/\n", + "goSourceGroups": [["first", "second"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "copy-tabs-and-lowercase", + "dockerfile": "from scratch\ncopy\tfile.txt\t/dest/\n", + "goSourceGroups": [["file.txt"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "copy-escaped-space", + "dockerfile": "FROM scratch\nCOPY one\\ file /dest/\n", + "goSourceGroups": [["one\\", "file"]], + "goFallback": "", + "pythonExpectation": "full" + }, + { + "name": "local-and-remote-add", + "dockerfile": "FROM scratch\nADD local.tar /local/\nADD https://example.com/archive.tar /remote/\nADD https://example.com/archive.tar?version=1 /remote-query/\nADD git://example.com/repository /git/\nADD git@github.com:moby/buildkit.git /ssh-git/\n", + "goSourceGroups": [["local.tar"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "add-arbitrary-uri-scheme", + "dockerfile": "FROM scratch\nADD oci-layout://example/image /image/\n", + "goSourceGroups": [["oci-layout://example/image"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "add-colon-paths-are-local", + "dockerfile": "FROM scratch\nADD assets:latest /asset/\nADD http:archive /archive/\n", + "goSourceGroups": [["assets:latest"], ["http:archive"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "add-file-uri-is-local", + "dockerfile": "FROM scratch\nADD file:///context/archive.tar /archive/\n", + "goSourceGroups": [["file:///context/archive.tar"]], + "goFallback": "", + "pythonExpectation": "exact" + }, + { + "name": "copy-variable", + "dockerfile": "FROM scratch\nARG SOURCE=src\nCOPY $SOURCE /app/\n", + "goSourceGroups": [], + "goFallback": "copy_source_requires_expansion", + "pythonExpectation": "full" + }, + { + "name": "add-variable", + "dockerfile": "FROM scratch\nADD ${SOURCE} /app/\n", + "goSourceGroups": [], + "goFallback": "add_source_requires_expansion", + "pythonExpectation": "full" + }, + { + "name": "copy-star-pattern", + "dockerfile": "FROM scratch\nCOPY src/*.py /app/\n", + "goSourceGroups": [["src/*.py"]], + "goFallback": "", + "pythonExpectation": "full" + }, + { + "name": "copy-doublestar-pattern", + "dockerfile": "# syntax=docker/dockerfile:1-labs\nFROM scratch\nCOPY --parents src/**/*.txt /app/\n", + "goSourceGroups": [["src/**/*.txt"]], + "goFallback": "", + "pythonExpectation": "full" + }, + { + "name": "copy-escaped-character-class", + "dockerfile": "FROM scratch\nCOPY arr[[]0].txt /app/\n", + "goSourceGroups": [["arr[[]0].txt"]], + "goFallback": "", + "pythonExpectation": "full" + }, + { + "name": "copy-heredoc-and-local-source", + "dockerfile": "# syntax=docker/dockerfile:1\nFROM scratch\nCOPY local.txt < new Promise((resolve) => setTimeout(resolve, ms)); +export const localHTTP = async (handler: RequestListener) => { + const sockets = new Set(); + const server = createServer(handler); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); + // IPv6 loopback isolates fixtures from development IPv4 proxy/port rules. + const host = "::1"; + await new Promise((resolve) => server.listen(0, host, resolve)); + return { + server, + sockets, + url: `http://[${host}]:${(server.address() as AddressInfo).port}`, + close: async () => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +}; diff --git a/tests/helpers/sandbox.ts b/tests/helpers/sandbox.ts index bf2f98c..f2e9b15 100644 --- a/tests/helpers/sandbox.ts +++ b/tests/helpers/sandbox.ts @@ -3,7 +3,7 @@ import type { SandboxHandle } from "../../src/services/sandboxes"; import type { CreateSandboxParams, SandboxSnapshotSummary } from "../../src/types"; import { DEFAULT_IMAGE_NAME } from "./config"; -const SNAPSHOT_LIST_LIMIT = 200; +const SNAPSHOT_LIST_LIMIT = 100; const LIST_POLL_DELAY_MS = 500; const LIST_POLL_TIMEOUT_MS = 90_000; diff --git a/tests/integration/build-context-fingerprint.test.ts b/tests/integration/build-context-fingerprint.test.ts new file mode 100644 index 0000000..6c8b1d8 --- /dev/null +++ b/tests/integration/build-context-fingerprint.test.ts @@ -0,0 +1,262 @@ +import { createHash } from "crypto"; +import { chmodSync, cpSync, linkSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, symlinkSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { Readable } from "stream"; +import { createGunzip } from "zlib"; +import { afterEach, describe, expect, test } from "vitest"; +import { + DockerBuildContextChangedError, + dockerBuildContextFingerprint, + packageDockerBuildContextManifest, +} from "../../src/sandbox/image-build/context"; +import { readTarEntries } from "../../src/sandbox/image-build/tar"; +import { imageBuildName } from "../../src/sandbox/image-build/resolution"; +import { lchmodSync, rmSync } from "fs"; + +const tempDirs: string[] = []; +const tempDir = (): string => { + const dir = mkdtempSync(path.join(tmpdir(), "hb-fingerprint-")); + tempDirs.push(dir); + return dir; +}; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +const context = (root: string, dockerfile = "FROM scratch\nCOPY . /app\n", ignore = ""): string => { + mkdirSync(root, { recursive: true }); + writeFileSync(path.join(root, "Dockerfile"), dockerfile); + writeFileSync(path.join(root, ".dockerignore"), ignore); + mkdirSync(path.join(root, "app")); + writeFileSync(path.join(root, "app", "main.py"), "print('hello')\n"); + writeFileSync(path.join(root, "app", "debug.log"), "diagnostics\n"); + writeFileSync(path.join(root, "unused"), "not copied by sparse builds\n"); + return root; +}; + +const collect = async (chunks: AsyncIterable): Promise => { + const parts: Buffer[] = []; + for await (const chunk of chunks) { + parts.push(chunk); + } + return Buffer.concat(parts); +}; + +describe("docker build context fingerprint", () => { + test("matches the Python SDK's golden fingerprint byte for byte", async () => { + const root = tempDir(); + writeFileSync(path.join(root, "Dockerfile"), Buffer.from("FROM scratch\nCOPY . /app\n")); + chmodSync(path.join(root, "Dockerfile"), 0o644); + const folder = path.join(root, "long-path-" + "x".repeat(110)); + mkdirSync(folder); + chmodSync(folder, 0o755); + writeFileSync(path.join(folder, "unicode-ü.txt"), Buffer.from("canonical\x00payload\n")); + chmodSync(path.join(folder, "unicode-ü.txt"), 0o640); + writeFileSync(path.join(root, "empty"), Buffer.alloc(0)); + chmodSync(path.join(root, "empty"), 0o600); + symlinkSync("empty", path.join(root, "link")); + // Fingerprints include permissions; create the same metadata as Python. + if (process.platform === "darwin") lchmodSync(path.join(root, "link"), 0o777); + + await expect(dockerBuildContextFingerprint(root)).resolves.toBe( + "8d66062b58007e23ed8844b026726ac24e4ea5b32e4d6c3e4590d48b252755f9" + ); + }); + + const cases: Array<[string, string]> = [ + ["FROM scratch\nCOPY app /app\n", ""], + ["FROM scratch\nCOPY . /app\n", "**/*.log\n"], + ["FROM scratch\nCOPY . /app\n", "app\n!app/main.py\n"], + ["FROM scratch\nCOPY app /app\nCOPY app/main.py /main\n", ""], + ["FROM scratch\nCOPY app/*.py /app/\n", "unused\n"], + ["FROM scratch\nARG SRC=app\nCOPY $SRC /app\n", ""], + ["FROM scratch\n", "*\n"], + ["FROM scratch AS source\nCOPY app /app\nFROM scratch\nCOPY --from=source /app /app\n", ""], + ["FROM busybox\nRUN --mount=type=bind,source=app,target=/app cat /app/main.py\n", ""], + ["FROM scratch\nCOPY < { + const root = context(path.join(tempDir(), "context"), dockerfile, ignore); + const expected = await dockerBuildContextFingerprint(root, { forceFullContext: full }); + const packaged = await packageDockerBuildContextManifest(root, { + forceFullContext: full, + expectedContextFingerprint: expected, + }); + try { + const hashes: string[] = []; + for (const artifact of Object.values(packaged.bundles)) { + const hasher = createHash("sha256"); + const stream = Readable.from([readFileSync(artifact.path)]).pipe(createGunzip()); + for await (const entry of readTarEntries(stream)) { + const kind = entry.isFile ? "file" : entry.typeflag === "5" ? "directory" : "symlink"; + const content = await collect(entry.content()); + const name = kind === "directory" ? entry.name.replace(/\/+$/, "") : entry.name; + const metadata = Buffer.from( + JSON.stringify([name, kind, entry.mode, entry.size, entry.linkname]) + ); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(metadata.length)); + hasher.update(length); + hasher.update(metadata); + if (entry.isFile) { + hasher.update(content); + } + } + hashes.push(hasher.digest("hex")); + } + const identity = { + bundles: [...new Set(hashes)].sort(), + contextMode: packaged.manifest.contextMode, + dockerfile: packaged.manifest.dockerfilePath, + version: 1, + }; + const actual = createHash("sha256").update(JSON.stringify(identity)).digest("hex"); + expect(expected).toBe(actual); + expect(packaged.fingerprint).toBe(expected); + } finally { + packaged.cleanup(); + } + }); + } + + test.each([ + ["contents", true], + ["mode", true], + ["path", true], + ["mtime", false], + ["ignored-file", false], + ["unused-file", false], + ["dockerfile", true], + ["ignore-rules", true], + ["symlink", true], + ])("identity tracks effective build inputs: %s", async (change, changesIdentity) => { + const root = context(path.join(tempDir(), "context"), "FROM scratch\nCOPY app /app\n", "**/*.log\n"); + const before = await dockerBuildContextFingerprint(root); + const main = path.join(root, "app", "main.py"); + switch (change) { + case "contents": + writeFileSync(main, "print('changed')\n"); + break; + case "mode": + chmodSync(main, 0o755); + break; + case "path": + cpSync(main, path.join(root, "app", "renamed.py")); + rmSync(main); + break; + case "mtime": + writeFileSync(main, readFileSync(main)); + break; + case "ignored-file": + writeFileSync(path.join(root, "app", "other.log"), "more\n"); + break; + case "unused-file": + writeFileSync(path.join(root, "unused"), "changed but not copied\n"); + break; + case "dockerfile": + writeFileSync(path.join(root, "Dockerfile"), "FROM scratch\nCOPY app /srv\n"); + break; + case "ignore-rules": + writeFileSync(path.join(root, ".dockerignore"), ""); + break; + case "symlink": + symlinkSync("main.py", path.join(root, "app", "link")); + break; + } + const after = await dockerBuildContextFingerprint(root); + expect(after !== before).toBe(changesIdentity); + }); + + test.each([false, true])("hard links preserve all paths and match independent copies (full=%s)", async (full) => { + const base = tempDir(); + const root = context(path.join(base, "linked"), "FROM scratch\nCOPY app /app\n"); + const first = path.join(root, "app", "main.py"); + const second = path.join(root, "app", "second.py"); + linkSync(first, second); + symlinkSync("main.py", path.join(root, "app", "link")); + const copied = path.join(base, "copied"); + cpSync(root, copied, { recursive: true, verbatimSymlinks: true }); + const expected = await dockerBuildContextFingerprint(root, { forceFullContext: full }); + await expect(dockerBuildContextFingerprint(copied, { forceFullContext: full })).resolves.toBe(expected); + const packaged = await packageDockerBuildContextManifest(root, { + forceFullContext: full, + expectedContextFingerprint: expected, + }); + try { + const files: Record = {}; + for (const artifact of Object.values(packaged.bundles)) { + const stream = Readable.from([readFileSync(artifact.path)]).pipe(createGunzip()); + for await (const entry of readTarEntries(stream)) { + const content = await collect(entry.content()); + if (entry.isFile) { + files[entry.name] = content; + } + if (entry.name === "app/link") { + expect(entry.typeflag).toBe("2"); + expect(entry.linkname).toBe("main.py"); + } + } + } + expect(files["app/main.py"]).toEqual(readFileSync(first)); + expect(files["app/second.py"]).toEqual(readFileSync(first)); + } finally { + packaged.cleanup(); + } + writeFileSync(second, "updated through hard link\n"); + await expect(dockerBuildContextFingerprint(root, { forceFullContext: full })).resolves.not.toBe(expected); + }); + + test("mutation is rejected using archived bytes and the workspace is removed", async () => { + const base = tempDir(); + const root = context(path.join(base, "context")); + const workspaces = path.join(base, "workspaces"); + mkdirSync(workspaces); + const expected = await dockerBuildContextFingerprint(root); + const script = path.join(root, "app", "main.py"); + writeFileSync(script, "x".repeat(readFileSync(script).length)); + await expect( + packageDockerBuildContextManifest(root, { tempDir: workspaces, expectedContextFingerprint: expected }) + ).rejects.toBeInstanceOf(DockerBuildContextChangedError); + expect(readdirSync(workspaces)).toEqual([]); + }); + + test.each(["", "a".repeat(63), "A".repeat(64), "g".repeat(64)])( + "invalid expected fingerprint %j is rejected before packaging", + async (invalid) => { + await expect( + packageDockerBuildContextManifest(path.join(tempDir(), "missing"), { + expectedContextFingerprint: invalid, + }) + ).rejects.toThrow(/SHA-256 hex digest/); + } + ); + + test("image names are content derived and validated", () => { + const fingerprint = "b".repeat(64); + const name = imageBuildName({ source: "dockerfile", fingerprint }); + expect(name).toMatch(/^hb__dockerfile__[0-9a-f]{16}__linux-amd64$/); + expect(imageBuildName({ source: "dockerfile", fingerprint })).toBe(name); + expect(imageBuildName({ source: "dockerfile", fingerprint, imageInit: {} })).toBe(name); + expect( + imageBuildName({ source: "dockerfile", fingerprint, imageInit: { env: { A: "1" } } }) + ).not.toBe(name); + expect(imageBuildName({ source: "dockerfile", fingerprint, namePrefix: "team" })).toMatch( + /^team__dockerfile__/ + ); + expect(() => imageBuildName({ source: "dockerfile", fingerprint: "nope" })).toThrow( + /SHA-256 hex digest/ + ); + expect(() => imageBuildName({ source: "prebuilt", fingerprint })).toThrow(/sha256:/); + expect(() => imageBuildName({ source: "dockerfile", fingerprint, platform: "linux/arm64" })).not.toThrow(); + expect(() => imageBuildName({ source: "dockerfile", fingerprint, namePrefix: "bad prefix" })).toThrow(); + }); +}); diff --git a/tests/integration/docker-context-parity.test.ts b/tests/integration/docker-context-parity.test.ts new file mode 100644 index 0000000..647613b --- /dev/null +++ b/tests/integration/docker-context-parity.test.ts @@ -0,0 +1,79 @@ +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { afterEach, describe, expect, test } from "vitest"; +import fixture from "../fixtures/docker_context_parity.json"; +import { analyzeDockerfileSources } from "../../src/sandbox/image-build/dockerfile-analysis"; +import { collectContextEntries, loadDockerignore } from "../../src/sandbox/image-build/context"; + +interface DockerfileCase { + name: string; + dockerfile: string; + goSourceGroups: string[][]; + goFallback: string; + pythonExpectation: "exact" | "full"; +} + +interface IgnoreCase { + name: string; + dockerignore: string; + files: string[]; + symlinks?: Array<{ path: string; target: string }>; + sources?: string[]; + expectedEntries: string[]; +} + +const dockerfiles = fixture.dockerfiles as DockerfileCase[]; +const ignoreContexts = fixture.ignoreContexts as IgnoreCase[]; + +const tempDirs: string[] = []; +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("docker context parity with Go/BuildKit fixtures", () => { + test.each(dockerfiles.map((c) => [c.name, c] as const))( + "dockerfile analysis matches Go or falls back to full: %s", + (_name, testCase) => { + const { groups, fallbackReason } = analyzeDockerfileSources(Buffer.from(testCase.dockerfile)); + if (testCase.pythonExpectation === "exact") { + expect(testCase.goFallback).toBe(""); + expect(fallbackReason).toBe(""); + expect(groups).toEqual(testCase.goSourceGroups); + } else { + expect(testCase.pythonExpectation).toBe("full"); + expect(fallbackReason).not.toBe(""); + expect(groups).toEqual([]); + } + } + ); + + test.each(ignoreContexts.map((c) => [c.name, c] as const))( + "dockerignore context selection matches Go fsutil: %s", + (_name, testCase) => { + const root = mkdtempSync(path.join(tmpdir(), "hb-parity-")); + tempDirs.push(root); + for (const relative of testCase.files) { + const destination = path.join(root, relative); + mkdirSync(path.dirname(destination), { recursive: true }); + writeFileSync( + destination, + relative === ".dockerignore" ? testCase.dockerignore : `${relative}\n` + ); + } + for (const symlink of testCase.symlinks ?? []) { + const destination = path.join(root, symlink.path); + mkdirSync(path.dirname(destination), { recursive: true }); + symlinkSync(symlink.target, destination); + } + const ignoreMatcher = loadDockerignore(path.join(root, ".dockerignore")); + const entries = collectContextEntries(root, testCase.sources ?? ["."], { + ignoreMatcher, + required: false, + }); + expect([...entries].sort()).toEqual(testCase.expectedEntries); + } + ); +}); diff --git a/tests/integration/docker-image-manifest.test.ts b/tests/integration/docker-image-manifest.test.ts new file mode 100644 index 0000000..e2c1ee9 --- /dev/null +++ b/tests/integration/docker-image-manifest.test.ts @@ -0,0 +1,264 @@ +import { createHash } from "crypto"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { PaxTarWriter } from "../../src/sandbox/image-build/tar"; +import { + dockerImageDigest, + packageDockerImageManifest, + prepareDockerImageManifestSource, +} from "../../src/sandbox/image-build/docker-image"; +import { deriveAutoImageInit, mergeImageInit } from "../../src/sandbox/image-build/image-init"; +import { SandboxesService } from "../../src/services/sandboxes"; +import { localHTTP } from "../helpers/local-http"; + +const sha256 = (data: Buffer): string => createHash("sha256").update(data).digest("hex"); + +let workspace: string; +let originalPath: string | undefined; + +const writeTar = async (entries: Array<[string, Buffer]>): Promise => { + const chunks: Buffer[] = []; + const writer = new PaxTarWriter(async (chunk: Buffer) => { + chunks.push(chunk); + }); + for (const [name, data] of entries) { + await writer.addEntry( + { name, type: "file", mode: 0o644, size: data.length, linkname: "" }, + (async function* () { + yield data; + })() + ); + } + await writer.close(); + const archive = path.join(workspace, "image.tar"); + writeFileSync(archive, Buffer.concat(chunks)); + return archive; +}; + +/** A stand-in `docker` CLI answering inspect/save from fixture files. */ +const installFakeDocker = ( + inspection: Record, + archive: string, + saveExit = 0 +): void => { + const bin = path.join(workspace, "bin"); + rmSync(bin, { recursive: true, force: true }); + require("fs").mkdirSync(bin); + writeFileSync(path.join(workspace, "inspect.json"), JSON.stringify(inspection)); + const script = `#!/bin/sh +if [ "$1" = "buildx" ] || { [ "$1" = "image" ] && [ "$2" = "rm" ]; }; then + exit 0 +fi +if [ "$1" = "image" ] && [ "$2" = "inspect" ]; then + cat "${path.join(workspace, "inspect.json")}" + exit 0 +fi +if [ "$1" = "image" ] && [ "$2" = "save" ]; then + cat "${archive}" + exit ${saveExit} +fi +echo "unexpected docker invocation: $*" >&2 +exit 1 +`; + writeFileSync(path.join(bin, "docker"), script); + chmodSync(path.join(bin, "docker"), 0o755); + process.env.PATH = `${bin}:${originalPath ?? ""}`; +}; + +beforeEach(() => { + workspace = mkdtempSync(path.join(tmpdir(), "hb-docker-image-")); + originalPath = process.env.PATH; +}); + +afterEach(() => { + process.env.PATH = originalPath; + rmSync(workspace, { recursive: true, force: true }); +}); + +describe("docker image manifest packaging", () => { + const configBytes = Buffer.from('{"architecture":"amd64","config":{}}'); + const layerBytes = Buffer.from("reusable-layer-tar"); + const saveManifest = Buffer.from( + JSON.stringify([{ Config: "config.json", Layers: ["layer.tar"] }]) + ); + + test.each( + ["remote-dockerfile", "local-dockerfile", "image"].flatMap((source) => + [false, true].map((custom) => ({ source, custom })) + ) + )("forwards builder resources through $source (custom=$custom)", async ({ source, custom }) => { + const archive = await writeTar([ + ["config.json", configBytes], + ["layer.tar", layerBytes], + ["manifest.json", saveManifest], + ]); + installFakeDocker( + { Id: `sha256:${sha256(configBytes)}`, Os: "linux", Architecture: "amd64", Config: {} }, + archive + ); + writeFileSync(path.join(workspace, "Dockerfile"), "FROM scratch\n"); + let captured: Record | undefined; + const build = { id: "b", imageName: "n", status: "completed", imageId: "image" }; + const server = await localHTTP(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + if (req.url === "/api/images/builds/reuse") { + res.end('{"hit":false}'); + return; + } + if (req.url === "/api/images/builds") { + captured = JSON.parse(Buffer.concat(chunks).toString()); + res.end(JSON.stringify({ build, uploads: [] })); + return; + } + expect(req.url).toBe("/api/images/builds/b/complete"); + res.end(JSON.stringify({ build })); + }); + try { + const sdk = new SandboxesService("test", server.url, 1000); + const options = { + imageName: "n", + wait: false, + ...(custom + ? { + builderCpus: 8, + builderMemoryMiB: 16384, + builderScratchMiB: 65536, + } + : {}), + }; + const result = + source === "image" + ? await sdk.buildImageFromDockerImage({ ...options, dockerImage: "local/app" }) + : await sdk.buildImageFromDockerfile({ + ...options, + contextPath: workspace, + remote: source === "remote-dockerfile", + }); + expect(result.id).toBe("b"); + expect(captured).toBeDefined(); + expect(captured!.vcpus).toBe(custom ? 8 : undefined); + expect(captured!.memMiB).toBe(custom ? 16384 : undefined); + expect(captured!.scratchMiB).toBe(custom ? 65536 : undefined); + expect(captured).not.toHaveProperty("builderCpus"); + } finally { + await server.close(); + } + }); + + test("streams docker save into verified reusable layers plus a manifest", async () => { + const archive = await writeTar([ + ["config.json", configBytes], + ["layer.tar", layerBytes], + ["manifest.json", saveManifest], + ]); + const digest = `sha256:${sha256(configBytes)}`; + installFakeDocker( + { Id: digest, Os: "linux", Architecture: "amd64", Config: { User: "node", Env: ["A=1"] } }, + archive + ); + await expect(dockerImageDigest("local/app:latest")).resolves.toBe(digest); + const source = await prepareDockerImageManifestSource("local/app:latest"); + expect(source.imageDigest).toBe(digest); + expect(source.imageConfigUser).toBe("node"); + expect(source.imageInit).toEqual({ env: { A: "1" } }); + await source.cleanup(); + + const packaged = await packageDockerImageManifest("local/app:latest", digest, source.config, { + tempDir: workspace, + }); + try { + expect(packaged.artifact.inputFormat).toBe("docker_image_manifest_v1"); + expect(packaged.manifest.imageDigest).toBe(digest); + expect(packaged.manifest.config.sha256).toBe(sha256(configBytes)); + expect(packaged.manifest.layers.map((layer) => layer.sha256)).toEqual([sha256(layerBytes)]); + expect(readFileSync(packaged.layers[sha256(layerBytes)].path)).toEqual(layerBytes); + } finally { + packaged.cleanup(); + } + }); + + test("rejects non-amd64 local images with rebuild guidance", async () => { + const archive = await writeTar([]); + installFakeDocker( + { Id: `sha256:${"a".repeat(64)}`, Os: "linux", Architecture: "arm64", Config: {} }, + archive + ); + await expect(dockerImageDigest("local/app:latest")).rejects.toThrow(/expected linux\/amd64/); + }); + + test.each(["../escape.tar", "/abs/layer.tar", "dir/../layer.tar"])( + "rejects unsafe docker save entry path %s", + async (unsafe) => { + const archive = await writeTar([ + ["config.json", configBytes], + [unsafe, layerBytes], + [ + "manifest.json", + Buffer.from(JSON.stringify([{ Config: "config.json", Layers: [unsafe] }])), + ], + ]); + const digest = `sha256:${sha256(configBytes)}`; + installFakeDocker({ Id: digest, Os: "linux", Architecture: "amd64", Config: {} }, archive); + await expect( + packageDockerImageManifest("local/app:latest", digest, {}, { tempDir: workspace }) + ).rejects.toThrow(/unsafe entry path/); + } + ); + + test("rejects a config that does not match the inspected digest", async () => { + const archive = await writeTar([ + ["config.json", configBytes], + ["layer.tar", layerBytes], + ["manifest.json", saveManifest], + ]); + const digest = `sha256:${"b".repeat(64)}`; + installFakeDocker({ Id: digest, Os: "linux", Architecture: "amd64", Config: {} }, archive); + await expect( + packageDockerImageManifest("local/app:latest", digest, {}, { tempDir: workspace }) + ).rejects.toThrow(); + }); + + test("rejects a failing docker save", async () => { + const archive = await writeTar([ + ["config.json", configBytes], + ["layer.tar", layerBytes], + ["manifest.json", saveManifest], + ]); + const digest = `sha256:${sha256(configBytes)}`; + installFakeDocker({ Id: digest, Os: "linux", Architecture: "amd64", Config: {} }, archive, 3); + await expect( + packageDockerImageManifest("local/app:latest", digest, {}, { tempDir: workspace }) + ).rejects.toThrow(); + }); +}); + +describe("image init derivation", () => { + test("derives env, args and working dir while excluding reserved variables", () => { + const init = deriveAutoImageInit({ + Env: ["PATH=/usr/bin", "HOME=/root", "APP=1", "SANDBOX_ENABLED=x"], + Cmd: ["node", "server.js"], + WorkingDir: "/srv", + }); + expect(init?.workingDir).toBe("/srv"); + expect(init?.env).toMatchObject({ APP: "1" }); + expect(init?.env?.SANDBOX_ENABLED).toBeUndefined(); + expect(init?.env?.HOME).toBeUndefined(); + expect(init?.args ?? init?.command).toBeTruthy(); + }); + + test("explicit values override automatic defaults", () => { + const merged = mergeImageInit( + { env: { A: "auto", B: "auto" }, workingDir: "/auto" }, + { env: { B: "explicit" }, command: "/bin/sh" } + ); + expect(merged).toMatchObject({ + env: { A: "auto", B: "explicit" }, + workingDir: "/auto", + command: "/bin/sh", + }); + expect(mergeImageInit(undefined, undefined)).toBeUndefined(); + }); +}); diff --git a/tests/integration/docker-lifecycle-conformance.test.ts b/tests/integration/docker-lifecycle-conformance.test.ts new file mode 100644 index 0000000..d4aa506 --- /dev/null +++ b/tests/integration/docker-lifecycle-conformance.test.ts @@ -0,0 +1,160 @@ +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { SandboxesService } from "../../src/services/sandboxes"; +import { + dockerImageDigest, + packageDockerImageManifest, +} from "../../src/sandbox/image-build/docker-image"; + +let root: string; +let originalPath: string | undefined; +const digest = "sha256:" + "a".repeat(64); +beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), "hb-docker-lifetime-")); + originalPath = process.env.PATH; + mkdirSync(path.join(root, "bin")); + writeFileSync(path.join(root, "Dockerfile"), "FROM scratch\n"); +}); +afterEach(() => { + process.env.PATH = originalPath; + vi.restoreAllMocks(); + rmSync(root, { recursive: true, force: true }); +}); +function docker(body: string) { + const script = path.join(root, "bin", "docker"); + writeFileSync( + script, + `#!${process.execPath}\nconst fs = require('fs'); const args = process.argv.slice(2); const root = ${JSON.stringify(root)}; fs.appendFileSync(root+'/calls', JSON.stringify(args)+'\\n');\n${body}\n` + ); + chmodSync(script, 0o755); + process.env.PATH = `${path.dirname(script)}:${originalPath}`; +} +function calls(): string[][] { + return readFileSync(path.join(root, "calls"), "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); +} +const service = () => new SandboxesService("local", "http://unused", 1000); + +test.each([undefined, "owned:tag"])( + "local build failure cleans only generated tags: %s", + async (dockerTag) => { + docker(`if (args[0] === 'buildx') { console.error('build failed'); process.exit(4); }`); + await expect( + service().buildImageFromDockerfile({ + contextPath: root, + imageName: "n", + remote: false, + dockerTag, + buildArgs: { A: "value with spaces" }, + }) + ).rejects.toThrow("build failed"); + const trace = calls(); + expect(trace[0]).toContain("linux/amd64"); + expect(trace[0]).toContain("value with spaces".replace(/^/, "A=")); + expect(trace[0]).toContain("--load"); + expect(trace.length).toBe(dockerTag ? 1 : 2); + if (!dockerTag) expect(trace[1].slice(0, 2)).toEqual(["image", "rm"]); + } +); +test.each([ + ['"--platform" requires API version 1.49', true], + ["unknown flag: --platform", true], + ["No such image", false], + ["Cannot connect to the Docker daemon", false], +] as const)("Docker inspection failure precedes HTTP: %s", async (message, unsupported) => { + docker(`console.error(${JSON.stringify(message)}); process.exit(2);`); + const sdk = service(); + const lookup = vi.spyOn(sdk, "findReadyImage"); + await expect(sdk.getOrBuildImage({ dockerImage: "local/app" })).rejects.toThrow( + unsupported ? "API 1.49" : message + ); + expect(lookup).not.toHaveBeenCalled(); + expect(calls()).toHaveLength(1); +}); +test("inspection platform comparison is case insensitive", async () => { + docker( + `console.log(JSON.stringify({ Id: ${JSON.stringify(digest)}, Os: 'Linux', Architecture: 'AMD64', Config: {} }));` + ); + expect(await dockerImageDigest("image")).toBe(digest); +}); +test("Docker identity mutation during lookup fails before importing", async () => { + writeFileSync(path.join(root, "digest"), digest); + docker( + `console.log(JSON.stringify({ Id: fs.readFileSync(root+'/digest','utf8'), Os: 'linux', Architecture: 'amd64', Config: {} }));` + ); + const sdk = service(); + vi.spyOn(sdk, "findReadyImage").mockImplementation(async () => { + writeFileSync(path.join(root, "digest"), "sha256:" + "b".repeat(64)); + return null; + }); + const reuse = vi.spyOn(sdk, "reuseDockerImage"); + await expect(sdk.getOrBuildImage({ dockerImage: "local/app" })).rejects.toThrow( + "Docker image changed" + ); + expect(reuse).not.toHaveBeenCalled(); + expect(calls()).toHaveLength(2); +}); +test("known digest cache hit requires no local Docker", async () => { + docker("throw new Error('Docker must not run');"); + const sdk = service(); + vi.spyOn(sdk, "findReadyImage").mockResolvedValue({ + id: "image", + imageName: "n", + namespace: "ns", + uploaded: true, + createdAt: "", + updatedAt: "", + }); + expect( + await sdk.getOrBuildImage({ dockerImage: "local/app", expectedImageDigest: digest }) + ).toMatchObject({ outcome: "reused", imageId: "image" }); + expect(readdirSync(root)).not.toContain("calls"); +}); +test("container inspection fallback and exact reuse preserve env and remove owned container", async () => { + docker(` +if (args[0] === 'image') process.exit(1); +if (args[0] === 'create') console.log('owned-container'); +else if (args[0] === 'container') console.log(args.includes('{{.Image}}') ? ${JSON.stringify(digest)} : JSON.stringify({ User:'node', Env:['PATH=/usr/local/bin','APP=1'], Cmd:['node'], WorkingDir:'/app' })); +else if (args[0] !== 'rm') process.exit(2);`); + const sdk = service(); + const reuse = vi + .spyOn(sdk, "reuseDockerImage") + .mockResolvedValue({ + hit: true, + build: { id: "b", imageName: "n", status: "completed", imageId: "image" }, + }); + expect((await sdk.buildImageFromDockerImage({ dockerImage: "app", imageName: "n" })).id).toBe( + "b" + ); + expect(reuse.mock.calls[0][0]).toMatchObject({ + imageConfigUser: "node", + imageInit: { env: { APP: "1" }, workingDir: "/app" }, + }); + expect(calls().at(-1)).toEqual(["rm", "-f", "owned-container"]); + expect(calls().some((args) => args.includes("save"))).toBe(false); +}); +test("archive parse failure terminates and reaps Docker save before removing workspace", async () => { + docker(` +fs.writeFileSync(root+'/pid',String(process.pid)); +process.on('SIGTERM', () => { fs.writeFileSync(root+'/terminated','yes'); process.exit(0); }); +process.stdout.write(Buffer.alloc(512, 0x61)); +setInterval(() => {}, 1000);`); + await expect(packageDockerImageManifest("app", digest, {}, { tempDir: root })).rejects.toThrow(); + const pid = Number(readFileSync(path.join(root, "pid"), "utf8")); + expect(() => process.kill(pid, 0)).toThrow(); + expect(readdirSync(root).filter((entry) => entry.startsWith("hb-docker-image-layers-"))).toEqual( + [] + ); +}); diff --git a/tests/integration/file-watch.test.ts b/tests/integration/file-watch.test.ts new file mode 100644 index 0000000..7004a69 --- /dev/null +++ b/tests/integration/file-watch.test.ts @@ -0,0 +1,221 @@ +import { afterEach, expect, test } from "vitest"; +import { WebSocketServer } from "ws"; +import { SandboxFilesApi } from "../../src/sandbox/files"; +import { RuntimeTransport } from "../../src/sandbox/base"; +import { openRuntimeWebSocket } from "../../src/sandbox/ws"; +import { localHTTP, delay } from "../helpers/local-http"; + +const cleanup: Array<() => Promise> = []; +afterEach(async () => { + for (const close of cleanup.splice(0)) await close(); +}); +const status = { + id: "w", + path: "/tmp", + recursive: true, + active: true, + createdAt: 1, + oldestSeq: 0, + lastSeq: 0, + eventCount: 0, +}; +async function setup(done: boolean) { + const calls: Array<{ method: string; path: string; body: unknown }> = []; + const server = await localHTTP(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + calls.push({ method: req.method!, path: req.url!, body: body ? JSON.parse(body) : undefined }); + res.end( + JSON.stringify({ + watch: { + ...status, + ...(req.url?.includes("includeEvents") + ? { events: [{ seq: 3, path: "/tmp/a", op: "WRITE", timestamp: 2 }], lastSeq: 3 } + : {}), + }, + }) + ); + }); + const ws = new WebSocketServer({ server: server.server }); + const targets: string[] = []; + ws.on("connection", (socket, request) => { + targets.push(request.url!); + socket.send( + JSON.stringify({ + type: "event", + event: { seq: 4, path: "/tmp/a", op: "WRITE", timestamp: 3 }, + }) + ); + if (done) + socket.send( + JSON.stringify({ + type: "done", + status: { ...status, active: false, lastSeq: 4, stoppedAt: 4 }, + }) + ); + }); + cleanup.push(async () => { + for (const socket of ws.clients) socket.terminate(); + await new Promise((resolve) => ws.close(() => resolve())); + await server.close(); + }); + const connection = { sandboxId: "s", baseUrl: server.url, token: "test" }; + const transport = new RuntimeTransport(async () => connection, 1000); + return { calls, targets, ws, files: new SandboxFilesApi(transport, async () => connection) }; +} +test("low-level watch resumes by ID/cursor, refreshes events, and yields done status", async () => { + const api = await setup(true); + const watch = await api.files.getWatch("w", true); + expect(watch.current.events?.[0].seq).toBe(3); + const copy = watch.current; + copy.events![0].seq = 99; + expect(watch.current.events![0].seq).toBe(3); + const messages = []; + for await (const message of watch.events({ cursor: 3, route: "stream" })) messages.push(message); + expect(messages.map((message) => message.type)).toEqual(["event", "done"]); + expect(watch.current).toMatchObject({ active: false, lastSeq: 4, stoppedAt: 4 }); + expect(api.targets[0]).toContain("/stream?sessionId=s&cursor=3"); + await watch.refresh(true); + expect(watch.toJSON().events?.[0].seq).toBe(3); + expect(api.calls.map((call) => call.path)).toEqual([ + "/sandbox/files/watch/w?includeEvents=true", + "/sandbox/files/watch/w?includeEvents=true", + ]); +}); +test("breaking watch iteration releases the WebSocket", async () => { + const api = await setup(false); + const watch = await api.files.withRunAs("root").watch("/tmp", { recursive: true }); + for await (const message of watch.events()) { + expect(message.type).toBe("event"); + // Match the receiver and Python: oldestSeq=0 is an empty-buffer sentinel. + expect(watch.current.oldestSeq).toBe(4); + break; + } + await delay(20); + expect(api.ws.clients.size).toBe(0); + expect(api.calls[0].body).toEqual({ path: "/tmp", recursive: true, runAs: "root" }); + await watch.stop(); + expect(watch.current.active).toBe(false); + expect(api.calls[1].method).toBe("DELETE"); +}); +test("callback watch receives file events and ends cleanly on a done envelope", async () => { + const api = await setup(true); + const events: unknown[] = []; + let exited!: (error?: Error) => void; + const exit = new Promise((resolve) => (exited = resolve)); + await api.files.watchDir( + "/tmp", + (event) => { + events.push(event); + }, + { onExit: exited } + ); + expect(await exit).toBeUndefined(); + expect(events).toEqual([{ type: "write", name: "a" }]); +}); +test("watch iteration supports caller cancellation", async () => { + const api = await setup(false); + const watch = await api.files.watch("/tmp"); + const controller = new AbortController(); + const events = watch.events({ signal: controller.signal }); + await events.next(); + controller.abort(); + await expect(events.next()).rejects.toMatchObject({ code: "request_aborted" }); + await delay(20); + expect(api.ws.clients.size).toBe(0); +}); + +test("watch cancellation also interrupts a stalled WebSocket handshake", async () => { + const server = await localHTTP((_req, res) => res.end(JSON.stringify({ watch: status }))); + server.server.on("upgrade", () => {}); + cleanup.push(server.close); + const connection = { sandboxId: "s", baseUrl: server.url, token: "test" }; + const files = new SandboxFilesApi( + new RuntimeTransport(async () => connection), + async () => connection + ); + const watch = await files.getWatch("w"); + const controller = new AbortController(); + const next = watch.events({ signal: controller.signal }).next(); + const rejected = expect(next).rejects.toMatchObject({ + code: "request_aborted", + retryable: false, + }); + await delay(20); + controller.abort(); + await rejected; +}); + +test("a stalled rejected handshake body respects the connection deadline", async () => { + const server = await localHTTP((_req, res) => { + res.writeHead(403); + res.write("{"); + }); + cleanup.push(server.close); + await expect( + openRuntimeWebSocket({ url: server.url.replace("http:", "ws:") }, {}, { timeoutMs: 40 }) + ).rejects.toMatchObject({ code: "request_timeout", service: "runtime" }); +}); + +test("a callback can stop its own watcher without waiting on itself", async () => { + const api = await setup(false); + let exited!: () => void; + const exit = new Promise((resolve) => { + exited = resolve; + }); + const handle = await api.files.watchDir( + "/tmp", + async () => { + await handle.stop(); + }, + { onExit: exited } + ); + await exit; + await delay(20); + expect(api.ws.clients.size).toBe(0); +}); + +test.each([false, true])("terminal attachment preserves immediate frames (unknown event=%s)", async (unknownEvent) => { + const { SandboxTerminalHandle } = await import("../../src/sandbox/terminal"); + const server = await localHTTP((_req, res) => res.end()); + const ws = new WebSocketServer({ server: server.server }); + const status = { + id: "p", + command: "sh", + cwd: "/tmp", + running: false, + rows: 24, + cols: 80, + startedAt: 1, + exitCode: 0, + }; + ws.on("connection", (socket) => { + if (unknownEvent) socket.send(JSON.stringify({ type: "keepalive" })); + socket.send( + JSON.stringify({ + type: "output", + seq: 1, + data: Buffer.from("first").toString("base64"), + timestamp: 1, + }) + ); + socket.send(JSON.stringify({ type: "exit", status })); + socket.close(); + }); + cleanup.push(async () => { + for (const socket of ws.clients) socket.terminate(); + await new Promise((resolve) => ws.close(() => resolve())); + await server.close(); + }); + const connection = { sandboxId: "s", baseUrl: server.url, token: "t" }; + const handle = new SandboxTerminalHandle( + new RuntimeTransport(async () => connection), + async () => connection, + status + ); + const terminal = await handle.attach(0); + const events = []; + for await (const event of terminal.events()) events.push(event); + expect(events.map((event) => event.type)).toEqual(["output", "exit"]); + expect(events[0]).toMatchObject({ data: "first" }); +}); diff --git a/tests/integration/image-build-conformance.test.ts b/tests/integration/image-build-conformance.test.ts new file mode 100644 index 0000000..3386d23 --- /dev/null +++ b/tests/integration/image-build-conformance.test.ts @@ -0,0 +1,361 @@ +import { mkdtempSync, rmSync, writeFileSync, readdirSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { afterEach, expect, test, vi } from "vitest"; +import { SandboxesService } from "../../src/services/sandboxes"; +import { HyperbrowserError } from "../../src/error"; +import { + uploadImageBuildArtifact, + uploadMissingImageBuildArtifacts, +} from "../../src/sandbox/image-build/upload"; +import { localHTTP, delay } from "../helpers/local-http"; + +const cleanup: Array<() => void | Promise> = []; +afterEach(async () => { + for (const close of cleanup.splice(0).reverse()) await close(); + vi.restoreAllMocks(); +}); +function workspace() { + const dir = mkdtempSync(path.join(tmpdir(), "hb-conformance-")); + cleanup.push(() => rmSync(dir, { recursive: true, force: true })); + writeFileSync(path.join(dir, "Dockerfile"), "FROM scratch\nCOPY data /data\n"); + writeFileSync(path.join(dir, "data"), "payload"); + return dir; +} +function upload(url: string, method = "PUT") { + return { + url, + method, + headers: { "x-test": "header" }, + maxUploadBytes: 100, + objectKey: "key", + expiresInSeconds: 60, + }; +} + +test.each([408, 429, 500, 503, 403])( + "artifact upload retry contract for HTTP %s", + async (status) => { + const received: string[] = []; + const server = await localHTTP(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + expect(req.headers["content-length"]).toBe("7"); + expect(req.headers["x-test"]).toBe("header"); + received.push(Buffer.concat(chunks).toString()); + res.writeHead(received.length === 1 ? status : 200); + res.end("reply"); + }); + cleanup.push(server.close); + const pending = uploadImageBuildArtifact(upload(server.url), path.join(workspace(), "data"), { + timeout: 1, + }); + if (status === 403) await expect(pending).rejects.toThrow("403"); + else await pending; + expect(received).toEqual(Array(status === 403 ? 1 : 2).fill("payload")); + } +); +test("non-idempotent artifact upload never replays and size limits fail before HTTP", async () => { + let calls = 0; + const server = await localHTTP((req, res) => { + calls++; + req.resume(); + res.writeHead(503); + res.end("busy"); + }); + cleanup.push(server.close); + const file = path.join(workspace(), "data"); + await expect(uploadImageBuildArtifact(upload(server.url, "POST"), file)).rejects.toThrow("503"); + await expect( + uploadImageBuildArtifact({ ...upload(server.url), maxUploadBytes: 1 }, file) + ).rejects.toThrow("limit"); + expect(calls).toBe(1); +}); +test("upload response inactivity times out, closes sockets, and bounds PUT retries", async () => { + let calls = 0; + const server = await localHTTP((req, res) => { + calls++; + req.resume(); + res.writeHead(200); + res.write("partial"); + }); + cleanup.push(server.close); + await expect( + uploadImageBuildArtifact(upload(server.url), path.join(workspace(), "data"), { timeout: 0.02 }) + ).rejects.toThrow("inactivity"); + await delay(15); + expect(calls).toBe(3); + expect(server.sockets.size).toBe(0); +}); +test.each(["unknown", "duplicate", "method", "size"])( + "invalid selective upload %s fails before any request", + async (kind) => { + const file = path.join(workspace(), "data"); + const digest = "a".repeat(64); + const artifact = { + path: file, + sha256Hex: digest, + sizeBytes: 7, + inputFormat: "docker_image_manifest_v1" as const, + sourcePlatform: "linux/amd64" as const, + imageConfigUser: "", + }; + const entry = { ...upload("http://unused.invalid"), sha256: digest }; + const requests = + kind === "duplicate" + ? [entry, entry] + : [ + { + ...entry, + ...(kind === "unknown" ? { sha256: "b".repeat(64) } : {}), + ...(kind === "method" ? { method: "POST" } : {}), + ...(kind === "size" ? { maxUploadBytes: 1 } : {}), + }, + ]; + await expect( + uploadMissingImageBuildArtifacts(requests, { [digest]: artifact }, { label: "layer" }) + ).rejects.toThrow(/unknown|duplicate|unsupported|limit/); + } +); +test("ready lookup searches later pages and requires an exact match", async () => { + const service = new SandboxesService("local", "http://unused", 1000); + const list = vi + .spyOn(service, "listImages") + .mockResolvedValueOnce({ + images: Array.from({ length: 100 }, (_, i) => ({ + id: String(i), + imageName: `other${i}`, + namespace: "ns", + uploaded: true, + createdAt: "", + updatedAt: "", + })), + totalCount: 101, + }) + .mockResolvedValueOnce({ + images: [ + { + id: "correct", + imageName: "target", + namespace: "ns", + uploaded: false, + ready: true, + createdAt: "", + updatedAt: "", + }, + ], + totalCount: 101, + }); + expect((await service.findReadyImage("target"))?.id).toBe("correct"); + expect(list.mock.calls.map(([params]) => params?.page)).toEqual([1, 2]); +}); +test.each([ + {}, + { contextPath: ".", dockerImage: "image" }, + { contextPath: ".", expectedImageDigest: "sha256:" + "a".repeat(64) }, + { dockerImage: "image", expectedContextFingerprint: "a".repeat(64) }, + { dockerImage: "image", remoteFullContext: true }, +])("invalid resolution options fail before HTTP or Docker: %j", async (options) => { + const service = new SandboxesService("local", "http://unused", 1000); + const find = vi.spyOn(service, "findReadyImage"); + await expect(service.getOrBuildImage(options)).rejects.toThrow(); + expect(find).not.toHaveBeenCalled(); +}); +test("public build fingerprint rejection precedes API calls and local Docker builds", async () => { + const service = new SandboxesService("local", "http://unused", 1000); + const create = vi.spyOn(service, "createImageBuild"); + const contextPath = workspace(); + await expect( + service.buildImageFromDockerfile({ + contextPath, + imageName: "n", + expectedContextFingerprint: "a".repeat(64), + }) + ).rejects.toThrow(/changed/i); + await expect( + service.buildImageFromDockerfile({ + contextPath, + imageName: "n", + remote: false, + expectedContextFingerprint: "a".repeat(64), + }) + ).rejects.toThrow(/remote/); + expect(create).not.toHaveBeenCalled(); +}); +test.each(["upload verification", "already in progress"])( + "completion race recovers: %s", + async (message) => { + const service = new SandboxesService("local", "http://unused", 1000); + const build = { id: "b", imageName: "n", status: "building" as const }; + vi.spyOn(service, "createImageBuild").mockResolvedValue({ build, uploads: [] }); + const complete = vi + .spyOn(service, "completeImageBuild") + .mockRejectedValueOnce(new HyperbrowserError(message, { statusCode: 409 })) + .mockResolvedValue(build); + vi.spyOn(service, "getImageBuild").mockResolvedValue( + message === "already in progress" ? build : { ...build, status: "awaiting_upload" } + ); + const cancel = vi.spyOn(service, "cancelImageBuild"); + const dir = workspace(); + expect( + ( + await service.buildImageFromDockerfile({ + contextPath: dir, + imageName: "n", + wait: false, + tempDir: dir, + }) + ).id + ).toBe("b"); + expect(complete).toHaveBeenCalledTimes(message === "already in progress" ? 1 : 2); + expect(cancel).not.toHaveBeenCalled(); + expect(readdirSync(dir).sort()).toEqual(["Dockerfile", "data"]); + } +); +test("submission failures cancel once and clean artifacts even when cancellation fails", async () => { + const service = new SandboxesService("local", "http://unused", 1000); + vi.spyOn(service, "createImageBuild").mockResolvedValue({ + build: { id: "b", imageName: "n", status: "awaiting_upload" }, + uploads: [{ ...upload("http://unused"), sha256: "bad" }], + }); + const cancel = vi + .spyOn(service, "cancelImageBuild") + .mockRejectedValue(new Error("cleanup failed")); + const dir = workspace(); + await expect( + service.buildImageFromDockerfile({ contextPath: dir, imageName: "n", tempDir: dir }) + ).rejects.toThrow("unknown"); + expect(cancel).toHaveBeenCalledExactlyOnceWith("b"); + expect(readdirSync(dir).sort()).toEqual(["Dockerfile", "data"]); +}); + +test.each(["cancel-one", "timeout-one", "cancel-both"])( + "concurrent resolution callers remain independent: %s", + async (leave) => { + const contextPath = workspace(); + const methods: string[] = []; + let name = ""; + let created = false; + let release = false; + let firstPoll!: () => void; + let secondPoll!: () => void; + const firstPolling = new Promise((resolve) => { + firstPoll = resolve; + }); + const secondPolling = new Promise((resolve) => { + secondPoll = resolve; + }); + let polls = 0; + const build = (status = "building") => ({ + id: "b", + imageName: name, + status, + imageId: "image", + metadata: { inputFormat: "dockerfile_context_manifest_v1", sourcePlatform: "linux/amd64" }, + }); + const server = await localHTTP(async (req, res) => { + const parts: Buffer[] = []; + for await (const chunk of req) parts.push(Buffer.from(chunk)); + const pathname = new URL(req.url!, "http://local").pathname; + methods.push(`${req.method} ${pathname}`); + if (pathname === "/api/images") { + res.end('{"images":[]}'); + return; + } + if (pathname === "/api/images/builds") { + name = JSON.parse(Buffer.concat(parts).toString()).imageName; + if (created) { + res.writeHead(409); + res.end( + JSON.stringify({ code: "image_build_in_progress", message: "building", build: build() }) + ); + } else { + created = true; + res.end(JSON.stringify({ build: build("awaiting_upload"), uploads: [] })); + } + return; + } + if (pathname.endsWith("/complete")) { + res.end(JSON.stringify({ build: build() })); + return; + } + polls++; + firstPoll(); + if (polls >= 2) secondPoll(); + while (!release && !res.destroyed) await delay(5); + if (!res.destroyed) res.end(JSON.stringify({ build: build("completed") })); + }); + cleanup.push(server.close); + const service = new SandboxesService("local", server.url, 2000); + const a = new AbortController(); + const b = new AbortController(); + const first = service + .getOrBuildImage({ + contextPath, + waitTimeout: leave === "timeout-one" ? 0.1 : 5, + signal: a.signal, + }) + .catch((error) => error); + await firstPolling; + const second = service + .getOrBuildImage({ contextPath, waitTimeout: 5, signal: b.signal }) + .catch((error) => error); + try { + await secondPolling; + if (leave !== "timeout-one") a.abort(); + expect((await first).code).toBe(leave === "timeout-one" ? "wait_timeout" : "request_aborted"); + if (leave === "cancel-both") { + b.abort(); + expect((await second).code).toBe("request_aborted"); + } else { + release = true; + expect(await second).toMatchObject({ outcome: "joined", imageId: "image" }); + } + expect(methods.filter((entry) => entry.endsWith("/complete"))).toHaveLength(1); + expect(methods.some((entry) => entry.endsWith("/cancel"))).toBe(false); + } finally { + release = true; + a.abort(); + b.abort(); + await Promise.allSettled([first, second]); + } + } +); + +test.each(["list", "listImages", "listSnapshots"] as const)( + "pagination validates bounds before HTTP: %s", + async (method) => { + const sdk = new SandboxesService("local", "http://unused", 1000); + for (const params of [{ page: 0 }, { page: 1.5 }, { limit: -1 }, { limit: NaN }]) + await expect(sdk[method](params)).rejects.toThrow(/positive integer/); + if (method !== "list") await expect(sdk[method]({ limit: 101 })).rejects.toThrow("at most 100"); + } +); +test.each(["linux/arm64", "LINUX/AMD64", " linux/amd64 "])( + "raw image build platform literals reject %s", + async (sourcePlatform) => { + const sdk = new SandboxesService("local", "http://unused", 1000); + await expect( + sdk.createImageBuild({ + imageName: "n", + inputSha256: "a", + inputSizeBytes: 1, + sourcePlatform, + } as never) + ).rejects.toThrow("sourcePlatform"); + await expect( + sdk.reuseDockerImage({ imageName: "n", sourceImageDigest: "a", sourcePlatform } as never) + ).rejects.toThrow("sourcePlatform"); + } +); +test("raw image build formats are validated before submission or completion", async () => { + const sdk = new SandboxesService("local", "http://unused", 1000); + const params = { + imageName: "n", + inputSha256: "a", + inputSizeBytes: 1, + inputFormat: "ROOTFS_EXPORT_TAR_GZ", + } as never; + await expect(sdk.createImageBuild(params)).rejects.toThrow("inputFormat"); + await expect(sdk.completeImageBuild("b", params)).rejects.toThrow("inputFormat"); +}); diff --git a/tests/integration/image-build-failures.test.ts b/tests/integration/image-build-failures.test.ts new file mode 100644 index 0000000..2ed265c --- /dev/null +++ b/tests/integration/image-build-failures.test.ts @@ -0,0 +1,165 @@ +import { mkdtempSync, rmSync, writeFileSync, existsSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { writeGzipTar } from "../../src/sandbox/image-build/gzip"; +import { uploadMissingImageBuildArtifacts } from "../../src/sandbox/image-build/upload"; +import { DockerImageBuildArtifact } from "../../src/sandbox/image-build/artifacts"; +import { SandboxesService } from "../../src/services/sandboxes"; +import { localHTTP, delay } from "../helpers/local-http"; + +const cleanup: Array<() => void | Promise> = []; +afterEach(async () => { + for (const close of cleanup.splice(0)) await close(); + vi.restoreAllMocks(); +}); +function workspace() { + const dir = mkdtempSync(path.join(tmpdir(), "hb-failure-")); + cleanup.push(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} +describe("image build failure lifetimes", () => { + test("packaging source errors are catchable without an unhandled compressor rejection", async () => { + await expect( + writeGzipTar(path.join(workspace(), "bundle.tgz"), async (writer) => { + await writer.addEntry( + { name: "removed", type: "file", mode: 0o644, size: 3, linkname: "" }, + (async function* () { + yield Buffer.from("a"); + throw new Error("source disappeared"); + })() + ); + }) + ).rejects.toThrow("source disappeared"); + await delay(15); // Vitest also rejects any background unhandled rejection. + }); + test("destination write errors do not escape as uncaught stream errors", async () => { + const file = path.join(workspace(), "exists"); + writeFileSync(file, "keep"); + await expect(writeGzipTar(file, async () => undefined)).rejects.toThrow(); + await delay(15); + expect(existsSync(file)).toBe(true); + }); + test("failed uploads settle active workers and stop scheduling before cleanup", async () => { + const requested: string[] = []; + let pending = 0; + const server = await localHTTP((req, res) => { + requested.push(req.url!); + pending++; + res.once("close", () => pending--); + req.resume(); + req.on("end", () => { + if (req.url === "/0") { + res.writeHead(400); + res.end("rejected"); + } else setTimeout(() => res.end("ok"), 35); + }); + }); + cleanup.push(server.close); + const dir = workspace(); + const artifacts: Record = {}; + const uploads = Array.from({ length: 8 }, (_, i) => { + const sha256 = String(i).repeat(64); + const file = path.join(dir, String(i)); + writeFileSync(file, "data"); + artifacts[sha256] = { + path: file, + sha256Hex: sha256, + sizeBytes: 4, + inputFormat: "docker_image_manifest_v1", + sourcePlatform: "linux/amd64", + imageConfigUser: "", + }; + return { + sha256, + url: `${server.url}/${i}`, + headers: {}, + method: "PUT", + maxUploadBytes: 100, + objectKey: String(i), + expiresInSeconds: 60, + }; + }); + await expect( + uploadMissingImageBuildArtifacts(uploads, artifacts, { label: "layer", timeout: 1 }) + ).rejects.toThrow("rejected"); + rmSync(dir, { recursive: true, force: true }); + await delay(20); + expect(pending).toBe(0); + expect(requested.length).toBe(4); + }); + test("resolution defaults uploads to 600 seconds and preserves explicit null", async () => { + const service = new SandboxesService("local", "http://unused", 1000); + vi.spyOn(service, "findReadyImage").mockResolvedValue(null); + const build = vi + .spyOn(service, "buildImageFromDockerfile") + .mockResolvedValue({ id: "b", imageName: "n", status: "building" }); + const input = { + contextPath: "/unused", + expectedContextFingerprint: "a".repeat(64), + wait: false, + }; + await service.getOrBuildImage(input); + expect(build.mock.calls[0][0].uploadTimeout).toBe(600); + await service.getOrBuildImage({ ...input, uploadTimeout: null }); + expect(build.mock.calls[1][0].uploadTimeout).toBeNull(); + }); + test("build wait deadline includes control retries without canceling the backend build", async () => { + const methods: string[] = []; + const server = await localHTTP((req, res) => { + methods.push(req.method!); + res.writeHead(503); + res.end('{"message":"retry"}'); + }); + cleanup.push(server.close); + const service = new SandboxesService("local", server.url, 1000); + const start = Date.now(); + await expect(service.waitForImageBuild("build", { timeout: 0.03 })).rejects.toMatchObject({ + code: "wait_timeout", + }); + expect(Date.now() - start).toBeLessThan(300); + expect(methods).toEqual(["GET"]); + }); + test("independent waiters can be canceled without canceling another waiter", async () => { + const server = await localHTTP((_req, res) => { + res.end('{"build":{"id":"b","imageName":"n","status":"building"}}'); + }); + cleanup.push(server.close); + const service = new SandboxesService("local", server.url, 1000); + const controller = new AbortController(); + const first = service + .waitForImageBuild("b", { signal: controller.signal, pollInterval: 0.01 }) + .catch((error) => error); + const second = service + .waitForImageBuild("b", { timeout: 0.05, pollInterval: 0.01 }) + .catch((error) => error); + controller.abort(); + expect((await first).code).toBe("request_aborted"); + expect((await second).code).toBe("wait_timeout"); + }); + test.each(["failed", "canceled"] as const)( + "an observed %s build is not masked by a racing deadline", + async (status) => { + const service = new SandboxesService("local", "http://unused.test", 1000); + const terminal = { + id: "b", + imageName: "n", + status, + errorCode: "builder_terminal", + errorMessage: "Build terminated", + }; + vi.spyOn(service, "getImageBuild").mockImplementationOnce(async (_id, options) => { + // Model a status response becoming available as cancellation is delivered. + await new Promise((resolve) => + options!.signal!.addEventListener("abort", () => resolve(), { once: true }) + ); + return terminal; + }); + await expect(service.waitForImageBuild("b", { timeout: 0.001 })).rejects.toMatchObject({ + code: "builder_terminal", + details: terminal, + message: "[Hyperbrowser]: Build terminated", + }); + } + ); +}); diff --git a/tests/integration/image-resolution.test.ts b/tests/integration/image-resolution.test.ts new file mode 100644 index 0000000..c1cd24b --- /dev/null +++ b/tests/integration/image-resolution.test.ts @@ -0,0 +1,221 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import type { RequestInit } from "node-fetch"; +import { HyperbrowserError } from "../../src/client"; +import { DockerBuildContextChangedError } from "../../src/sandbox/image-build/context"; +import { SandboxesService } from "../../src/services/sandboxes"; + +type Request = { method: string; path: string; body?: Record; params?: Record }; + +class Backend { + name: string | null = null; + requests: Request[] = []; + polls = 0; + transform: (payload: Record) => Record = (v) => v; + onListImages: (() => void) | null = null; + imageRow: Record = { uploaded: false, ready: true }; + + constructor(private readonly options: { ready?: boolean; conflict?: boolean } = {}) {} + + build(status = "building"): Record { + return { + id: "build-1", + imageName: this.name, + imageId: "image-1", + status, + metadata: { inputFormat: "dockerfile_context_manifest_v1", sourcePlatform: "linux/amd64" }, + }; + } + + respond(request: Request): unknown { + this.requests.push(request); + if (request.method === "GET" && request.path === "/images") { + this.name = String(request.params?.search); + this.onListImages?.(); + const image = { + id: "image-1", + imageName: this.name, + namespace: "team-local", + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + ...this.imageRow, + }; + return { images: this.options.ready ? [image] : [] }; + } + if (request.method === "POST" && request.path === "/images/builds") { + this.name = String(request.body?.imageName); + if (this.options.conflict) { + const payload = this.transform({ + message: "already building", + code: "image_build_in_progress", + build: this.build(), + }); + throw new HyperbrowserError(String(payload.message), { + statusCode: 409, + code: payload.code as string, + details: payload, + service: "control", + }); + } + return { build: this.build("awaiting_upload"), uploads: [] }; + } + if (request.method === "POST" && request.path.endsWith("/complete")) { + return { build: this.build() }; + } + if (request.method === "GET" && request.path === "/images/builds/build-1") { + this.polls += 1; + return { build: this.build("completed") }; + } + throw new Error(`unexpected request ${request.method} ${request.path}`); + } +} + +const serviceFor = (backend: Backend): SandboxesService => { + const service = new SandboxesService("local-only", "http://local.test", 30_000); + vi.spyOn(service as unknown as { request: (...args: unknown[]) => Promise }, "request").mockImplementation( + async (...args: unknown[]) => { + const [rawPath, init, params] = args as [string, RequestInit | undefined, Record | undefined]; + const body = typeof init?.body === "string" ? (JSON.parse(init.body) as Record) : undefined; + return backend.respond({ method: init?.method ?? "GET", path: rawPath, body, params }); + } + ); + return service; +}; + +let context: string; +beforeEach(() => { + context = mkdtempSync(path.join(tmpdir(), "hb-resolution-")); + writeFileSync(path.join(context, "Dockerfile"), "FROM scratch\nCOPY data /data\n"); + writeFileSync(path.join(context, "data"), "first"); +}); +afterEach(() => { + rmSync(context, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +describe("getOrBuildImage resolution", () => { + test.each(["ready", "created", "joined", "forced", "detached"])( + "resolves ready, new and concurrent builds: %s", + async (mode) => { + const backend = new Backend({ ready: mode === "ready" || mode === "forced", conflict: mode === "joined" }); + const result = await serviceFor(backend).getOrBuildImage({ + contextPath: context, + forceBuild: mode === "forced", + wait: mode !== "detached", + pollInterval: 0, + }); + expect(result.outcome).toBe({ ready: "reused", joined: "joined" }[mode] ?? "created"); + expect(result.imageName.startsWith("hb__dockerfile__")).toBe(true); + expect(result.imageId).toBe(mode === "detached" ? undefined : "image-1"); + expect(backend.polls).toBe(mode === "ready" || mode === "detached" ? 0 : 1); + if (mode === "ready") { + expect(backend.requests.map((r) => [r.method, r.path])).toEqual([["GET", "/images"]]); + } + if (mode === "forced") { + expect(backend.requests.some((r) => r.path === "/images")).toBe(false); + } + expect(backend.requests.some((r) => r.path.endsWith("/cancel"))).toBe(false); + } + ); + + test.each(["name", "format", "platform", "missing-status", "missing-build", "code"])( + "incompatible conflicts are not joined: %s", + async (mismatch) => { + const backend = new Backend({ conflict: true }); + backend.transform = (payload) => { + const build = payload.build as Record; + const metadata = build.metadata as Record; + if (mismatch === "name") build.imageName = "unrelated"; + else if (mismatch === "format") metadata.inputFormat = "docker_image_manifest_v1"; + else if (mismatch === "platform") metadata.sourcePlatform = "linux/arm64"; + else if (mismatch === "missing-status") delete build.status; + else if (mismatch === "missing-build") delete payload.build; + else payload.code = "different_conflict"; + return payload; + }; + await expect( + serviceFor(backend).getOrBuildImage({ contextPath: context, pollInterval: 0 }) + ).rejects.toMatchObject({ statusCode: 409 }); + expect(backend.polls).toBe(0); + } + ); + + test("context change during lookup fails before submission", async () => { + const backend = new Backend(); + backend.onListImages = () => writeFileSync(path.join(context, "data"), "changed"); + await expect( + serviceFor(backend).getOrBuildImage({ contextPath: context, pollInterval: 0 }) + ).rejects.toBeInstanceOf(DockerBuildContextChangedError); + expect(backend.requests.map((r) => [r.method, r.path])).toEqual([["GET", "/images"]]); + }); + + test.each([ + [true, undefined, true], + [false, true, true], + [false, false, false], + [false, undefined, false], + ])("ready lookup supports old servers (uploaded=%s ready=%s) -> reused=%s", async (uploaded, ready, reused) => { + const backend = new Backend({ ready: true }); + backend.imageRow = { uploaded, ...(ready === undefined ? {} : { ready }) }; + const result = await serviceFor(backend).getOrBuildImage({ contextPath: context, pollInterval: 0 }); + expect(result.outcome).toBe(reused ? "reused" : "created"); + }); + + test("requires exactly one of contextPath or dockerImage", async () => { + const service = serviceFor(new Backend()); + await expect(service.getOrBuildImage({})).rejects.toThrow(); + await expect( + service.getOrBuildImage({ contextPath: context, dockerImage: "node:20" }) + ).rejects.toThrow(); + }); + + test("builder resources are serialized to the API wire names", async () => { + const backend = new Backend(); + await serviceFor(backend).getOrBuildImage({ + contextPath: context, + pollInterval: 0, + builderCpus: 4, + builderMemoryMiB: 8192, + builderScratchMiB: 20480, + }); + const create = backend.requests.find((r) => r.method === "POST" && r.path === "/images/builds"); + expect(create?.body).toMatchObject({ + vcpus: 4, + memMiB: 8192, + scratchMiB: 20480, + inputFormat: "dockerfile_context_manifest_v1", + sourcePlatform: "linux/amd64", + dockerfilePath: "Dockerfile", + }); + expect(create?.body).not.toHaveProperty("builderCpus"); + expect(create?.body?.contextManifest).toBeTruthy(); + }); + + test("waitForImageBuild surfaces failures and timeouts", async () => { + const service = new SandboxesService("local-only", "http://local.test", 30_000); + const statuses = ["building", "failed"]; + vi.spyOn(service, "getImageBuild").mockImplementation(async () => ({ + id: "build-1", + imageName: "x", + status: statuses.shift() as "building", + errorCode: "build_failed", + errorMessage: "boom", + })); + await expect(service.waitForImageBuild("build-1", { pollInterval: 0 })).rejects.toThrow(/boom/); + + vi.spyOn(service, "getImageBuild").mockResolvedValue({ id: "build-1", imageName: "x", status: "building" }); + await expect( + service.waitForImageBuild("build-1", { pollInterval: 0.001, timeout: 0.01 }) + ).rejects.toThrow(/timed out|Timed out/i); + }); + + test("dockerfile builds reject missing context directories", async () => { + const missing = path.join(context, "missing"); + mkdirSync(path.join(context, "other")); + await expect( + serviceFor(new Backend()).getOrBuildImage({ contextPath: missing, pollInterval: 0 }) + ).rejects.toThrow(); + }); +}); diff --git a/tests/integration/python-context-reference.test.ts b/tests/integration/python-context-reference.test.ts new file mode 100644 index 0000000..a873b93 --- /dev/null +++ b/tests/integration/python-context-reference.test.ts @@ -0,0 +1,103 @@ +import { + chmodSync, + lchmodSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { expect, test, vi } from "vitest"; +import { + dockerBuildContextFingerprint, + packageDockerBuildContextManifest, +} from "../../src/sandbox/image-build/context"; +import * as gzip from "../../src/sandbox/image-build/gzip"; +vi.mock("fs", async () => { + const actual = await vi.importActual("fs"); + return { + ...actual, + readFileSync: vi.fn(actual.readFileSync), + readdirSync: vi.fn(actual.readdirSync), + mkdtempSync: vi.fn(actual.mkdtempSync), + }; +}); +const reference: typeof import("../fixtures/python_reference.json") = JSON.parse( + readFileSync(path.join(__dirname, "../fixtures/python_reference.json"), "utf8") +); +test.each(reference.contexts)("Python filesystem fingerprint: $name", async (fixture) => { + const root = mkdtempSync(path.join(tmpdir(), "hb-python-context-")); + try { + for (const directory of fixture.directories) + mkdirSync(path.join(root, directory), { recursive: true, mode: 0o755 }); + for (const [file, content] of Object.entries(fixture.files)) { + if (content === undefined) continue; + const location = path.join(root, file); + mkdirSync(path.dirname(location), { recursive: true, mode: 0o755 }); + writeFileSync(location, content); + chmodSync(location, (fixture.modes as Record)[file] ?? 0o644); + } + for (const [link, target] of Object.entries(fixture.links)) { + if (target !== undefined) { + const location = path.join(root, link); + symlinkSync(target, location); + // macOS applies umask to symlinks; the Python golden uses mode 0777. + if (process.platform === "darwin") lchmodSync(location, 0o777); + expect(lstatSync(location).mode & 0o7777).toBe(0o777); + } + } + const compress = vi.spyOn(gzip, "writeGzipTar"); + try { + expect(await dockerBuildContextFingerprint(root, { forceFullContext: fixture.full })).toBe( + fixture.expected + ); + expect(compress).not.toHaveBeenCalled(); + } finally { + compress.mockRestore(); + } + const packaged = await packageDockerBuildContextManifest(root, { + forceFullContext: fixture.full, + expectedContextFingerprint: fixture.expected, + }); + try { + expect(packaged.fingerprint).toBe(fixture.expected); + } finally { + packaged.cleanup(); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("fingerprinting streams payloads without staging and prunes ignored subtrees", async () => { + const fs = await import("fs"); + const root = mkdtempSync(path.join(tmpdir(), "hb-fingerprint-stream-")); + writeFileSync(path.join(root, "Dockerfile"), "FROM scratch\nCOPY . /app\n"); + writeFileSync(path.join(root, ".dockerignore"), "node_modules\n"); + writeFileSync(path.join(root, "payload"), Buffer.alloc(10 * 1024 * 1024, 1)); + mkdirSync(path.join(root, "node_modules", "nested"), { recursive: true }); + writeFileSync(path.join(root, "node_modules", "nested", "ignored"), "ignored"); + const read = vi.mocked(fs.readFileSync).mockClear(); + const listing = vi.mocked(fs.readdirSync).mockClear(); + const staging = vi.mocked(fs.mkdtempSync).mockClear(); + const compression = vi.spyOn(gzip, "writeGzipTar"); + try { + expect(await dockerBuildContextFingerprint(root)).toMatch(/^[a-f0-9]{64}$/); + expect(read.mock.calls.some(([filename]) => String(filename).endsWith("/payload"))).toBe(false); + expect( + listing.mock.calls.some(([directory]) => String(directory).includes("node_modules")) + ).toBe(false); + expect(staging).not.toHaveBeenCalled(); + expect(compression).not.toHaveBeenCalled(); + } finally { + read.mockClear(); + listing.mockClear(); + staging.mockClear(); + compression.mockRestore(); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tests/integration/python-sse-reference.test.ts b/tests/integration/python-sse-reference.test.ts new file mode 100644 index 0000000..0d2b49c --- /dev/null +++ b/tests/integration/python-sse-reference.test.ts @@ -0,0 +1,28 @@ +import { afterEach, expect, test } from "vitest"; +import fixtures from "../fixtures/python_sse_reference.json"; +import { RuntimeTransport } from "../../src/sandbox/base"; +import { localHTTP } from "../helpers/local-http"; +// Golden decoding results captured from Python SDK 1.9.1 (c36d3d9). +const cleanup: Array<() => Promise> = []; +afterEach(async () => { + for (const close of cleanup.splice(0)) await close(); +}); +test.each(fixtures)("Python SSE decoding: $name", async (fixture) => { + const server = await localHTTP(async (_req, res) => { + res.writeHead(200, { "content-type": "text/event-stream" }); + for (const chunk of fixture.chunks) { + res.write(Buffer.from(chunk, "hex")); + await new Promise((resolve) => setImmediate(resolve)); + } + res.end(); + }); + cleanup.push(server.close); + const transport = new RuntimeTransport( + async () => ({ sandboxId: "s", baseUrl: server.url, token: "local" }), + 2000 + ); + const received = []; + for await (const event of transport.streamSSE("/stream")) + received.push({ ...event, id: event.id ?? null }); + expect(received).toEqual(fixture.expected); +}); diff --git a/tests/integration/python-wire-reference.test.ts b/tests/integration/python-wire-reference.test.ts new file mode 100644 index 0000000..bcda162 --- /dev/null +++ b/tests/integration/python-wire-reference.test.ts @@ -0,0 +1,103 @@ +import { afterEach, expect, test } from "vitest"; +import fixtures from "../fixtures/python_wire_reference.json"; +import { HyperbrowserClient } from "../../src/client"; +import { RuntimeTransport } from "../../src/sandbox/base"; +import { SandboxFilesApi } from "../../src/sandbox/files"; +import { SandboxProcessesApi, SandboxProcessHandle } from "../../src/sandbox/process"; +import { SandboxTerminalApi, SandboxTerminalHandle } from "../../src/sandbox/terminal"; +import { localHTTP } from "../helpers/local-http"; + +// Golden requests and responses captured from Python SDK 1.9.1 (c36d3d9). +const cleanup: Array<() => Promise> = []; +afterEach(async () => { + for (const close of cleanup.splice(0)) await close(); +}); + +function expectedValues(value: unknown): unknown { + if (Array.isArray(value)) return value.map(expectedValues); + if (value && typeof value === "object") + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + key === "modifiedTime" && typeof item === "string" ? new Date(item) : expectedValues(item), + ]) + ); + return value; +} + +test.each(fixtures)("Python HTTP contract: $name", async (fixture) => { + const requests: unknown[] = []; + const server = await localHTTP(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const raw = Buffer.concat(chunks).toString(); + const url = new URL(req.url!, "http://fixture.test"); + const expected = fixture.requests[requests.length]; + requests.push({ + method: req.method, + path: url.pathname, + query: Object.fromEntries( + [...url.searchParams.keys()].map((key) => [key, url.searchParams.getAll(key)]) + ), + body: raw ? JSON.parse(raw) : null, + }); + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify(expected?.reply ?? {})); + }); + cleanup.push(server.close); + const client = new HyperbrowserClient({ apiKey: "local", baseUrl: server.url, timeout: 2000 }); + const connection = { sandboxId: "s", baseUrl: server.url, token: "local" }; + const transport = new RuntimeTransport(async () => connection, 2000); + const files = new SandboxFilesApi(transport, async () => connection); + const subjects: Record = { + sandboxes: client.sandboxes, + volumes: client.volumes, + processes: new SandboxProcessesApi(transport), + terminal: new SandboxTerminalApi(transport, async () => connection), + processHandle: new SandboxProcessHandle(transport, { + id: "proc_1", + command: "bash", + cwd: "/tmp", + status: "running", + startedAt: 1, + }), + terminalHandle: new SandboxTerminalHandle(transport, async () => connection, { + id: "pty_1", + command: "bash", + cwd: "/tmp", + rows: 24, + cols: 80, + running: true, + startedAt: 1, + }), + files, + filesRoot: files.withRunAs("root"), + }; + const subject = subjects[fixture.subject] as Record< + string, + (...args: unknown[]) => Promise + >; + let result = await subject[fixture.nodeMethod](...fixture.nodeArgs); + if (result && typeof (result as { toJSON?: unknown }).toJSON === "function") + result = (result as { toJSON(): unknown }).toJSON(); + if (fixture.expectedResult === null) expect(result).toBeUndefined(); + else if (typeof fixture.expectedResult === "object") + expect(result).toMatchObject(expectedValues(fixture.expectedResult) as object); + else expect(result).toEqual(fixture.expectedResult); + // Python capitalizes booleans; the receiver accepts both forms. Node avoids + // Python's redundant detail GET when expose already returned an explicit URL. + const expected = fixture.requests + .filter((_request, i) => !(fixture.nodeMethod === "expose" && i > 0)) + .map(({ reply: _reply, ...request }) => ({ + ...request, + query: Object.fromEntries( + Object.entries(request.query).map(([key, values]) => [ + key, + values?.map((value) => + ["includeOutput", "includeEvents"].includes(key) ? value.toLowerCase() : value + ), + ]) + ), + })); + expect(requests).toEqual(expected); +}); diff --git a/tests/integration/runtime-http.test.ts b/tests/integration/runtime-http.test.ts new file mode 100644 index 0000000..70f39f1 --- /dev/null +++ b/tests/integration/runtime-http.test.ts @@ -0,0 +1,349 @@ +import { afterEach, describe, expect, test } from "vitest"; +import { RuntimeTransport } from "../../src/sandbox/base"; +import { SandboxProcessesApi } from "../../src/sandbox/process"; +import { SandboxFilesApi } from "../../src/sandbox/files"; +import { BaseService } from "../../src/services/base"; +import { delay, localHTTP } from "../helpers/local-http"; + +const started = + 'event: started\ndata: {"id":"p","status":"running","command":"sleep 300","cwd":"/tmp","started_at":1}\n\n'; +const cleanup: Array<() => Promise> = []; +afterEach(async () => { + for (const close of cleanup.splice(0)) await close(); +}); +async function setup(handler: Parameters[0], timeout = 1000) { + const server = await localHTTP(handler); + cleanup.push(server.close); + let refreshes = 0; + const connection = { sandboxId: "s", token: "local", baseUrl: server.url }; + const transport = new RuntimeTransport(async (refresh) => { + if (refresh) refreshes++; + return connection; + }, timeout); + return { + ...server, + transport, + files: new SandboxFilesApi(transport, async () => connection), + processes: new SandboxProcessesApi(transport), + refreshes: () => refreshes, + }; +} +describe("runtime HTTP lifetime", () => { + test("CR-only events are delivered before the response ends", async () => { + const api = await setup((_req, res) => { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write(started.replace(/\n/g, "\r")); + }); + const stream = await api.transport.openSSE("/stream", undefined, { idleTimeoutMs: 100 }); + try { + expect((await stream.events.next()).value?.event).toBe("started"); + } finally { + stream.close(); + } + }); + test("disconnect releases the actual process socket", async () => { + let closed = false; + const api = await setup((_req, res) => { + res.on("close", () => (closed = true)); + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write(started); + }); + const handle = await api.processes.start("sleep 300"); + handle.disconnect(); + await expect(handle.wait()).rejects.toMatchObject({ code: "incomplete_output" }); + await delay(25); + expect(closed).toBe(true); + expect(api.sockets.size).toBe(0); + }); + test("idle timeout rejects instead of completing and releases its socket", async () => { + const api = await setup((_req, res) => { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write(started); + }); + const stream = await api.transport.openSSE("/processes", undefined, { + method: "POST", + idleTimeoutMs: 20, + }); + expect((await stream.events.next()).value?.event).toBe("started"); + await expect(stream.events.next()).rejects.toMatchObject({ + code: "stream_idle_timeout", + method: "POST", + path: "/processes", + statusCode: 200, + }); + await delay(20); + expect(api.sockets.size).toBe(0); + }); + test("heartbeats reset the idle budget and split UTF-8 is preserved", async () => { + const api = await setup((_req, res) => { + res.writeHead(200, { "content-type": "text/event-stream" }); + const data = Buffer.from('event: output\ndata: {"data":"€"}\n\n'); + const split = data.indexOf(Buffer.from("€")) + 1; + res.write(data.subarray(0, split)); + setTimeout(() => res.write(data.subarray(split)), 10); + const timer = setInterval(() => res.write(": ping\n\n"), 10); + setTimeout(() => { + clearInterval(timer); + res.end(); + }, 70); + res.once("close", () => clearInterval(timer)); + }); + const stream = await api.transport.openSSE("/stream", undefined, { idleTimeoutMs: 35 }); + const events = []; + for await (const event of stream.events) events.push(event); + expect(events).toHaveLength(1); + expect(events[0].data).toEqual({ data: "€" }); + }); + test("AbortSignal cancels collection without another process POST", async () => { + let starts = 0; + const api = await setup((_req, res) => { + starts++; + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write(started); + }); + const controller = new AbortController(); + const handle = await api.processes.start("sleep 300", { signal: controller.signal }); + controller.abort(); + await expect(handle.wait()).rejects.toMatchObject({ + code: "request_aborted", + retryable: false, + }); + await delay(20); + expect(starts).toBe(1); + expect(api.sockets.size).toBe(0); + }); + test("JSON-only receivers are rejected without a repeated command or socket leak", async () => { + let starts = 0; + const api = await setup((_req, res) => { + starts++; + res.writeHead(200, { "content-type": "application/json" }); + res.write('{"process":'); + }); + await expect(api.processes.start("echo hi")).rejects.toMatchObject({ + code: "streaming_not_supported", + }); + await delay(20); + expect(starts).toBe(1); + expect(api.sockets.size).toBe(0); + }); + test("normal response body reads have a deadline after headers", async () => { + const api = await setup((_req, res) => { + res.writeHead(200); + res.write('{"partial":'); + }, 25); + await expect(api.transport.requestJSON("/files/read")).rejects.toMatchObject({ + service: "runtime", + retryable: true, + }); + }); + test("empty successful chunked responses return an empty object", async () => { + const api = await setup((_req, res) => { + res.writeHead(200, { "transfer-encoding": "chunked" }); + res.end(); + }); + expect(await api.transport.requestJSON("/files/delete", { method: "POST" })).toEqual({}); + }); + test("401 refresh closes the replaced response before retrying", async () => { + let count = 0; + let firstClosed = false; + const api = await setup((_req, res) => { + count++; + if (count === 1) { + res.on("close", () => (firstClosed = true)); + res.writeHead(401); + res.write("expired"); + } else res.end('{"ok":true}'); + }); + expect(await api.transport.requestJSON("/stat")).toEqual({ ok: true }); + await delay(20); + expect(firstClosed).toBe(true); + expect(api.refreshes()).toBe(1); + }); +}); + +test("control body failures retain response context and caller cancellation", async () => { + const api = await setup((_req, res) => { + res.writeHead(200, { "x-request-id": "partial-response" }); + res.write("{"); + }); + class Service extends BaseService { + read(signal: AbortSignal) { + return this.request("/body?private=value", { signal }); + } + } + const controller = new AbortController(); + const pending = new Service("local", api.url, 1000).read(controller.signal); + const rejected = expect(pending).rejects.toMatchObject({ + code: "request_aborted", + statusCode: 200, + requestId: "partial-response", + method: "GET", + path: "/body", + retryable: false, + service: "control", + }); + await delay(25); + controller.abort(); + await rejected; +}); + +test("runtime HTTP errors include method and query-free path context", async () => { + const api = await setup((_req, res) => { + res.writeHead(409, { "x-request-id": "conflict" }); + res.end(JSON.stringify({ code: "conflict", message: "Cannot move" })); + }); + await expect( + api.transport.requestJSON("/files/move?private=value", { method: "POST", body: "{}" }) + ).rejects.toMatchObject({ + statusCode: 409, + requestId: "conflict", + method: "POST", + path: "/files/move", + }); +}); +describe("true file streaming", () => { + test("downloads expose chunks before EOF and break releases the connection", async () => { + let closed = false; + let url = ""; + const api = await setup((req, res) => { + url = req.url!; + res.on("close", () => (closed = true)); + res.write(Buffer.alloc(128 * 1024, 7)); + }); + for await (const chunk of api.files.withRunAs("root").downloadStream("/large")) { + expect(chunk[0]).toBe(7); + break; + } + await delay(20); + expect(closed).toBe(true); + expect(url).toContain("runAs=root"); + }); + test("uploads are pull-based and preserve content length and runAs", async () => { + let firstRead!: () => void; + const first = new Promise((resolve) => (firstRead = resolve)); + let bytes = 0; + let url = ""; + let length = ""; + const api = await setup((req, res) => { + url = req.url!; + length = req.headers["content-length"]!; + req.on("data", (chunk) => { + bytes += chunk.length; + firstRead(); + }); + req.on("end", () => res.end(JSON.stringify({ path: "/large", bytesWritten: bytes }))); + }); + const size = 20 * 1024 * 1024; + async function* source() { + yield Buffer.alloc(65536); + await first; + for (let i = 65536; i < size; i += 65536) yield Buffer.alloc(65536); + } + expect( + await api.files.withRunAs("root").uploadStream("/large", source(), { contentLength: size }) + ).toEqual({ path: "/large", bytesWritten: size }); + expect(length).toBe(String(size)); + expect(url).toContain("runAs=root"); + }); + test("a streamed upload receiving 401 is never replayed", async () => { + let requests = 0; + let released = false; + const api = await setup((req, res) => { + requests++; + req.once("data", () => { + res.writeHead(401); + res.end("expired"); + }); + }); + async function* source() { + try { + for (let i = 0; i < 100; i++) { + yield Buffer.alloc(65536); + await delay(2); + } + } finally { + released = true; + } + } + await expect(api.files.uploadStream("/large", source())).rejects.toMatchObject({ + code: "stream_not_replayable", + }); + await delay(25); + expect(requests).toBe(1); + expect(api.refreshes()).toBe(0); + expect(released).toBe(true); + }); +}); + +test("file aliases preserve operation payloads and overwrite false", async () => { + const calls: Array<{ method: string; url: string; body: unknown }> = []; + const file = { + path: "/b", + name: "b", + type: "file", + size: 0, + mode: 0, + permissions: "", + owner: "root", + group: "root", + }; + const api = await setup(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + calls.push({ method: req.method!, url: req.url!, body: body ? JSON.parse(body) : undefined }); + res.end(JSON.stringify({ file, entry: file, created: true })); + }); + const files = api.files.withRunAs("root"); + expect(await files.stat("/b")).toMatchObject({ name: "b" }); + expect(await files.mkdir("/b", { parents: true })).toBe(true); + await files.rename("/a", "/b", { overwrite: false }); + await files.move({ source: "/b", destination: "/c", overwrite: true }); + await files.delete("/c", { recursive: true }); + expect(calls[0].url).toContain("runAs=root"); + expect(calls.slice(1).map((call) => call.body)).toEqual([ + { path: "/b", parents: true, runAs: "root" }, + { from: "/a", to: "/b", overwrite: false, runAs: "root" }, + { from: "/b", to: "/c", overwrite: true, runAs: "root" }, + { path: "/c", recursive: true, runAs: "root" }, + ]); +}); + +test.each([false, true])("heartbeats outlive ordinary request deadlines (refresh=%s)", async (refresh) => { + let calls = 0; + const api = await setup((_req, res) => { + calls++; + if (refresh && calls === 1) { res.writeHead(401); res.end("expired"); return; } + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write(started); + const heartbeat = setInterval(() => res.write(": heartbeat\n\n"), 15); + const finish = setTimeout(() => res.end('event: done\ndata: {"last_seq":0}\n\n'), 150); + res.once("close", () => { clearInterval(heartbeat); clearTimeout(finish); }); + }, 50); + const stream = await api.transport.openSSE("/processes", undefined, { method: "POST", idleTimeoutMs: 100 }); + const events = []; + for await (const event of stream.events) events.push(event.event); + expect(events).toEqual(["started", "done"]); + expect(calls).toBe(refresh ? 2 : 1); + expect(api.refreshes()).toBe(refresh ? 1 : 0); +}); +test("SSE response headers retain the ordinary deadline without replaying POST", async () => { + let requests = 0; + const api = await setup(() => { requests++; }, 30); + await expect(api.transport.openSSE("/processes", undefined, { method: "POST" })).rejects.toMatchObject({ service: "runtime", method: "POST", retryable: true }); + await delay(15); + expect(requests).toBe(1); + expect(api.sockets.size).toBe(0); +}); +test("canceling an upload releases its producer and socket without replay", async () => { + const controller = new AbortController(); + let requests = 0; let released = false; + const api = await setup((req) => { requests++; req.once("data", () => controller.abort()); }); + async function* chunks() { + try { for (let i = 0; i < 1000; i++) { yield Buffer.alloc(32768); await delay(5); } } + finally { released = true; } + } + await expect(api.files.uploadStream("/file", chunks(), { signal: controller.signal })).rejects.toMatchObject({ code: "request_aborted", retryable: false }); + await expect.poll(() => released, { timeout: 1000 }).toBe(true); + await expect.poll(() => api.sockets.size, { timeout: 1000 }).toBe(0); + expect(requests).toBe(1); +}); diff --git a/tests/integration/sandbox-parity.test.ts b/tests/integration/sandbox-parity.test.ts new file mode 100644 index 0000000..cb177cc --- /dev/null +++ b/tests/integration/sandbox-parity.test.ts @@ -0,0 +1,130 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { HyperbrowserClient } from "../../src/client"; +import { CreateSandboxParams } from "../../src/types"; +import { localHTTP } from "../helpers/local-http"; +const cleanup: Array<() => Promise> = []; +afterEach(async () => { + for (const close of cleanup.splice(0)) await close(); + vi.unstubAllEnvs(); +}); +const detail = { + id: "s", + teamId: "t", + status: "active", + region: "us", + duration: 10, + createdAt: "now", + updatedAt: "now", + sessionUrl: "url", + runtime: { transport: "regional_proxy", host: "s.example.com", baseUrl: "https://s.example.com" }, + token: "token", + tokenExpiresAt: null, +}; +async function setup() { + const calls: Array<{ method: string; url: string; body: unknown }> = []; + const server = await localHTTP(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + calls.push({ method: req.method!, url: req.url!, body: body ? JSON.parse(body) : undefined }); + if (req.url === "/api/sandbox/s/expose") res.end('{"port":8080,"auth":true}'); + else if (req.url?.startsWith("/api/volume")) + res.end( + JSON.stringify( + req.url === "/api/volume" && req.method === "GET" + ? { volumes: [{ id: "v", name: "n", size: "42", transferAmount: "" }], totalCount: "1" } + : { id: "v", name: "n", size: "42", transferAmount: null } + ) + ); + else + res.end( + JSON.stringify({ + ...detail, + vcpus: "2", + memMiB: "2048", + diskSizeMiB: "", + dataConsumed: "7", + proxyDataConsumed: "", + proxyBytesUsed: null, + timeoutMinutes: "15", + }) + ); + }); + cleanup.push(server.close); + return { + calls, + url: server.url, + client: new HyperbrowserClient({ apiKey: "local", baseUrl: server.url }), + }; +} +test.each([ + {}, + { imageName: "" }, + { imageName: "a", snapshotName: "b" }, + { imageId: "id" }, + { snapshotId: "id" }, + { snapshotName: "s", cpu: 2 }, + { imageName: "a", cpu: NaN }, + { imageName: "a", diskMiB: 1.5 }, +])("invalid launch never reaches the server: %j", async (params) => { + const api = await setup(); + await expect(api.client.sandboxes.create(params as CreateSandboxParams)).rejects.toThrow(); + expect(api.calls).toHaveLength(0); +}); +test("snapshot startup and runtime auth reuse the existing lifecycle", async () => { + const api = await setup(); + const sandbox = await api.client.sandboxes.startFromSnapshot({ + snapshotName: "snap", + snapshotId: "id", + timeoutMinutes: 5, + }); + expect(api.calls[0].body).toEqual({ snapshotName: "snap", snapshotId: "id", timeoutMinutes: 5 }); + const session = await sandbox.createRuntimeSession(); + session.runtime.host = "mutated"; + expect((await sandbox.createRuntimeSession()).runtime.host).toBe("s.example.com"); + expect(api.calls).toHaveLength(1); + await sandbox.createRuntimeSession({ forceRefresh: true }); + expect(api.calls).toHaveLength(2); + expect((await api.client.sandboxes.getRuntimeSession("s")).token).toBe("token"); +}); +test("exposure URL fallback uses the handle's cached runtime", async () => { + const api = await setup(); + const sandbox = await api.client.sandboxes.get("s"); + expect((await sandbox.expose({ port: 8080 })).url).toBe("https://8080-s.example.com/"); + expect(api.calls).toHaveLength(2); + expect((await api.client.sandboxes.expose("s", { port: 8080 })).url).toBe( + "https://8080-s.example.com/" + ); + expect(api.calls).toHaveLength(4); +}); +test("sandbox and volume numeric strings, empty values, and absent tokens normalize", async () => { + const api = await setup(); + const sandbox = await api.client.sandboxes.get("s"); + expect(sandbox.toJSON()).toMatchObject({ + cpu: 2, + memoryMiB: 2048, + diskMiB: null, + dataConsumed: 7, + proxyDataConsumed: null, + proxyBytesUsed: null, + exposedPorts: [], + creditsUsed: null, + }); + expect(await api.client.volumes.get("v")).toEqual({ + id: "v", + name: "n", + size: 42, + transferAmount: null, + }); + expect(await api.client.volumes.list()).toMatchObject({ + totalCount: 1, + volumes: [{ size: 42, transferAmount: null }], + }); +}); +test("environment base URL is honored and explicit config wins", async () => { + const api = await setup(); + vi.stubEnv("HYPERBROWSER_BASE_URL", api.url); + await new HyperbrowserClient({ apiKey: "local" }).sandboxes.get("s"); + vi.stubEnv("HYPERBROWSER_BASE_URL", "http://invalid.test"); + await new HyperbrowserClient({ apiKey: "local", baseUrl: api.url }).sandboxes.get("s"); + expect(api.calls).toHaveLength(2); +}); diff --git a/tests/sandbox/e2e/process-api.test.ts b/tests/sandbox/e2e/process-api.test.ts deleted file mode 100644 index a4ede9f..0000000 --- a/tests/sandbox/e2e/process-api.test.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { describe, expect, test, vi } from "vitest"; -import type { RuntimeTransport } from "../../../src/sandbox/base"; -import { SandboxProcessesApi } from "../../../src/sandbox/process"; -import { SandboxHandle } from "../../../src/services/sandboxes"; - -const execResponse = { - result: { - id: "proc_exec", - status: "exited" as const, - exit_code: 0, - stdout: "ok\n", - stderr: "", - started_at: 1, - completed_at: 2, - }, -}; - -const startResponse = { - process: { - id: "proc_start", - status: "running" as const, - command: "sleep 30", - cwd: "/tmp", - started_at: 1, - }, -}; - -describe("sandbox process api", () => { - test("exec string overload forwards runAs in the runtime payload", async () => { - const requestJSON = vi - .fn() - .mockResolvedValueOnce(execResponse) - .mockResolvedValueOnce(startResponse); - const transport = { - requestJSON, - } as unknown as RuntimeTransport; - const api = new SandboxProcessesApi(transport); - - await api.exec("whoami", { - cwd: "/tmp", - env: { FOO: "bar" }, - timeoutMs: 5_000, - runAs: "root", - }); - await api.start("sleep 30", { - cwd: "/tmp", - runAs: "root", - }); - - expect(requestJSON).toHaveBeenNthCalledWith( - 1, - "/sandbox/exec", - expect.objectContaining({ - method: "POST", - body: JSON.stringify({ - command: "whoami", - cwd: "/tmp", - env: { FOO: "bar" }, - timeoutMs: 5_000, - runAs: "root", - }), - }) - ); - expect(requestJSON).toHaveBeenNthCalledWith( - 2, - "/sandbox/processes", - expect.objectContaining({ - method: "POST", - body: JSON.stringify({ - command: "sleep 30", - cwd: "/tmp", - runAs: "root", - }), - }) - ); - }); - - test("exec object form preserves runAs in the runtime payload", async () => { - const requestJSON = vi.fn().mockResolvedValue(execResponse); - const transport = { - requestJSON, - } as unknown as RuntimeTransport; - const api = new SandboxProcessesApi(transport); - - await api.exec({ - command: "whoami", - runAs: "root", - timeoutSec: 5, - }); - - expect(requestJSON).toHaveBeenCalledWith( - "/sandbox/exec", - expect.objectContaining({ - method: "POST", - body: JSON.stringify({ - command: "whoami", - timeout_sec: 5, - runAs: "root", - }), - }) - ); - }); - - test("legacy args and useShell are normalized out of process payloads", async () => { - const requestJSON = vi - .fn() - .mockResolvedValueOnce(execResponse) - .mockResolvedValueOnce(startResponse); - const transport = { - requestJSON, - } as unknown as RuntimeTransport; - const api = new SandboxProcessesApi(transport); - - await api.exec({ - command: "/bin/echo", - args: ["legacy args value"], - useShell: false, - runAs: "root", - }); - await api.start({ - command: "bash", - args: ["-lc", "echo process-started"], - useShell: true, - cwd: "/tmp", - }); - - const execPayload = JSON.parse(requestJSON.mock.calls[0][1].body); - expect(execPayload).toEqual({ - command: "/bin/echo 'legacy args value'", - runAs: "root", - }); - expect(execPayload).not.toHaveProperty("args"); - expect(execPayload).not.toHaveProperty("useShell"); - - const startPayload = JSON.parse(requestJSON.mock.calls[1][1].body); - expect(startPayload).toEqual({ - command: "bash -lc 'echo process-started'", - cwd: "/tmp", - }); - expect(startPayload).not.toHaveProperty("args"); - expect(startPayload).not.toHaveProperty("useShell"); - }); - - test("sandbox handle exec forwards string options to processes.exec", async () => { - const exec = vi.fn().mockResolvedValue(execResponse.result); - - await SandboxHandle.prototype.exec.call( - { - processes: { exec }, - }, - "whoami", - { runAs: "root" } - ); - - expect(exec).toHaveBeenCalledWith("whoami", { runAs: "root" }); - }); -}); diff --git a/tests/unit/control-get-retries.test.ts b/tests/unit/control-get-retries.test.ts new file mode 100644 index 0000000..bf13c25 --- /dev/null +++ b/tests/unit/control-get-retries.test.ts @@ -0,0 +1,89 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import { Headers, Response } from "node-fetch"; + +const fetchMock = vi.fn(); +vi.mock("node-fetch", async () => { + const actual = await vi.importActual("node-fetch"); + return { ...actual, default: (...args: unknown[]) => fetchMock(...args) }; +}); +vi.mock("../../src/retry", async () => { + const actual = await vi.importActual("../../src/retry"); + return { ...actual, retryDelay: vi.fn().mockResolvedValue(undefined) }; +}); + +import { HyperbrowserError } from "../../src/client"; +import { retryDelay } from "../../src/retry"; +import { BaseService } from "../../src/services/base"; + +class TestService extends BaseService { + full(path: string) { + return this.request(path, { method: "POST" }, undefined, true); + } + get(path: string) { + return this.request(path); + } + post(path: string) { + return this.request(path, { method: "POST", body: "{}" }); + } +} + +const response = (status: number, body: unknown = {}): Response => + new Response(JSON.stringify(body), { + status, + headers: new Headers({ "content-type": "application/json" }), + }); + +afterEach(() => { + fetchMock.mockReset(); + vi.mocked(retryDelay).mockClear(); +}); + +describe("control transport GET retries", () => { + test("retries transient statuses and returns the successful response", async () => { + fetchMock.mockResolvedValueOnce(response(502, "Bad Gateway")).mockResolvedValueOnce(response(200, { ok: true })); + const service = new TestService("key", "https://api.example.com", 1_000); + await expect(service.get("/test")).resolves.toEqual({ ok: true }); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(retryDelay).toHaveBeenCalledTimes(1); + }); + + test("stops after three attempts", async () => { + fetchMock.mockResolvedValue(response(503, "Service Unavailable")); + const service = new TestService("key", "https://api.example.com", 1_000); + const error = await service.get("/test").then(() => null, (e: unknown) => e); + expect(error).toBeInstanceOf(HyperbrowserError); + expect((error as HyperbrowserError).statusCode).toBe(503); + expect((error as HyperbrowserError).retryable).toBe(true); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + test("does not retry non-retryable statuses", async () => { + fetchMock.mockResolvedValue(response(404, { message: "missing" })); + const service = new TestService("key", "https://api.example.com", 1_000); + await expect(service.get("/test")).rejects.toMatchObject({ statusCode: 404 }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + test("retries network failures on GET only", async () => { + const reset = Object.assign(new Error("socket hang up"), { code: "ECONNRESET" }); + fetchMock.mockRejectedValueOnce(reset).mockResolvedValueOnce(response(200, { ok: true })); + const service = new TestService("key", "https://api.example.com", 1_000); + await expect(service.get("/test")).resolves.toEqual({ ok: true }); + expect(fetchMock).toHaveBeenCalledTimes(2); + + fetchMock.mockReset(); + fetchMock.mockResolvedValue(response(502, "Bad Gateway")); + await expect(service.post("/test")).rejects.toMatchObject({ statusCode: 502 }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); + + +test("empty network errors retain their type and omit full URL credentials from context", async () => { + const failure = Object.assign(new Error(""), { name: "ConnectError", code: "ECONNRESET" }); + fetchMock.mockRejectedValue(failure); + const service = new TestService("local", "http://unused", 1000); + const error = await service.full("https://user:private@api.example.com/sandbox?token=secret").catch((value) => value); + expect(error).toMatchObject({ message: "[Hyperbrowser]: Unknown error occurred (ConnectError)", method: "POST", path: "/sandbox", retryable: true }); + expect(fetchMock).toHaveBeenCalledTimes(1); +}); diff --git a/tests/sandbox/e2e/image-build-contract.test.ts b/tests/unit/image-build-contract.test.ts similarity index 97% rename from tests/sandbox/e2e/image-build-contract.test.ts rename to tests/unit/image-build-contract.test.ts index 2fa0dd5..6d20dfc 100644 --- a/tests/sandbox/e2e/image-build-contract.test.ts +++ b/tests/unit/image-build-contract.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test, vi } from "vitest"; -import { SandboxesService } from "../../../src/services/sandboxes"; +import { SandboxesService } from "../../src/services/sandboxes"; const parseJsonRequestBody = (init: unknown): Record => { if (!init || typeof init !== "object" || !("body" in init) || typeof init.body !== "string") { diff --git a/tests/sandbox/e2e/list-contract.test.ts b/tests/unit/list-contract.test.ts similarity index 98% rename from tests/sandbox/e2e/list-contract.test.ts rename to tests/unit/list-contract.test.ts index 0b0dc5c..ce5aabe 100644 --- a/tests/sandbox/e2e/list-contract.test.ts +++ b/tests/unit/list-contract.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, vi } from "vitest"; -import { SandboxesService } from "../../../src/services/sandboxes"; +import { SandboxesService } from "../../src/services/sandboxes"; describe("sandbox control list contract", () => { test("list forwards status, start, end, search, page, and limit to the control API", async () => { diff --git a/tests/unit/process-api.test.ts b/tests/unit/process-api.test.ts new file mode 100644 index 0000000..60cce45 --- /dev/null +++ b/tests/unit/process-api.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, test, vi } from "vitest"; +import type { RuntimeSSEEvent, RuntimeSSEInit, RuntimeTransport } from "../../src/sandbox/base"; +import { SandboxProcessesApi } from "../../src/sandbox/process"; +import { SandboxHandle } from "../../src/services/sandboxes"; + +const execResponse = { + result: { + id: "proc_exec", + status: "exited" as const, + exitCode: 0, + stdout: "ok\n", + stderr: "", + startedAt: 1, + completedAt: 2, + }, +}; + +const startedEvent: RuntimeSSEEvent = { + event: "started", + data: { id: "proc_start", status: "running", command: "sleep 30", cwd: "/tmp", started_at: 1 }, +}; + +const doneEvent: RuntimeSSEEvent = { + event: "done", + data: { + id: "proc_start", + status: "exited", + exit_code: 0, + started_at: 1, + completed_at: 2, + last_seq: 0, + }, +}; + +/** Every start opens one streamed POST whose body is the runtime payload. */ +const streamingTransport = () => { + const openSSE = vi.fn(async (_path: string, _params: unknown, _init?: RuntimeSSEInit) => ({ + events: (async function* () { + yield startedEvent; + yield doneEvent; + })(), + close: () => undefined, + })); + return { openSSE, transport: { openSSE } as unknown as RuntimeTransport }; +}; + +const payloadOf = (openSSE: ReturnType, index: number) => + JSON.parse((openSSE.mock.calls[index][2] as RuntimeSSEInit).body ?? ""); + +describe("sandbox process api", () => { + test("exec string overload forwards runAs in the runtime payload", async () => { + const { openSSE, transport } = streamingTransport(); + const api = new SandboxProcessesApi(transport); + + await api.exec("whoami", { + cwd: "/tmp", + env: { FOO: "bar" }, + timeoutMs: 5_000, + runAs: "root", + }); + await api.start("sleep 30", { + cwd: "/tmp", + runAs: "root", + }); + + expect(openSSE).toHaveBeenCalledTimes(2); + expect(openSSE.mock.calls[0][0]).toBe("/sandbox/processes"); + expect(openSSE.mock.calls[0][2]).toMatchObject({ method: "POST" }); + expect(payloadOf(openSSE, 0)).toEqual({ + command: "whoami", + cwd: "/tmp", + env: { FOO: "bar" }, + timeoutMs: 5_000, + runAs: "root", + }); + expect(payloadOf(openSSE, 1)).toEqual({ + command: "sleep 30", + cwd: "/tmp", + runAs: "root", + }); + }); + + test("exec object form preserves runAs in the runtime payload", async () => { + const { openSSE, transport } = streamingTransport(); + const api = new SandboxProcessesApi(transport); + + await api.exec({ + command: "whoami", + runAs: "root", + timeoutSec: 5, + }); + + expect(payloadOf(openSSE, 0)).toEqual({ + command: "whoami", + timeout_sec: 5, + runAs: "root", + }); + }); + + test("legacy args and useShell are normalized out of process payloads", async () => { + const { openSSE, transport } = streamingTransport(); + const api = new SandboxProcessesApi(transport); + + await api.exec({ + command: "/bin/echo", + args: ["legacy args value"], + useShell: false, + runAs: "root", + }); + await api.start({ + command: "bash", + args: ["-lc", "echo process-started"], + useShell: true, + cwd: "/tmp", + }); + + const execPayload = payloadOf(openSSE, 0); + expect(execPayload).toEqual({ + command: "/bin/echo 'legacy args value'", + runAs: "root", + }); + expect(execPayload).not.toHaveProperty("args"); + expect(execPayload).not.toHaveProperty("useShell"); + + const startPayload = payloadOf(openSSE, 1); + expect(startPayload).toEqual({ + command: "bash -lc 'echo process-started'", + cwd: "/tmp", + }); + expect(startPayload).not.toHaveProperty("args"); + expect(startPayload).not.toHaveProperty("useShell"); + }); + + test("sandbox handle exec forwards string options to processes.exec", async () => { + const exec = vi.fn().mockResolvedValue(execResponse.result); + + const execString = SandboxHandle.prototype.exec as ( + input: string, + options?: { runAs?: string } + ) => Promise; + await execString.call( + { + processes: { exec }, + }, + "whoami", + { runAs: "root" } + ); + + expect(exec).toHaveBeenCalledWith("whoami", { runAs: "root" }); + }); +}); diff --git a/tests/unit/process-collection.test.ts b/tests/unit/process-collection.test.ts new file mode 100644 index 0000000..276a963 --- /dev/null +++ b/tests/unit/process-collection.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, test } from "vitest"; +import { HyperbrowserError } from "../../src/client"; +import type { RuntimeSSEEvent, RuntimeSSEInit, RuntimeSSEStream, RuntimeTransport } from "../../src/sandbox/base"; +import { SandboxProcessesApi } from "../../src/sandbox/process"; + +const output = (seq: number, data: Buffer, stream = "stdout"): RuntimeSSEEvent => ({ + event: "output", + data: { seq, stream, data: data.toString("base64"), encoding: "base64", timestamp: 1 }, +}); + +const done = (seq: number, extra: Record = {}): RuntimeSSEEvent => ({ + event: "done", + data: { + id: "p1", + status: "exited", + exit_code: 7, + started_at: 1, + completed_at: 2, + last_seq: seq, + ...extra, + }, +}); + +const STARTED: RuntimeSSEEvent = { + event: "started", + data: { id: "p1", status: "running", command: "test", cwd: "/tmp", started_at: 1 }, +}; + +class Gate { + private resolve: (() => void) | null = null; + readonly promise = new Promise((resolve) => { + this.resolve = resolve; + }); + open(): void { + this.resolve?.(); + } +} + +class FakeTransport { + calls: Array<{ path: string; init: RuntimeSSEInit }> = []; + closed = false; + gate: Gate | null = null; + + constructor(private events: RuntimeSSEEvent[]) {} + + async openSSE(path: string, _params: unknown, init: RuntimeSSEInit = {}): Promise { + this.calls.push({ path, init }); + const transport = this; + let closed = false; + const events = (async function* () { + try { + yield STARTED; + if (transport.gate) { + await Promise.race([transport.gate.promise, new Promise((resolve) => { + const timer = setInterval(() => { + if (closed) { + clearInterval(timer); + resolve(); + } + }, 5); + })]); + if (closed) { + return; + } + } + for (const event of transport.events) { + if (closed) { + return; + } + yield event; + } + } finally { + transport.closed = true; + } + })(); + return { + events, + close: () => { + closed = true; + transport.closed = true; + }, + }; + } + + asTransport(): RuntimeTransport { + return this as unknown as RuntimeTransport; + } +} + +const largeOutput = (): { events: RuntimeSSEEvent[]; expected: string } => { + const chunk = Buffer.alloc(32768, "x"); + const events = Array.from({ length: 160 }, (_, i) => output(i + 1, chunk)); + events.push( + output(161, Buffer.from([0xe2])), + output(162, Buffer.from([0x82, 0xac])), + output(163, Buffer.from("error"), "stderr"), + done(163) + ); + return { events, expected: "x".repeat(160 * chunk.length) + "€" }; +}; + +describe("sandbox process output collection", () => { + test("collects large output and streams from the same request", async () => { + const { events, expected } = largeOutput(); + const transport = new FakeTransport(events); + const handle = await new SandboxProcessesApi(transport.asTransport()).start("test"); + const result = await handle.wait({ timeoutSec: 5 }); + expect([result.stdout, result.stderr, result.exitCode]).toEqual([expected, "error", 7]); + expect(handle.status).toBe("exited"); + const streamed = []; + for await (const event of handle.stream()) { + streamed.push(event); + } + expect( + streamed + .filter((e) => e.type === "stdout") + .map((e) => (e.type === "stdout" ? e.data : "")) + .join("") + ).toBe(expected); + const last = streamed[streamed.length - 1]; + expect(last.type === "exit" && last.result).toEqual(result); + handle.disconnect(); + expect(transport.closed).toBe(true); + expect(transport.calls).toHaveLength(1); + expect(transport.calls[0].path).toBe("/sandbox/processes"); + expect(transport.calls[0].init.method).toBe("POST"); + expect(JSON.parse(transport.calls[0].init.body ?? "")).toEqual({ command: "test" }); + }); + + test.each([ + [[output(2, Buffer.from("gap")), done(2)], 100, "incomplete_output"], + [[output(1, Buffer.from("no completion"))], 100, "incomplete_output"], + [[output(1, Buffer.from("tail missing")), done(2)], 100, "incomplete_output"], + [[done(0, { output_truncated: true })], 100, "incomplete_output"], + [[output(1, Buffer.from("too much")), done(1)], 4, "output_limit_exceeded"], + ])("incomplete output is not success or re-executed (%#)", async (events, limit, code) => { + const transport = new FakeTransport(events as RuntimeSSEEvent[]); + const error = await new SandboxProcessesApi(transport.asTransport()) + .exec("test", { maxOutputBytes: limit as number }) + .then(() => null, (e: unknown) => e); + expect(error).toBeInstanceOf(HyperbrowserError); + const failure = error as HyperbrowserError; + expect(failure.code).toBe(code); + expect((failure.details as Record).process_id).toBe("p1"); + expect(failure.retryable).toBe(false); + expect(transport.closed).toBe(true); + expect(transport.calls).toHaveLength(1); + }); + + test("wait timeout keeps the collector alive", async () => { + const transport = new FakeTransport([output(1, Buffer.from("later")), done(1)]); + transport.gate = new Gate(); + const handle = await new SandboxProcessesApi(transport.asTransport()).start("test"); + await expect(handle.wait({ timeoutMs: 1 })).rejects.toMatchObject({ code: "wait_timeout" }); + expect(transport.closed).toBe(false); + transport.gate.open(); + expect((await handle.wait()).stdout).toBe("later"); + }); + + test("disconnect closes the stream without killing the command", async () => { + const transport = new FakeTransport([]); + transport.gate = new Gate(); + const handle = await new SandboxProcessesApi(transport.asTransport()).start("test"); + handle.disconnect(); + expect(transport.closed).toBe(true); + await expect(handle.wait()).rejects.toThrow(/disconnected/); + expect(transport.calls).toHaveLength(1); + }); + + test("a stream that ends before its completion event is incomplete", async () => { + const transport = new FakeTransport([output(1, Buffer.from("partial"))]); + const handle = await new SandboxProcessesApi(transport.asTransport()).start("test"); + await expect(handle.wait()).rejects.toMatchObject({ code: "incomplete_output" }); + }); + + test.each([0, -1, 1.5, NaN])("invalid collection limit %s is rejected before start", async (limit) => { + const transport = new FakeTransport([]); + await expect( + new SandboxProcessesApi(transport.asTransport()).start("test", { maxOutputBytes: limit }) + ).rejects.toThrow(/positive integer/); + expect(transport.calls).toHaveLength(0); + }); + + test("a non-started first event fails and closes the stream", async () => { + const transport = new FakeTransport([]); + transport.openSSE = async (path, _params, init = {}) => { + transport.calls.push({ path, init }); + const events = (async function* () { + yield { event: "output", data: {} } as RuntimeSSEEvent; + })(); + return { events, close: () => (transport.closed = true) }; + }; + await expect(new SandboxProcessesApi(transport.asTransport()).start("test")).rejects.toThrow( + /Expected process start event/ + ); + expect(transport.closed).toBe(true); + }); +}); diff --git a/tests/sandbox/e2e/public-types-contract.test.ts b/tests/unit/public-types-contract.test.ts similarity index 60% rename from tests/sandbox/e2e/public-types-contract.test.ts rename to tests/unit/public-types-contract.test.ts index 5fbc9eb..3cfc02e 100644 --- a/tests/sandbox/e2e/public-types-contract.test.ts +++ b/tests/unit/public-types-contract.test.ts @@ -3,15 +3,18 @@ import type { CompleteSandboxImageBuildParams, CreateSandboxImageBuildParams, Sandbox, + SandboxExecParams, + SandboxImageBuildInputFormat, SandboxImageBuildListParams, SandboxImageBuildStatus, + SandboxProcessResult, SandboxImageListResponse, SandboxNetworkPolicy, SandboxSnapshotListResponse, SessionRegion, SessionStatus, VolumeListResponse, -} from "../../../src/types"; +} from "../../src/types"; describe("public type compatibility", () => { test("keeps newly available response data optional", () => { @@ -22,7 +25,7 @@ describe("public type compatibility", () => { expect(imageResponse.totalCount).toBeUndefined(); expect(snapshotResponse.page).toBeUndefined(); expect(volumeResponse.perPage).toBeUndefined(); - expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); }); test("includes public server region and status values", () => { @@ -33,18 +36,30 @@ describe("public type compatibility", () => { expect(status).toBe("close-error"); }); - test("only accepts the image build format and platform supported by the server", () => { + test("only accepts the image build formats and platform supported by the server", () => { expectTypeOf().toEqualTypeOf< - "rootfs_export_tar_gz" | undefined + SandboxImageBuildInputFormat | undefined + >(); + expectTypeOf().toEqualTypeOf< + | "rootfs_export_tar_gz" + | "dockerfile_context_tar_gz" + | "dockerfile_context_manifest_v1" + | "docker_image_manifest_v1" >(); expectTypeOf().toEqualTypeOf< "linux/amd64" | undefined >(); expectTypeOf().toEqualTypeOf< - "rootfs_export_tar_gz" | undefined + SandboxImageBuildInputFormat | undefined >(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf< SandboxImageBuildStatus | undefined >(); + expectTypeOf<"cancelled">().not.toExtend(); + expectTypeOf<"BUILDING">().not.toExtend(); + expectTypeOf<"canceled">().toExtend(); }); }); diff --git a/tests/unit/python-reference.test.ts b/tests/unit/python-reference.test.ts new file mode 100644 index 0000000..f8cafc1 --- /dev/null +++ b/tests/unit/python-reference.test.ts @@ -0,0 +1,85 @@ +import { deriveAutoImageInit, mergeImageInit } from "../../src/sandbox/image-build/image-init"; +import { HyperbrowserError } from "../../src/error"; +import { ProcessOutput } from "../../src/sandbox/process-output"; +import { imageBuildName, ImageBuildNameOptions } from "../../src/sandbox/image-build/resolution"; +import { expect, test } from "vitest"; +import { readFileSync } from "fs"; +import path from "path"; +// Preserve literal __proto__ keys; transformed JSON object literals can lose them. +const reference: typeof import("../fixtures/python_reference.json") = JSON.parse( + readFileSync(path.join(__dirname, "../fixtures/python_reference.json"), "utf8") +); +import { DockerIgnoreMatcher } from "../../src/sandbox/image-build/dockerignore"; +import { analyzeDockerfileSources } from "../../src/sandbox/image-build/dockerfile-analysis"; + +test.each(reference.ignore)("Python ignore: $pattern → $path", ({ pattern, path, expected }) => { + expect(DockerIgnoreMatcher.fromText(pattern).matches(path)).toBe(expected); +}); +test.each(reference.invalidIgnore)("Python rejects invalid ignore %s", (pattern) => { + expect(() => DockerIgnoreMatcher.fromText(pattern)).toThrow(); +}); +test.each(reference.dockerfile)( + "Python Dockerfile: $dockerfile", + ({ dockerfile, groups, fallback }) => { + expect(analyzeDockerfileSources(Buffer.from(dockerfile))).toEqual({ + groups, + fallbackReason: fallback, + }); + } +); +test("invalid UTF-8 Dockerfiles fall back to full context", () => { + expect(analyzeDockerfileSources(Buffer.from([0xff]))).toEqual({ + groups: [], + fallbackReason: "dockerfile_parse_failed", + }); +}); +test("ignore preprocessing, comments and root match Python", () => { + const matcher = DockerIgnoreMatcher.fromText( + "\ufeff# comment\n /build/../node_modules/ \n! /node_modules/keep.js\n" + ); + expect(matcher.hasNegations).toBe(true); + expect(matcher.matches("node_modules/drop.js")).toBe(true); + expect(matcher.matches("node_modules/keep.js")).toBe(false); + expect(matcher.matches("nested/node_modules/drop.js")).toBe(false); + expect(DockerIgnoreMatcher.fromText(" #literal\n# actual comment\n").matches("#literal")).toBe( + true + ); + expect(DockerIgnoreMatcher.fromText("**\n").matches(".")).toBe(false); +}); + +test.each(reference.imageNames)( + "image identity matches Python: $options", + ({ options, expected }) => { + expect(imageBuildName(options as ImageBuildNameOptions)).toBe(expected); + } +); +test.each(reference.processes)("process collection matches Python: $name", (fixture) => { + const output = new ProcessOutput("p", fixture.limit); + const consume = () => { + for (const event of fixture.events) output.consume(event); + }; + if ("error" in fixture) { + let caught: unknown; + try { + consume(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(HyperbrowserError); + expect(caught).toMatchObject(fixture.error!); + } else { + consume(); + expect(output.result).toMatchObject(fixture.expected!); + } +}); + +test.each(reference.imageInit)( + "Docker initialization matches Python: $config / $explicit", + (fixture) => { + const automatic = deriveAutoImageInit(fixture.config); + expect(automatic ?? null).toEqual(fixture.automatic); + expect(mergeImageInit(automatic, fixture.explicit ?? undefined) ?? null).toEqual( + fixture.expected + ); + } +); diff --git a/tests/sandbox/e2e/runtime-transport.test.ts b/tests/unit/runtime-transport.test.ts similarity index 98% rename from tests/sandbox/e2e/runtime-transport.test.ts rename to tests/unit/runtime-transport.test.ts index 94be667..2960efd 100644 --- a/tests/sandbox/e2e/runtime-transport.test.ts +++ b/tests/unit/runtime-transport.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from "vitest"; import { resolveRuntimeTransportTarget, toWebSocketUrl, -} from "../../../src/sandbox/ws"; +} from "../../src/sandbox/ws"; const ORIGINAL_REGIONAL_PROXY_DEV_HOST = process.env.REGIONAL_PROXY_DEV_HOST; diff --git a/tests/sandbox/e2e/sandbox-contract.test.ts b/tests/unit/sandbox-contract.test.ts similarity index 91% rename from tests/sandbox/e2e/sandbox-contract.test.ts rename to tests/unit/sandbox-contract.test.ts index bb23fd3..ae1a51a 100644 --- a/tests/sandbox/e2e/sandbox-contract.test.ts +++ b/tests/unit/sandbox-contract.test.ts @@ -1,9 +1,9 @@ import { describe, expect, test, vi, afterEach } from "vitest"; -import { SandboxFilesApi } from "../../../src/sandbox/files"; -import { SandboxTerminalHandle } from "../../../src/sandbox/terminal"; -import * as wsModule from "../../../src/sandbox/ws"; -import { SandboxesService } from "../../../src/services/sandboxes"; -import type { SandboxExposeResult } from "../../../src/types"; +import { SandboxFilesApi } from "../../src/sandbox/files"; +import { SandboxTerminalHandle } from "../../src/sandbox/terminal"; +import * as wsModule from "../../src/sandbox/ws"; +import { SandboxesService } from "../../src/services/sandboxes"; +import type { SandboxExposeResult } from "../../src/types"; const parseJsonRequestBody = (init: unknown): unknown => { if (!init || typeof init !== "object" || !("body" in init) || typeof init.body !== "string") { @@ -117,6 +117,24 @@ describe("sandbox control and runtime contract", () => { expect(sandbox.getExposedUrl(3000)).toBe("https://3000-sbx_123.runtime.example.com/"); }); + test("create forwards the explicit CPU runtime and retains its capabilities", async () => { + const service = new SandboxesService("test-key", "https://api.example.com", 30_000); + const requestSpy = vi.spyOn(service as any, "request").mockResolvedValue({ + ...wireSandboxDetail(), + runtimeClass: "gvisor-cpu", + capabilities: { pty: true, gpu: false }, + }); + const sandbox = await service.create({ imageName: "cpu", runtimeClass: "gvisor-cpu" }); + expect(parseJsonRequestBody(requestSpy.mock.calls[0][1])).toEqual({ + imageName: "cpu", + runtimeClass: "gvisor-cpu", + }); + expect(sandbox.toJSON()).toMatchObject({ + runtimeClass: "gvisor-cpu", + capabilities: { pty: true, gpu: false }, + }); + }); + test("create forwards mounts for snapshot launches", async () => { const service = new SandboxesService("test-key", "https://api.example.com", 30_000); const requestSpy = vi.spyOn(service as any, "request").mockResolvedValue(wireSandboxDetail()); @@ -148,23 +166,15 @@ describe("sandbox control and runtime contract", () => { }); }); - test("create leaves resource value validation to the server", async () => { + test("create validates resource values before a request", async () => { const service = new SandboxesService("test-key", "https://api.example.com", 30_000); const requestSpy = vi.spyOn(service as any, "request").mockResolvedValue(wireSandboxDetail()); - - await service.create({ - imageName: "node", - cpu: 0, - memoryMiB: -1, - diskMiB: 1.5, - }); - - expect(parseJsonRequestBody(requestSpy.mock.calls[0][1])).toEqual({ - imageName: "node", - vcpus: 0, - memMiB: -1, - diskSizeMiB: 1.5, - }); + for (const resources of [{ cpu: 0 }, { memoryMiB: -1 }, { diskMiB: 1.5 }]) { + await expect(service.create({ imageName: "node", ...resources })).rejects.toThrow( + /positive integer/ + ); + } + expect(requestSpy).not.toHaveBeenCalled(); }); test("create treats an explicitly undefined imageName as a snapshot launch", async () => { diff --git a/tests/sandbox/e2e/volumes-contract.test.ts b/tests/unit/volumes-contract.test.ts similarity index 97% rename from tests/sandbox/e2e/volumes-contract.test.ts rename to tests/unit/volumes-contract.test.ts index fb43491..e8521ac 100644 --- a/tests/sandbox/e2e/volumes-contract.test.ts +++ b/tests/unit/volumes-contract.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, vi } from "vitest"; -import { VolumesService } from "../../../src/services/volumes"; +import { VolumesService } from "../../src/services/volumes"; describe("volume control contract", () => { test("create forwards payload and returns created volume", async () => { diff --git a/tsconfig.tests.json b/tsconfig.tests.json index eb7c83b..df734c5 100644 --- a/tsconfig.tests.json +++ b/tsconfig.tests.json @@ -2,7 +2,8 @@ "extends": "./tsconfig.json", "compilerOptions": { "noEmit": true, - "rootDir": "." + "rootDir": ".", + "resolveJsonModule": true }, "include": ["src/**/*.ts", "tests/**/*.ts"], "exclude": ["dist", "node_modules"] diff --git a/vitest.config.ts b/vitest.config.ts index 2ea13a1..82daa7f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,12 +2,29 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { - include: ["tests/integration/**/*.test.ts", "tests/sandbox/e2e/**/*.test.ts"], - setupFiles: ["./tests/load-env.ts"], + project: ["unit", "integration"], environment: "node", fileParallelism: false, testTimeout: 120_000, hookTimeout: 120_000, reporters: ["default"], + projects: [ + { + extends: true, + test: { name: "unit", include: ["tests/unit/**/*.test.ts"] }, + }, + { + extends: true, + test: { name: "integration", include: ["tests/integration/**/*.test.ts"] }, + }, + { + extends: true, + test: { + name: "e2e", + include: ["tests/e2e/**/*.test.ts"], + setupFiles: ["./tests/load-env.ts"], + }, + }, + ], }, });