From b243b99e76286111c0d498ba2163b9e414704c49 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 16:11:16 -0400 Subject: [PATCH 001/178] v2 core: JCS, input rules, content-addressed trees, version roots @underlay/core implements the protocol half of the edge redesign: RFC 8785 canonical JSON, the input-rule scanner (duplicate keys, unsafe integer literals, lone surrogates, depth), record and schema hashing with v1 legacy hashes, the key-defined tree (streaming builder, incremental merge, seek/iterate/diff, fsck), and version roots with the salted private commitment. Interior entries carry the child's last key and record leaves carry the record size; see edge-redesign-build.md findings 1-2. --- packages/core/package.json | 22 ++ packages/core/src/constants.ts | 43 ++++ packages/core/src/file-refs.ts | 31 +++ packages/core/src/hash.ts | 80 +++++++ packages/core/src/index.ts | 50 ++++ packages/core/src/input-rules.ts | 272 ++++++++++++++++++++++ packages/core/src/jcs.ts | 54 +++++ packages/core/src/root.ts | 129 +++++++++++ packages/core/src/tree/builder.ts | 241 +++++++++++++++++++ packages/core/src/tree/chunking.ts | 84 +++++++ packages/core/src/tree/merge.ts | 189 +++++++++++++++ packages/core/src/tree/node.ts | 257 +++++++++++++++++++++ packages/core/src/tree/read.ts | 194 ++++++++++++++++ packages/core/src/tree/source.ts | 58 +++++ packages/core/src/tree/verify.ts | 89 +++++++ packages/core/src/utf8.ts | 50 ++++ packages/core/test/input-rules.test.ts | 141 +++++++++++ packages/core/test/jcs.test.ts | 96 ++++++++ packages/core/test/tree.test.ts | 308 +++++++++++++++++++++++++ packages/core/tsconfig.json | 17 ++ packages/core/vitest.config.ts | 7 + pnpm-lock.yaml | 31 +++ vitest.config.ts | 2 +- 23 files changed, 2444 insertions(+), 1 deletion(-) create mode 100644 packages/core/package.json create mode 100644 packages/core/src/constants.ts create mode 100644 packages/core/src/file-refs.ts create mode 100644 packages/core/src/hash.ts create mode 100644 packages/core/src/index.ts create mode 100644 packages/core/src/input-rules.ts create mode 100644 packages/core/src/jcs.ts create mode 100644 packages/core/src/root.ts create mode 100644 packages/core/src/tree/builder.ts create mode 100644 packages/core/src/tree/chunking.ts create mode 100644 packages/core/src/tree/merge.ts create mode 100644 packages/core/src/tree/node.ts create mode 100644 packages/core/src/tree/read.ts create mode 100644 packages/core/src/tree/source.ts create mode 100644 packages/core/src/tree/verify.ts create mode 100644 packages/core/src/utf8.ts create mode 100644 packages/core/test/input-rules.test.ts create mode 100644 packages/core/test/jcs.test.ts create mode 100644 packages/core/test/tree.test.ts create mode 100644 packages/core/tsconfig.json create mode 100644 packages/core/vitest.config.ts diff --git a/packages/core/package.json b/packages/core/package.json new file mode 100644 index 0000000..a62ca03 --- /dev/null +++ b/packages/core/package.json @@ -0,0 +1,22 @@ +{ + "name": "@underlay/core", + "version": "0.0.0", + "private": true, + "description": "Underlay protocol v2: canonical JSON, hashing, input rules, content-addressed trees, version roots.", + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit", + "vectors": "tsx scripts/gen-vectors.ts" + }, + "devDependencies": { + "@types/node": "^25.0.0", + "fast-check": "^4.3.0", + "tsx": "^4.19.0", + "typescript": "^6.0.0", + "vitest": "^4.1.6" + } +} diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts new file mode 100644 index 0000000..c3df3ad --- /dev/null +++ b/packages/core/src/constants.ts @@ -0,0 +1,43 @@ +/** + * Protocol constants for Underlay format 2. + * + * Everything here is protocol: a second implementation must use the same values + * or it will build different trees, accept different records, and compute + * different version hashes. None of them may change without a new format + * number. See docs/protocol-v2.md. + * + * Status: PROVISIONAL until the tree-parameter experiments are done and the + * values are frozen (edge-redesign-build.md, "Decisions made while building"). + */ + +/** The `underlay` field of every version root. */ +export const FORMAT_VERSION = 2 + +/** Prefix of a format-2 version hash string: `ulv2:<64 hex>`. */ +export const VERSION_HASH_PREFIX = 'ulv2:' + +// --- Tree shape ------------------------------------------------------------- + +/** A leaf ends after a key whose boundary hash has at least this many trailing zero bits (mean 1,024 entries). */ +export const LEAF_BOUNDARY_BITS = 10 +/** Forced leaf split: a leaf never holds more entries than this. */ +export const LEAF_MAX_ENTRIES = 16_384 +/** Level i ≥ 1 needs LEAF_BOUNDARY_BITS + i × this many trailing zero bits (mean fanout 64). */ +export const INTERIOR_BOUNDARY_BITS_STEP = 6 +/** Forced interior split: an interior node never holds more children than this. */ +export const INTERIOR_MAX_CHILDREN = 1_024 + +// --- Input rules ------------------------------------------------------------ + +/** Largest integer literal magnitude accepted (2^53 − 1). */ +export const MAX_SAFE_INTEGER_LITERAL = '9007199254740991' +/** Maximum nesting depth of a record's JSON (the envelope object is depth 1). */ +export const MAX_JSON_DEPTH = 64 +/** Maximum canonical record size in bytes: `{"id":…,"type":…,"data":…}` as UTF-8. */ +export const MAX_RECORD_BYTES = 8 * 1024 * 1024 +/** Maximum record id length in UTF-8 bytes. Bounds leaf node size. */ +export const MAX_ID_BYTES = 1_024 +/** Maximum type slug length in UTF-8 bytes. */ +export const MAX_TYPE_BYTES = 128 +/** Maximum canonical schema size in bytes. */ +export const MAX_SCHEMA_BYTES = 256 * 1024 diff --git a/packages/core/src/file-refs.ts b/packages/core/src/file-refs.ts new file mode 100644 index 0000000..631305b --- /dev/null +++ b/packages/core/src/file-refs.ts @@ -0,0 +1,31 @@ +/** + * File references in record data. + * + * One rule (v1 had two): a reference is any object, at any depth, whose `$file` + * is a string `sha256:<64 lowercase hex>`. The walk doesn't descend into a + * reference object. Returns bare hex hashes, deduplicated, in first-seen order. + */ +const FILE_REF = /^sha256:([0-9a-f]{64})$/ + +export function fileRefs(data: unknown): string[] { + const out = new Set() + const walk = (v: unknown) => { + if (v === null || typeof v !== 'object') return + if (Array.isArray(v)) { + for (const x of v) walk(x) + return + } + const o = v as Record + const ref = o.$file + if (typeof ref === 'string') { + const m = FILE_REF.exec(ref) + if (m) { + out.add(m[1]!) + return + } + } + for (const k in o) walk(o[k]) + } + walk(data) + return [...out] +} diff --git a/packages/core/src/hash.ts b/packages/core/src/hash.ts new file mode 100644 index 0000000..85ac329 --- /dev/null +++ b/packages/core/src/hash.ts @@ -0,0 +1,80 @@ +import { createHash } from 'node:crypto' + +import { jcs } from './jcs.js' + +/** Lowercase hex SHA-256 of a string (as UTF-8) or bytes. */ +export function sha256Hex(input: string | Uint8Array): string { + return createHash('sha256').update(input).digest('hex') +} + +/** + * The canonical record: a fixed `{"id","type","data"}` envelope with only `data` + * canonicalized by JCS. The envelope keeps today's field order so that records + * without integer-like keys keep their v1 hashes; JCS over the whole record would + * sort it to `data, id, type` and change every hash. + */ +export function recordCanonical(id: string, type: string, data: unknown): string { + return ( + '{"id":' + JSON.stringify(id) + ',"type":' + JSON.stringify(type) + ',"data":' + jcs(data) + '}' + ) +} + +export function hashRecord( + id: string, + type: string, + data: unknown, +): { hash: string; canonical: string } { + const canonical = recordCanonical(id, type, data) + return { hash: sha256Hex(canonical), canonical } +} + +export function hashSchema(schema: unknown): string { + return sha256Hex(jcs(schema)) +} + +// --- Legacy (format 1) hashing ---------------------------------------------- +// +// Format 1 canonicalized by sorting keys into a new object and stringifying it, +// so integer-like keys came out first in numeric order. Kept for the negotiate +// compatibility layer and migration: v1 clients still send these hashes. + +/** An "array index" key: JS enumerates these first, in numeric order. */ +function isArrayIndexKey(k: string): boolean { + if (k === '0') return true + if (k.length === 0 || k.length > 10 || k.charCodeAt(0) < 0x31 || k.charCodeAt(0) > 0x39) { + return false + } + for (let i = 1; i < k.length; i++) { + const c = k.charCodeAt(i) + if (c < 0x30 || c > 0x39) return false + } + return Number(k) < 4294967295 +} + +/** True when some object at any depth has an array-index key, the only case where v1 and v2 hashes can differ. */ +export function hasArrayIndexKey(value: unknown): boolean { + if (value === null || typeof value !== 'object') return false + if (Array.isArray(value)) return value.some(hasArrayIndexKey) + for (const [k, v] of Object.entries(value as Record)) { + if (isArrayIndexKey(k) || hasArrayIndexKey(v)) return true + } + return false +} + +function legacyCanonicalize(value: unknown): unknown { + if (value === null || typeof value !== 'object') return value + if (Array.isArray(value)) return value.map(legacyCanonicalize) + const sorted: Record = {} + for (const key of Object.keys(value as Record).sort()) { + sorted[key] = legacyCanonicalize((value as Record)[key]) + } + return sorted +} + +export function legacyRecordHash(id: string, type: string, data: unknown): string { + return sha256Hex(JSON.stringify({ id, type, data: legacyCanonicalize(data) })) +} + +export function legacySchemaHash(schema: unknown): string { + return sha256Hex(JSON.stringify(legacyCanonicalize(schema))) +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts new file mode 100644 index 0000000..a8da0c9 --- /dev/null +++ b/packages/core/src/index.ts @@ -0,0 +1,50 @@ +export * from './constants.js' +export { jcs } from './jcs.js' +export { compareUtf8, utf8, utf8ByteLength } from './utf8.js' +export { + hashRecord, + hashSchema, + hasArrayIndexKey, + legacyRecordHash, + legacySchemaHash, + recordCanonical, + sha256Hex, +} from './hash.js' +export { + checkRecordId, + checkTypeSlug, + InputRuleError, + type InputRuleCode, + parseRecordLine, + parseStrict, + type RecordInput, + scanJson, +} from './input-rules.js' +export { fileRefs } from './file-refs.js' +export * from './root.js' +export * from './tree/node.js' +export { + boundaryBytes, + type Chunking, + fixedChunking, + type FixedChunkingParams, + protocolChunking, + trailingZeros, +} from './tree/chunking.js' +export { + type BuilderOptions, + buildTree, + MemorySink, + TreeBuilder, + type TreeSink, +} from './tree/builder.js' +export { MapSource, type NodeSource, resolveRoot, rootDesc } from './tree/source.js' +export { + type Change, + mergeTree, + type MergeOptions, + type MergeResult, + type MergeStats, +} from './tree/merge.js' +export { type DiffEntry, diffTrees, getEntry, iterate, type IterateOptions } from './tree/read.js' +export { verifyTree, type VerifyResult } from './tree/verify.js' diff --git a/packages/core/src/input-rules.ts b/packages/core/src/input-rules.ts new file mode 100644 index 0000000..d299fdd --- /dev/null +++ b/packages/core/src/input-rules.ts @@ -0,0 +1,272 @@ +/** + * Input rules (protocol v2, "Input rules"). + * + * `JSON.parse` silently changes some inputs: it rounds large integers, keeps only + * the last of duplicate keys, and accepts lone UTF-16 surrogates. A record changed + * that way is not what the client sent, and an implementation in another language + * would hash something different. So v2 rejects such input, and it has to look at + * the source text to do it: after parsing, `1e20` and `100000000000000000000` are + * the same double and duplicate keys are gone. + * + * `scanJson` is a single pass over the text that checks the rules without building + * values. `JSON.parse` still builds the value; the scan only rejects. Both the CLI + * and the server run the same scan, so they accept exactly the same inputs. + */ +import { + MAX_ID_BYTES, + MAX_JSON_DEPTH, + MAX_RECORD_BYTES, + MAX_SAFE_INTEGER_LITERAL, + MAX_TYPE_BYTES, +} from './constants.js' +import { recordCanonical } from './hash.js' +import { utf8ByteLength } from './utf8.js' + +export type InputRuleCode = + | 'syntax' + | 'duplicate_key' + | 'unsafe_integer' + | 'lone_surrogate' + | 'too_deep' + | 'record_too_large' + | 'bad_id' + | 'bad_type' + | 'bad_envelope' + +export class InputRuleError extends Error { + constructor( + readonly code: InputRuleCode, + message: string, + ) { + super(message) + this.name = 'InputRuleError' + } +} + +const enum C { + Quote = 0x22, + Backslash = 0x5c, + OpenBrace = 0x7b, + CloseBrace = 0x7d, + OpenBracket = 0x5b, + CloseBracket = 0x5d, + Comma = 0x2c, + Colon = 0x3a, + Minus = 0x2d, + Zero = 0x30, + Nine = 0x39, + Dot = 0x2e, + LowerE = 0x65, + UpperE = 0x45, + LowerU = 0x75, +} + +interface Frame { + isObject: boolean + keys: Set | null + expectKey: boolean +} + +const isHigh = (u: number) => u >= 0xd800 && u <= 0xdbff +const isLow = (u: number) => u >= 0xdc00 && u <= 0xdfff + +/** + * Check `text` against the input rules: no duplicate object keys (compared after + * unescaping), no integer literal outside ±(2^53 − 1), no lone surrogate (raw or + * `\u`-escaped), nesting no deeper than `maxDepth` (each object or array is one + * level). Throws InputRuleError on the first violation. + * + * Syntax is left to `JSON.parse`; on malformed input this may report a rule + * violation instead of a syntax error, but it always terminates and never + * accepts anything `JSON.parse` would reject. + */ +export function scanJson(text: string, maxDepth: number = MAX_JSON_DEPTH): void { + const n = text.length + const stack: Frame[] = [] + let i = 0 + while (i < n) { + const c = text.charCodeAt(i) + if (c === C.Quote) { + const start = i + i++ + let escaped = false + let pendingHigh = false + for (;;) { + if (i >= n) throw new InputRuleError('syntax', 'Unterminated string') + let u = text.charCodeAt(i) + if (u === C.Quote) break + if (u === C.Backslash) { + escaped = true + const e = text.charCodeAt(i + 1) + if (e === C.LowerU) { + u = parseInt(text.slice(i + 2, i + 6), 16) + if (Number.isNaN(u)) throw new InputRuleError('syntax', 'Bad \\u escape') + i += 6 + } else { + u = 0 // any other escape is a non-surrogate code unit + i += 2 + } + } else { + i++ + } + if (pendingHigh) { + if (isLow(u)) { + pendingHigh = false + continue + } + throw new InputRuleError('lone_surrogate', 'Lone UTF-16 surrogate in string') + } + if (isHigh(u)) pendingHigh = true + else if (isLow(u)) + throw new InputRuleError('lone_surrogate', 'Lone UTF-16 surrogate in string') + } + if (pendingHigh) throw new InputRuleError('lone_surrogate', 'Lone UTF-16 surrogate in string') + i++ // closing quote + const top = stack[stack.length - 1] + if (top?.isObject && top.expectKey) { + const raw = text.slice(start + 1, i - 1) + const key = escaped ? (JSON.parse(text.slice(start, i)) as string) : raw + top.keys ??= new Set() + if (top.keys.has(key)) { + throw new InputRuleError('duplicate_key', `Duplicate object key ${JSON.stringify(key)}`) + } + top.keys.add(key) + top.expectKey = false + } + continue + } + if (c === C.OpenBrace || c === C.OpenBracket) { + if (stack.length >= maxDepth) { + throw new InputRuleError('too_deep', `JSON nested deeper than ${maxDepth} levels`) + } + stack.push({ isObject: c === C.OpenBrace, keys: null, expectKey: c === C.OpenBrace }) + i++ + continue + } + if (c === C.CloseBrace || c === C.CloseBracket) { + stack.pop() + i++ + continue + } + if (c === C.Comma) { + const top = stack[stack.length - 1] + if (top?.isObject) top.expectKey = true + i++ + continue + } + if (c === C.Minus || (c >= C.Zero && c <= C.Nine)) { + let j = c === C.Minus ? i + 1 : i + const digitsStart = j + while (j < n && text.charCodeAt(j) >= C.Zero && text.charCodeAt(j) <= C.Nine) j++ + const next = text.charCodeAt(j) + const isInteger = next !== C.Dot && next !== C.LowerE && next !== C.UpperE + if (isInteger) { + const len = j - digitsStart + const max = MAX_SAFE_INTEGER_LITERAL + if (len > max.length || (len === max.length && text.slice(digitsStart, j) > max)) { + throw new InputRuleError( + 'unsafe_integer', + `Integer ${text.slice(i, j)} is outside ±(2^53−1); send large integers as strings`, + ) + } + i = j + } else { + // Fraction and/or exponent: skip the rest of the number token. + j++ + while (j < n) { + const d = text.charCodeAt(j) + if ( + (d >= C.Zero && d <= C.Nine) || + d === 0x2b || + d === C.Minus || + d === C.LowerE || + d === C.UpperE || + d === C.Dot + ) + j++ + else break + } + i = j + } + continue + } + // Whitespace, colon, and the literals true/false/null. + i++ + } +} + +/** `scanJson` then `JSON.parse`. Throws InputRuleError (code `syntax` for parse errors). */ +export function parseStrict(text: string, maxDepth: number = MAX_JSON_DEPTH): unknown { + scanJson(text, maxDepth) + try { + return JSON.parse(text) + } catch (err) { + throw new InputRuleError('syntax', `Invalid JSON: ${(err as Error).message}`) + } +} + +export interface RecordInput { + id: string + type: string + data: unknown + private?: boolean +} + +/** Check a record id against the protocol limits. Returns an error message or null. */ +export function checkRecordId(id: unknown): string | null { + if (typeof id !== 'string' || id.length === 0) return 'Record id must be a non-empty string' + if (utf8ByteLength(id) > MAX_ID_BYTES) return `Record id exceeds ${MAX_ID_BYTES} bytes` + return null +} + +/** + * Check a type slug. Slugs become export entry names (`records/.ndjson`), + * so path separators, control characters and a leading dot are refused. + */ +export function checkTypeSlug(type: unknown): string | null { + if (typeof type !== 'string' || type.length === 0) return 'Type must be a non-empty string' + if (utf8ByteLength(type) > MAX_TYPE_BYTES) return `Type exceeds ${MAX_TYPE_BYTES} bytes` + if (type.startsWith('.')) return 'Type must not start with "."' + for (let i = 0; i < type.length; i++) { + const u = type.charCodeAt(i) + if (u === 0x2f || u === 0x5c || u < 0x20 || u === 0x7f) { + return 'Type must not contain slashes or control characters' + } + } + return null +} + +/** + * Parse one pushed record line, `{"id":…,"type":…,"data":…,"private"?:…}`, under + * the input rules. The envelope counts as one nesting level, so `data` may nest + * MAX_JSON_DEPTH levels deep. Size is checked on the canonical form, which is what + * is stored and hashed; the canonical string is returned so callers don't build it + * twice. + */ +export function parseRecordLine(line: string): RecordInput & { canonical: string } { + const value = parseStrict(line, MAX_JSON_DEPTH + 1) + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new InputRuleError('bad_envelope', 'Record must be a JSON object') + } + const rec = value as Record + const idError = checkRecordId(rec.id) + if (idError) throw new InputRuleError('bad_id', idError) + const typeError = checkTypeSlug(rec.type) + if (typeError) throw new InputRuleError('bad_type', typeError) + if (!('data' in rec)) throw new InputRuleError('bad_envelope', 'Record is missing "data"') + if (rec.private !== undefined && typeof rec.private !== 'boolean') { + throw new InputRuleError('bad_envelope', '"private" must be a boolean') + } + const canonical = recordCanonical(rec.id as string, rec.type as string, rec.data) + if (utf8ByteLength(canonical) > MAX_RECORD_BYTES) { + throw new InputRuleError('record_too_large', `Record exceeds ${MAX_RECORD_BYTES} bytes`) + } + const out: RecordInput & { canonical: string } = { + id: rec.id as string, + type: rec.type as string, + data: rec.data, + canonical, + } + if (rec.private !== undefined) out.private = rec.private as boolean + return out +} diff --git a/packages/core/src/jcs.ts b/packages/core/src/jcs.ts new file mode 100644 index 0000000..cf934f2 --- /dev/null +++ b/packages/core/src/jcs.ts @@ -0,0 +1,54 @@ +/** + * RFC 8785 JSON Canonicalization Scheme (JCS). + * + * JCS is `JSON.stringify` for every primitive (ECMAScript number formatting and + * string escaping) plus object keys sorted by UTF-16 code units. The one trap is + * JavaScript objects themselves: they enumerate integer-like keys ("9", "10") + * first, in numeric order, whatever order they were inserted in. So "sort the keys + * into a new object, then stringify" cannot produce sorted output for those keys. + * This writes objects out as strings instead, and never relies on enumeration + * order. + * + * Inputs come from JSON, so they never hold `undefined`, functions, bigints or + * non-finite numbers. Those throw rather than serialize to something a different + * implementation would not produce. + */ +export function jcs(value: unknown): string { + switch (typeof value) { + case 'string': + return JSON.stringify(value) + case 'boolean': + return value ? 'true' : 'false' + case 'number': + if (!Number.isFinite(value)) throw new TypeError('JCS: non-finite number') + // JSON.stringify renders -0 as "0", which is what JCS requires. + return JSON.stringify(value) + case 'object': { + if (value === null) return 'null' + if (Array.isArray(value)) { + let out = '[' + for (let i = 0; i < value.length; i++) { + if (i > 0) out += ',' + out += jcs(value[i]) + } + return out + ']' + } + const o = value as Record + // Array.prototype.sort() with no comparator compares UTF-16 code units, + // which is exactly JCS's key order. + const keys = Object.keys(o).sort() + let out = '{' + let first = true + for (const k of keys) { + const v = o[k] + if (v === undefined) continue + if (!first) out += ',' + first = false + out += JSON.stringify(k) + ':' + jcs(v) + } + return out + '}' + } + default: + throw new TypeError(`JCS: cannot serialize ${typeof value}`) + } +} diff --git a/packages/core/src/root.ts b/packages/core/src/root.ts new file mode 100644 index 0000000..986e66b --- /dev/null +++ b/packages/core/src/root.ts @@ -0,0 +1,129 @@ +/** + * Version roots, set objects and the private-set commitment. + * + * root = {"underlay":2,"metadata":{…}|null,"public":SetObject,"private":commitment|null} + * SetObject = {"types":{slug:{"schema","root","count","bytes"}},"files":{"root","count","bytes"}} + * private set object = SetObject + {"salt": 64 hex} + * commitment = sha256(JCS(private set object)) + * version hash = "ulv2:" + sha256(JCS(root)) + */ +import { randomBytes } from 'node:crypto' + +import { FORMAT_VERSION, VERSION_HASH_PREFIX } from './constants.js' +import { sha256Hex } from './hash.js' +import { jcs } from './jcs.js' + +export interface TreeSummary { + root: string | null + count: number + bytes: number +} + +export interface TypeEntry extends TreeSummary { + schema: string +} + +export interface SetObject { + types: Record + files: TreeSummary +} + +export interface PrivateSetObject extends SetObject { + salt: string +} + +export interface VersionRoot { + underlay: typeof FORMAT_VERSION + metadata: Record | null + public: SetObject + private: string | null +} + +export const EMPTY_TREE: TreeSummary = Object.freeze({ root: null, count: 0, bytes: 0 }) + +export function emptySet(): SetObject { + return { types: {}, files: { ...EMPTY_TREE } } +} + +/** A private set is empty when it has no types and no files; its commitment is then null. */ +export function isEmptySet(s: SetObject): boolean { + return Object.keys(s.types).length === 0 && s.files.count === 0 +} + +/** 32 random bytes as hex: one per collection, reused across its versions. */ +export function newSalt(): string { + return randomBytes(32).toString('hex') +} + +export function privateCommitment(p: PrivateSetObject): string { + return sha256Hex(jcs(p)) +} + +export function makeRoot( + metadata: Record | null, + pub: SetObject, + priv: PrivateSetObject | null, +): VersionRoot { + return { + underlay: FORMAT_VERSION, + metadata, + public: pub, + private: priv && !isEmptySet(priv) ? privateCommitment(priv) : null, + } +} + +export function encodeRoot(root: VersionRoot): string { + return jcs(root) +} + +export function versionHash(root: VersionRoot): string { + return VERSION_HASH_PREFIX + sha256Hex(encodeRoot(root)) +} + +/** The bare hex digest of a `ulv2:` version hash (the object key under roots/). */ +export function versionDigest(hash: string): string { + if (!hash.startsWith(VERSION_HASH_PREFIX)) throw new Error(`Not a v2 version hash: ${hash}`) + return hash.slice(VERSION_HASH_PREFIX.length) +} + +/** Totals over every type tree of a set (records only). */ +export function setRecordTotals(s: SetObject): { count: number; bytes: number } { + let count = 0 + let bytes = 0 + for (const t of Object.values(s.types)) { + count += t.count + bytes += t.bytes + } + return { count, bytes } +} + +const HEX64 = /^[0-9a-f]{64}$/ + +function checkSummary(t: unknown, where: string): string | null { + const s = t as TreeSummary + if (!s || typeof s !== 'object') return `${where}: not an object` + if (s.root !== null && (typeof s.root !== 'string' || !HEX64.test(s.root))) + return `${where}: bad root` + if (!Number.isSafeInteger(s.count) || s.count < 0) return `${where}: bad count` + if (!Number.isSafeInteger(s.bytes) || s.bytes < 0) return `${where}: bad bytes` + if ((s.root === null) !== (s.count === 0)) return `${where}: root and count disagree` + return null +} + +/** Shape check for a set object read from storage or a peer. Returns an error or null. */ +export function checkSetObject(s: unknown, isPrivate = false): string | null { + const set = s as PrivateSetObject + if (!set || typeof set !== 'object' || !set.types || typeof set.types !== 'object') + return 'set: bad types' + const expected = isPrivate ? 3 : 2 + if (Object.keys(set).length !== expected) return 'set: unexpected fields' + if (isPrivate && (typeof set.salt !== 'string' || !HEX64.test(set.salt))) return 'set: bad salt' + for (const [slug, t] of Object.entries(set.types)) { + const e = checkSummary(t, `type ${slug}`) + if (e) return e + if (typeof t.schema !== 'string' || !HEX64.test(t.schema)) + return `type ${slug}: bad schema hash` + if (Object.keys(t).length !== 4) return `type ${slug}: unexpected fields` + } + return checkSummary(set.files, 'files') +} diff --git a/packages/core/src/tree/builder.ts b/packages/core/src/tree/builder.ts new file mode 100644 index 0000000..d1bd963 --- /dev/null +++ b/packages/core/src/tree/builder.ts @@ -0,0 +1,241 @@ +/** + * Streaming tree builder. + * + * Feed entries in strictly increasing key order (UTF-8 byte order) with + * `addEntry`, or whole existing nodes with `addNode`, then call `finish` for the + * root. Each level has its own chunker; when a node ends, the builder hands it to + * the sink and pushes its descriptor into the level above. Memory is one pending + * node per level. + * + * `addNode` is what makes incremental commits O(changes): a node from the base + * tree can be reused unchanged when every level at or below it is empty, i.e. + * the output stream is sitting exactly on a boundary. The node's own end was a + * boundary in the base tree (or the end of the tree), and boundaries depend only + * on keys and the position since the previous boundary, so it ends in the same + * place here. Its descriptor goes into the level above as if it had just been + * built. + * + * The root is the only node of the first level that has exactly one node. A + * level above it may have started a node with that one child (when its last key + * also satisfies the higher boundary); that node is written but unreachable, + * which is harmless and rare. + */ +import { compareUtf8 } from '../utf8.js' +import { boundaryBytes, type Chunking, protocolChunking } from './chunking.js' +import { encodeInterior, encodeLeaf, hashNode, type NodeDesc, type TreeSpec } from './node.js' + +export interface TreeSink { + /** + * A finished leaf. `entries` are all of the leaf's entries; the first `spilled` + * of them were already passed to `spill` (and may have dropped their payloads). + */ + leaf(desc: NodeDesc, json: string, entries: readonly E[], spilled: number): void + interior(desc: NodeDesc, json: string, children: readonly NodeDesc[]): void + /** + * Called when the pending leaf's payload passes `spillBytes`, with the entries + * added since the last spill, so a sink can write record bodies in bounded parts + * instead of holding a whole leaf's bodies in memory. + */ + spill?(entries: readonly E[]): void +} + +export interface BuilderOptions { + chunking?: Chunking + /** Payload size of an entry (e.g. record body length). Needed for spilling. */ + payloadBytes?: (e: E) => number + /** Drop an entry's payload after it was spilled. */ + dropPayload?: (e: E) => void + /** Spill the pending leaf's payload when it reaches this many bytes. */ + spillBytes?: number +} + +export class TreeBuilder { + readonly #spec: TreeSpec + readonly #sink: TreeSink + readonly #chunking: Chunking + readonly #opts: BuilderOptions + + // Level 0: pending leaf entries. + #leaf: E[] = [] + #leafBytes = 0 + #spilled = 0 + #payload = 0 + // Level h ≥ 1: pending children (level h−1 descriptors) of the level-h node. + readonly #pending: NodeDesc[][] = [] + // Nodes completed (built or reused) at each level, and the most recent one. + readonly #nodes: number[] = [] + readonly #last: NodeDesc[] = [] + // The last interior node built at each level, with its children (for finish). + readonly #lastBuilt: ({ hash: string; children: NodeDesc[] } | undefined)[] = [] + #lastKey: string | null = null + #finished = false + + constructor(spec: TreeSpec, sink: TreeSink, opts: BuilderOptions = {}) { + this.#spec = spec + this.#sink = sink + this.#chunking = opts.chunking ?? protocolChunking + this.#opts = opts + } + + /** True when no level at or below `level` holds a partial node. */ + emptyThrough(level: number): boolean { + if (this.#leaf.length > 0) return false + for (let h = 1; h <= level; h++) { + if ((this.#pending[h]?.length ?? 0) > 0) return false + } + return true + } + + get lastKey(): string | null { + return this.#lastKey + } + + addEntry(e: E): void { + const key = this.#spec.key(e) + this.#advance(key) + this.#leaf.push(e) + this.#leafBytes += this.#spec.bytes(e) + const { payloadBytes, spillBytes } = this.#opts + if (payloadBytes && spillBytes !== undefined) { + this.#payload += payloadBytes(e) + if (this.#payload >= spillBytes) this.#spill() + } + if (this.#chunking.ends(0, boundaryBytes(key), this.#leaf.length)) this.#emitLeaf() + } + + /** Reuse an existing node. Requires `emptyThrough(desc.level)`. */ + addNode(desc: NodeDesc): void { + if (!this.emptyThrough(desc.level)) { + throw new Error('TreeBuilder.addNode: a lower level holds a partial node') + } + this.#advance(desc.lastKey) + this.#push(desc) + } + + /** + * Finish every level and return the root, or null for an empty tree. + * + * The root is the node of the lowest level that has exactly one node. Flush + * levels upward until the highest level holds a single node, then walk down + * while that node has a single child. A node reused through `addNode` has + * children the builder never saw; if the walk reaches one, `unresolved` is set + * and the caller finishes the walk by loading nodes (`resolveRoot`). + */ + finish(): { root: NodeDesc | null; unresolved: boolean } { + if (this.#finished) throw new Error('TreeBuilder.finish called twice') + this.#finished = true + if (this.#leaf.length > 0) this.#emitLeaf() + if (this.#nodes.length === 0) return { root: null, unresolved: false } + for (let h = 0; ; h++) { + const top = this.#nodes.length - 1 + if (h === top && this.#nodes[h] === 1) break + if ((this.#pending[h + 1]?.length ?? 0) > 0) this.#emitInterior(h + 1) + } + let root = this.#last[this.#nodes.length - 1]! + while (root.level > 0) { + // The chain down from the top is always the last node of each level. + const built = this.#lastBuilt[root.level] + const kids = built?.hash === root.hash ? built.children : undefined + if (!kids) return { root, unresolved: true } + if (kids.length !== 1) break + root = kids[0]! + } + return { root, unresolved: false } + } + + #advance(key: string): void { + if (this.#finished) throw new Error('TreeBuilder: add after finish') + if (this.#lastKey !== null && compareUtf8(this.#lastKey, key) >= 0) { + throw new Error( + `TreeBuilder: keys out of order (${JSON.stringify(key)} after ${JSON.stringify(this.#lastKey)})`, + ) + } + this.#lastKey = key + } + + #spill(): void { + const batch = this.#leaf.slice(this.#spilled) + this.#sink.spill?.(batch) + if (this.#opts.dropPayload) for (const e of batch) this.#opts.dropPayload(e) + this.#spilled = this.#leaf.length + this.#payload = 0 + } + + #emitLeaf(): void { + const entries = this.#leaf + const json = encodeLeaf(this.#spec, entries) + const desc: NodeDesc = { + level: 0, + hash: hashNode(json), + lastKey: this.#spec.key(entries[entries.length - 1]!), + count: entries.length, + bytes: this.#leafBytes, + } + this.#sink.leaf(desc, json, entries, this.#spilled) + this.#leaf = [] + this.#leafBytes = 0 + this.#spilled = 0 + this.#payload = 0 + this.#push(desc) + } + + #emitInterior(level: number): void { + const children = this.#pending[level]! + this.#pending[level] = [] + const json = encodeInterior(level, children) + let count = 0 + let bytes = 0 + for (const c of children) { + count += c.count + bytes += c.bytes + } + const desc: NodeDesc = { + level, + hash: hashNode(json), + lastKey: children[children.length - 1]!.lastKey, + count, + bytes, + } + this.#sink.interior(desc, json, children) + this.#lastBuilt[level] = { hash: desc.hash, children } + this.#push(desc) + } + + /** Record a completed level-h node and push it into level h+1. */ + #push(desc: NodeDesc): void { + const h = desc.level + this.#nodes[h] = (this.#nodes[h] ?? 0) + 1 + this.#last[h] = desc + const up = (this.#pending[h + 1] ??= []) + up.push(desc) + if (this.#chunking.ends(h + 1, boundaryBytes(desc.lastKey), up.length)) { + this.#emitInterior(h + 1) + } + } +} + +/** A sink that only collects nodes in memory: for tests, fsck and small trees. */ +export class MemorySink implements TreeSink { + readonly nodes = new Map() + readonly leaves = new Map() + leaf(desc: NodeDesc, json: string, entries: readonly E[]): void { + this.nodes.set(desc.hash, json) + this.leaves.set(desc.hash, entries.slice()) + } + interior(desc: NodeDesc, json: string): void { + this.nodes.set(desc.hash, json) + } +} + +/** Build a tree from entries already sorted by key. */ +export function buildTree( + spec: TreeSpec, + sink: TreeSink, + entries: Iterable, + opts?: BuilderOptions, +): NodeDesc | null { + const b = new TreeBuilder(spec, sink, opts) + for (const e of entries) b.addEntry(e) + // Nothing was reused, so the builder knows every node and the root is resolved. + return b.finish().root +} diff --git a/packages/core/src/tree/chunking.ts b/packages/core/src/tree/chunking.ts new file mode 100644 index 0000000..33ac524 --- /dev/null +++ b/packages/core/src/tree/chunking.ts @@ -0,0 +1,84 @@ +/** + * Where nodes end. + * + * Boundary hash: u(k) = the first 8 bytes of sha256(utf8(k)) as a big-endian + * unsigned 64-bit integer. Under the protocol rule a leaf ends after key k when + * u(k) mod 2^10 == 0 (equivalently: u(k) has at least 10 trailing zero bits), and + * a level-i node ends after a child whose last key has u mod 2^(10+6i) == 0. Each + * also ends at a forced maximum size, or at the end of its level. + * + * The rule is behind an interface only so the parameter experiments can compare + * alternatives (a size-aware chunker). The protocol has exactly one rule: + * `protocolChunking`. + */ +import { createHash } from 'node:crypto' + +import { + INTERIOR_BOUNDARY_BITS_STEP, + INTERIOR_MAX_CHILDREN, + LEAF_BOUNDARY_BITS, + LEAF_MAX_ENTRIES, +} from '../constants.js' + +/** The first 8 bytes of sha256(utf8(key)). */ +export function boundaryBytes(key: string): Uint8Array { + return createHash('sha256').update(key, 'utf8').digest().subarray(0, 8) +} + +/** Trailing zero bits of u(k), 0–64. */ +export function trailingZeros(u: Uint8Array): number { + let tz = 0 + for (let i = 7; i >= 0; i--) { + const b = u[i]! + if (b === 0) { + tz += 8 + continue + } + return tz + (31 - Math.clz32(b & -b)) + } + return tz +} + +export interface Chunking { + readonly name: string + /** + * Does a node at `level` end after this element? `u` is the boundary hash of the + * element's key (level 0) or of the child's last key (level ≥ 1); `size` is the + * number of elements in the node so far, including this one. + */ + ends(level: number, u: Uint8Array, size: number): boolean +} + +export interface FixedChunkingParams { + leafBits: number + stepBits: number + leafMax: number + interiorMax: number +} + +/** + * The fixed-probability rule with explicit parameters. The protocol uses exactly + * one parameter set (`protocolChunking`); others exist for tests, which need + * tiny nodes to exercise deep trees and forced splits, and for the experiments. + */ +export function fixedChunking(p: FixedChunkingParams, name = 'fixed'): Chunking { + return { + name, + ends(level, u, size) { + if (level === 0) return size >= p.leafMax || trailingZeros(u) >= p.leafBits + // Past 64 bits nothing is a natural boundary; only forced splits remain. + const bits = p.leafBits + p.stepBits * level + return size >= p.interiorMax || (bits <= 64 && trailingZeros(u) >= bits) + }, + } +} + +export const protocolChunking: Chunking = fixedChunking( + { + leafBits: LEAF_BOUNDARY_BITS, + stepBits: INTERIOR_BOUNDARY_BITS_STEP, + leafMax: LEAF_MAX_ENTRIES, + interiorMax: INTERIOR_MAX_CHILDREN, + }, + 'protocol', +) diff --git a/packages/core/src/tree/merge.ts b/packages/core/src/tree/merge.ts new file mode 100644 index 0000000..009f7bb --- /dev/null +++ b/packages/core/src/tree/merge.ts @@ -0,0 +1,189 @@ +/** + * Incremental merge: apply a sorted stream of changes to a base tree. + * + * Walk the base tree top-down. A base node with no change in its key range is + * reused whole when the builder is sitting on a boundary at its level (see + * TreeBuilder.addNode); otherwise it is expanded, and at a leaf the base entries + * are merged with the changes and fed to the builder entry by entry. A change + * belongs to the first node whose last key is ≥ the change's key; changes past + * the base's last key are appended at the end. + * + * Work is O(changes × fanout × height): each change expands one path, plus at + * most one neighbour per level to re-align with the base's boundaries. The result + * is identical to building the new entry set from scratch, which the property + * tests check. + */ +import { compareUtf8 } from '../utf8.js' +import { type BuilderOptions, TreeBuilder, type TreeSink } from './builder.js' +import type { NodeDesc } from './node.js' +import { type NodeSource, resolveRoot, rootDesc } from './source.js' + +/** Upsert (`entry`) or delete (`entry: null`) of one key. */ +export interface Change { + key: string + entry: E | null +} + +export interface MergeStats { + added: number + removed: number + updated: number + /** Upserts identical to the base entry. */ + unchanged: number + /** Deletes of keys the base doesn't have. */ + missingDeletes: number + /** Base nodes reused without being read. */ + reusedNodes: number + /** Base nodes read (expanded or rewritten). */ + readNodes: number +} + +export interface MergeOptions extends BuilderOptions { + /** Called for every effective change, in key order: (old, new). */ + onChange?: (before: E | null, after: E | null) => void +} + +export interface MergeResult { + root: NodeDesc | null + stats: MergeStats +} + +class Peekable { + readonly #it: Iterator | AsyncIterator + #head: IteratorResult | undefined + + constructor(src: Iterable | AsyncIterable) { + this.#it = + Symbol.asyncIterator in src + ? (src as AsyncIterable)[Symbol.asyncIterator]() + : (src as Iterable)[Symbol.iterator]() + } + + async peek(): Promise { + this.#head ??= await this.#it.next() + return this.#head.done ? undefined : this.#head.value + } + + async next(): Promise { + const v = await this.peek() + this.#head = undefined + return v + } +} + +export async function mergeTree( + source: NodeSource, + sink: TreeSink, + base: string | null, + changes: Iterable> | AsyncIterable>, + opts: MergeOptions = {}, +): Promise { + const spec = source.spec + const builder = new TreeBuilder(spec, sink, opts) + const pending = new Peekable(changes) + const stats: MergeStats = { + added: 0, + removed: 0, + updated: 0, + unchanged: 0, + missingDeletes: 0, + reusedNodes: 0, + readNodes: 0, + } + let prevChangeKey: string | null = null + + const nextChange = async (): Promise | undefined> => { + const c = await pending.next() + if (c) { + if (prevChangeKey !== null && compareUtf8(prevChangeKey, c.key) >= 0) { + throw new Error(`mergeTree: changes out of order at ${JSON.stringify(c.key)}`) + } + if (c.entry && spec.key(c.entry) !== c.key) { + throw new Error('mergeTree: change key does not match its entry') + } + prevChangeKey = c.key + } + return c + } + + /** Apply one change that has no base entry. */ + const insert = (c: Change) => { + if (c.entry) { + stats.added++ + opts.onChange?.(null, c.entry) + builder.addEntry(c.entry) + } else { + stats.missingDeletes++ + } + } + + const rewriteLeaf = async (desc: NodeDesc) => { + stats.readNodes++ + const entries = await source.leafEntries(desc.hash) + for (const old of entries) { + const oldKey = spec.key(old) + // Changes before this entry are inserts (or deletes of absent keys). + for (;;) { + const c = await pending.peek() + if (!c || compareUtf8(c.key, oldKey) >= 0) break + insert((await nextChange())!) + } + const c = await pending.peek() + if (c && c.key === oldKey) { + await nextChange() + if (!c.entry) { + stats.removed++ + opts.onChange?.(old, null) + } else if (spec.same(old, c.entry)) { + stats.unchanged++ + builder.addEntry(old) + } else { + stats.updated++ + opts.onChange?.(old, c.entry) + builder.addEntry(c.entry) + } + } else { + builder.addEntry(old) + } + } + // Changes between this leaf's last entry and its last key can't exist (the + // last entry is the last key), so nothing else belongs to this leaf. + } + + // `rightEdge`: the node is the last of its level in the base. It may have ended + // because the tree ended rather than at a boundary, so it can only be reused + // when nothing is appended after it, i.e. no change remains at all. + const visit = async (desc: NodeDesc, rightEdge: boolean): Promise => { + const c = await pending.peek() + const changed = c !== undefined && (rightEdge || compareUtf8(c.key, desc.lastKey) <= 0) + if (!changed && builder.emptyThrough(desc.level)) { + stats.reusedNodes++ + builder.addNode(desc) + return + } + if (desc.level === 0) { + await rewriteLeaf(desc) + return + } + stats.readNodes++ + const node = await source.node(desc.hash) + if (node.kind !== 'node' || node.level !== desc.level) { + throw new Error(`mergeTree: node ${desc.hash} is not at level ${desc.level}`) + } + const last = node.children.length - 1 + for (let i = 0; i <= last; i++) await visit(node.children[i]!, rightEdge && i === last) + } + + if (base !== null) { + if ((await pending.peek()) === undefined) { + // Nothing to apply: the base is the result. + return { root: await rootDesc(source, base), stats } + } + stats.readNodes++ + await visit(await rootDesc(source, base), true) + } + for (let c = await nextChange(); c; c = await nextChange()) insert(c) + + const { root, unresolved } = builder.finish() + return { root: root && unresolved ? await resolveRoot(source, root) : root, stats } +} diff --git a/packages/core/src/tree/node.ts b/packages/core/src/tree/node.ts new file mode 100644 index 0000000..a8f789c --- /dev/null +++ b/packages/core/src/tree/node.ts @@ -0,0 +1,257 @@ +/** + * Tree nodes: canonical encoding, hashing and decoding. + * + * leaf: {"e":[entry, ...],"t":"leaf"} + * interior: {"e":[[lastKey, childHash, count, bytes], ...],"l":level,"t":"node"} + * + * Both are JCS (keys sorted: "e" < "l" < "t"). A node's hash is the lowercase hex + * SHA-256 of those exact bytes. Leaf entries depend on the tree kind (TreeSpec). + */ +import { sha256Hex } from '../hash.js' +import { compareUtf8 } from '../utf8.js' + +const HEX64 = /^[0-9a-f]{64}$/ + +/** A child pointer as stored in an interior node, plus the child's level. */ +export interface NodeDesc { + level: number + hash: string + /** The last key under the node. Boundaries are decided on it. */ + lastKey: string + /** Entries under the node. */ + count: number + /** Sum of entry sizes under the node (TreeSpec.bytes). */ + bytes: number +} + +/** + * A tree kind: how its leaf entries are encoded. The encoding of an entry must be + * the JCS form of a JSON array, so that a node is JCS as a whole. + */ +export interface TreeSpec { + readonly name: string + key(e: E): string + /** JCS of the entry's tuple, e.g. `["id","",123]`. */ + encode(e: E): string + /** Inverse of encode, from the parsed tuple. Throws on a malformed tuple. */ + decode(tuple: unknown): E + /** Size contributed to `bytes`. */ + bytes(e: E): number + /** Same content (for change detection): same key and same value. */ + same(a: E, b: E): boolean +} + +export type DecodedNode = + | { kind: 'leaf'; hash: string; entries: E[] } + | { kind: 'node'; hash: string; level: number; children: NodeDesc[] } + +export function encodeLeaf(spec: TreeSpec, entries: readonly E[]): string { + let out = '{"e":[' + for (let i = 0; i < entries.length; i++) { + if (i > 0) out += ',' + out += spec.encode(entries[i]!) + } + return out + '],"t":"leaf"}' +} + +export function encodeInterior(level: number, children: readonly NodeDesc[]): string { + let out = '{"e":[' + for (let i = 0; i < children.length; i++) { + const c = children[i]! + if (i > 0) out += ',' + out += '[' + JSON.stringify(c.lastKey) + ',"' + c.hash + '",' + c.count + ',' + c.bytes + ']' + } + return out + '],"l":' + level + ',"t":"node"}' +} + +export function hashNode(json: string): string { + return sha256Hex(json) +} + +export class NodeFormatError extends Error { + constructor(message: string) { + super(message) + this.name = 'NodeFormatError' + } +} + +const isCount = (n: unknown): n is number => Number.isSafeInteger(n) && (n as number) >= 0 + +/** + * Decode a node from its stored JSON and check it: the bytes hash to `expectedHash` + * (when given), the shape is right for the tree kind, and the bytes are canonical + * (re-encoding gives the same string). A non-canonical encoding of the same content + * would have a different hash, so accepting one would let two hashes name one node. + * + * Child levels can't be checked here: an interior node's children are one level + * below it, which the decoder records; the reader checks it when it loads them. + */ +export function decodeNode( + spec: TreeSpec, + json: string, + expectedHash?: string, +): DecodedNode { + const hash = hashNode(json) + if (expectedHash !== undefined && hash !== expectedHash) { + throw new NodeFormatError(`Node hash mismatch: expected ${expectedHash}, got ${hash}`) + } + let parsed: unknown + try { + parsed = JSON.parse(json) + } catch { + throw new NodeFormatError('Node is not valid JSON') + } + const obj = parsed as { t?: unknown; e?: unknown; l?: unknown } + if (!obj || typeof obj !== 'object' || !Array.isArray(obj.e) || obj.e.length === 0) { + throw new NodeFormatError('Node must have a non-empty "e" array') + } + if (obj.t === 'leaf') { + if (Object.keys(obj).length !== 2) throw new NodeFormatError('Leaf has unexpected fields') + const entries = obj.e.map((t) => spec.decode(t)) + for (let i = 1; i < entries.length; i++) { + if (compareUtf8(spec.key(entries[i - 1]!), spec.key(entries[i]!)) >= 0) { + throw new NodeFormatError('Leaf keys are not strictly increasing') + } + } + if (encodeLeaf(spec, entries) !== json) throw new NodeFormatError('Leaf is not canonical') + return { kind: 'leaf', hash, entries } + } + if (obj.t === 'node') { + if (Object.keys(obj).length !== 3) throw new NodeFormatError('Node has unexpected fields') + const level = obj.l + if (!Number.isSafeInteger(level) || (level as number) < 1) { + throw new NodeFormatError('Interior node level must be an integer ≥ 1') + } + const children: NodeDesc[] = obj.e.map((t) => { + if (!Array.isArray(t) || t.length !== 4) throw new NodeFormatError('Bad interior entry') + const [lastKey, h, count, bytes] = t as unknown[] + if (typeof lastKey !== 'string' || typeof h !== 'string' || !HEX64.test(h)) { + throw new NodeFormatError('Bad interior entry') + } + if (!isCount(count) || count < 1 || !isCount(bytes)) { + throw new NodeFormatError('Bad interior entry counts') + } + return { level: (level as number) - 1, hash: h, lastKey, count, bytes } + }) + for (let i = 1; i < children.length; i++) { + if (compareUtf8(children[i - 1]!.lastKey, children[i]!.lastKey) >= 0) { + throw new NodeFormatError('Interior keys are not strictly increasing') + } + } + if (encodeInterior(level as number, children) !== json) { + throw new NodeFormatError('Interior node is not canonical') + } + return { kind: 'node', hash, level: level as number, children } + } + throw new NodeFormatError('Node "t" must be "leaf" or "node"') +} + +/** The descriptor of a decoded node, as its parent would hold it. */ +export function describeNode(spec: TreeSpec, node: DecodedNode): NodeDesc { + if (node.kind === 'leaf') { + let bytes = 0 + for (const e of node.entries) bytes += spec.bytes(e) + return { + level: 0, + hash: node.hash, + lastKey: spec.key(node.entries[node.entries.length - 1]!), + count: node.entries.length, + bytes, + } + } + let count = 0 + let bytes = 0 + for (const c of node.children) { + count += c.count + bytes += c.bytes + } + return { + level: node.level, + hash: node.hash, + lastKey: node.children[node.children.length - 1]!.lastKey, + count, + bytes, + } +} + +// --- Tree kinds --------------------------------------------------------------- + +/** + * Record trees: key = record id, value = record hash, plus the canonical record's + * size in bytes. `body` (the canonical record line) travels with an entry while + * it is being written or read; it is never part of the node. + */ +export interface RecordEntry { + key: string + hash: string + size: number + body?: string +} + +export const recordTree: TreeSpec = { + name: 'record', + key: (e) => e.key, + encode: (e) => '[' + JSON.stringify(e.key) + ',"' + e.hash + '",' + e.size + ']', + decode(t) { + if (!Array.isArray(t) || t.length !== 3) throw new NodeFormatError('Bad record entry') + const [key, hash, size] = t as unknown[] + if ( + typeof key !== 'string' || + typeof hash !== 'string' || + !HEX64.test(hash) || + !isCount(size) + ) { + throw new NodeFormatError('Bad record entry') + } + return { key, hash, size } + }, + bytes: (e) => e.size, + same: (a, b) => a.key === b.key && a.hash === b.hash, +} + +/** File trees: key = file hash (hex), value = file size in bytes. */ +export interface FileEntry { + key: string + size: number +} + +export const fileTree: TreeSpec = { + name: 'file', + key: (e) => e.key, + encode: (e) => '["' + e.key + '",' + e.size + ']', + decode(t) { + if (!Array.isArray(t) || t.length !== 2) throw new NodeFormatError('Bad file entry') + const [key, size] = t as unknown[] + if (typeof key !== 'string' || !HEX64.test(key) || !isCount(size)) { + throw new NodeFormatError('Bad file entry') + } + return { key, size } + }, + bytes: (e) => e.size, + same: (a, b) => a.key === b.key && a.size === b.size, +} + +/** + * Not protocol: a sidecar tree of reference counts (key = file hash, value = + * number of references from a set's records). Lets a commit keep file sets + * correct in O(changes). Same tree code, so the same reuse and caching. + */ +export interface CountEntry { + key: string + n: number +} + +export const countTree: TreeSpec = { + name: 'count', + key: (e) => e.key, + encode: (e) => '[' + JSON.stringify(e.key) + ',' + e.n + ']', + decode(t) { + if (!Array.isArray(t) || t.length !== 2) throw new NodeFormatError('Bad count entry') + const [key, n] = t as unknown[] + if (typeof key !== 'string' || !isCount(n) || n < 1) + throw new NodeFormatError('Bad count entry') + return { key, n } + }, + bytes: () => 0, + same: (a, b) => a.key === b.key && a.n === b.n, +} diff --git a/packages/core/src/tree/read.ts b/packages/core/src/tree/read.ts new file mode 100644 index 0000000..0b727fb --- /dev/null +++ b/packages/core/src/tree/read.ts @@ -0,0 +1,194 @@ +/** + * Reading trees: point lookups, ordered iteration from a key or an offset, and + * diff. Interior entries carry each child's last key and entry count, so a seek by + * key or by offset reads one node per level. + */ +import { compareUtf8 } from '../utf8.js' +import type { NodeDesc } from './node.js' +import { type NodeSource, rootDesc } from './source.js' + +/** Look up one key. One node read per level. */ +export async function getEntry( + source: NodeSource, + root: string | null, + key: string, +): Promise { + if (root === null) return null + let hash = root + for (;;) { + const node = await source.node(hash) + if (node.kind === 'leaf') { + return node.entries.find((e) => source.spec.key(e) === key) ?? null + } + const child = node.children.find((c) => compareUtf8(key, c.lastKey) <= 0) + if (!child) return null + hash = child.hash + } +} + +export interface IterateOptions { + /** Start after this key (keyset pagination). Exclusive with `offset`. */ + after?: string + /** Skip this many entries first (offset pagination; O(height) via counts). Exclusive with `after`. */ + offset?: number + /** Leaves to fetch ahead in parallel. Default 4. */ + prefetch?: number + /** Read leaves with payloads (record bodies) through `leafEntries`. */ + payloads?: boolean +} + +/** + * The leaves covering a seek, in order: yields each leaf's descriptor and how + * many of its entries to skip. Interior nodes are read on demand, one path at a + * time. + */ +async function* leavesFrom( + source: NodeSource, + root: NodeDesc, + after: string | undefined, + offset: number, +): AsyncGenerator<{ leaf: NodeDesc; skip: number; after: string | undefined }> { + // Stack of (children, next index) frames, rooted at the root. + type Frame = { children: NodeDesc[]; i: number } + const stack: Frame[] = [{ children: [root], i: 0 }] + let first = true + let remaining = offset + while (stack.length > 0) { + const top = stack[stack.length - 1]! + if (top.i >= top.children.length) { + stack.pop() + continue + } + const d = top.children[top.i++]! + if (first) { + // Still seeking: skip whole subtrees before the start position. + if (after !== undefined && compareUtf8(d.lastKey, after) <= 0) continue + if (remaining >= d.count) { + remaining -= d.count + continue + } + } + if (d.level === 0) { + yield { leaf: d, skip: first ? remaining : 0, after: first ? after : undefined } + first = false + continue + } + const node = await source.node(d.hash) + if (node.kind !== 'node') throw new Error(`Expected interior node at ${d.hash}`) + stack.push({ children: node.children, i: 0 }) + } +} + +/** Iterate entries in key order from a position, prefetching leaves ahead. */ +export async function* iterate( + source: NodeSource, + root: string | null, + opts: IterateOptions = {}, +): AsyncGenerator { + if (root === null) return + if (opts.after !== undefined && opts.offset) + throw new Error('iterate: pass after or offset, not both') + const spec = source.spec + const load = (h: string) => + opts.payloads + ? source.leafEntries(h) + : source.node(h).then((n) => (n.kind === 'leaf' ? n.entries : [])) + const ahead = Math.max(1, opts.prefetch ?? 4) + const queue: { entries: Promise; skip: number; after: string | undefined }[] = [] + const leaves = leavesFrom(source, await rootDesc(source, root), opts.after, opts.offset ?? 0) + let exhausted = false + const fill = async () => { + while (!exhausted && queue.length < ahead) { + const r = await leaves.next() + if (r.done) { + exhausted = true + break + } + const entries = load(r.value.leaf.hash) + entries.catch(() => {}) // surfaced when awaited below + queue.push({ entries, skip: r.value.skip, after: r.value.after }) + } + } + for (;;) { + await fill() + const next = queue.shift() + if (!next) return + const entries = await next.entries + let i = 0 + if (next.after !== undefined) { + while (i < entries.length && compareUtf8(spec.key(entries[i]!), next.after) <= 0) i++ + } + i += next.skip + for (; i < entries.length; i++) yield entries[i]! + } +} + +export interface DiffEntry { + key: string + /** Entry in the first tree, or null when added. */ + before: E | null + /** Entry in the second tree, or null when removed. */ + after: E | null +} + +type Item = { node: NodeDesc } | { entry: E } + +/** + * Entries that differ between two trees, in key order. Subtrees with equal hashes + * hold identical entries and are skipped without being read; because node + * boundaries depend only on keys, unchanged regions line up and the walk costs + * O(changes × height) node reads. + */ +export async function* diffTrees( + source: NodeSource, + a: string | null, + b: string | null, +): AsyncGenerator> { + const spec = source.spec + const xs: Item[] = a === null ? [] : [{ node: await rootDesc(source, a) }] + const ys: Item[] = b === null ? [] : [{ node: await rootDesc(source, b) }] + const expand = async (items: Item[]) => { + const head = items.shift() as { node: NodeDesc } + const n = await source.node(head.node.hash) + const children: Item[] = + n.kind === 'leaf' + ? n.entries.map((entry) => ({ entry })) + : n.children.map((node) => ({ node })) + items.unshift(...children) + } + for (;;) { + const x = xs[0] + const y = ys[0] + if (!x && !y) return + if (x && 'node' in x && y && 'node' in y && x.node.hash === y.node.hash) { + xs.shift() + ys.shift() + continue + } + if (x && 'node' in x && (!y || 'entry' in y || x.node.level >= y.node.level)) { + await expand(xs) + continue + } + if (y && 'node' in y) { + await expand(ys) + continue + } + // Both heads are entries (or one side is exhausted). + const ex = x && 'entry' in x ? x.entry : undefined + const ey = y && 'entry' in y ? y.entry : undefined + if (ex !== undefined && (ey === undefined || compareUtf8(spec.key(ex), spec.key(ey)) < 0)) { + xs.shift() + yield { key: spec.key(ex), before: ex, after: null } + } else if ( + ey !== undefined && + (ex === undefined || compareUtf8(spec.key(ey), spec.key(ex)) < 0) + ) { + ys.shift() + yield { key: spec.key(ey), before: null, after: ey } + } else { + xs.shift() + ys.shift() + if (!spec.same(ex!, ey!)) yield { key: spec.key(ex!), before: ex!, after: ey! } + } + } +} diff --git a/packages/core/src/tree/source.ts b/packages/core/src/tree/source.ts new file mode 100644 index 0000000..8c66b6d --- /dev/null +++ b/packages/core/src/tree/source.ts @@ -0,0 +1,58 @@ +/** + * Where tree readers get nodes from. The server backs this with the blob store + * plus caches; tests use a Map. + */ +import { type DecodedNode, decodeNode, describeNode, type NodeDesc, type TreeSpec } from './node.js' + +export interface NodeSource { + readonly spec: TreeSpec + /** Load and verify a node. */ + node(hash: string): Promise> + /** + * A leaf's entries with their payloads (record bodies), for rewriting a leaf. + * Sources whose entries carry no payload return the node's entries. + */ + leafEntries(hash: string): Promise +} + +/** A source over node JSON held in memory, keyed by hash. */ +export class MapSource implements NodeSource { + constructor( + readonly spec: TreeSpec, + readonly nodes: Map, + readonly payloads?: Map, + ) {} + + async node(hash: string): Promise> { + const json = this.nodes.get(hash) + if (json === undefined) throw new Error(`Node ${hash} not found`) + return decodeNode(this.spec, json, hash) + } + + async leafEntries(hash: string): Promise { + const withPayload = this.payloads?.get(hash) + if (withPayload) return withPayload.slice() + const n = await this.node(hash) + if (n.kind !== 'leaf') throw new Error(`Node ${hash} is not a leaf`) + return n.entries + } +} + +/** The full descriptor of a tree's root (level and last key come from the node). */ +export async function rootDesc(source: NodeSource, hash: string): Promise { + return describeNode(source.spec, await source.node(hash)) +} + +/** + * Finish `TreeBuilder.finish` when it reports `unresolved`: walk down while the + * node has a single child. See the builder for why. + */ +export async function resolveRoot(source: NodeSource, desc: NodeDesc): Promise { + let d = desc + while (d.level > 0) { + const n = await source.node(d.hash) + if (n.kind !== 'node' || n.children.length !== 1) break + d = n.children[0]! + } + return d +} diff --git a/packages/core/src/tree/verify.ts b/packages/core/src/tree/verify.ts new file mode 100644 index 0000000..ea8004f --- /dev/null +++ b/packages/core/src/tree/verify.ts @@ -0,0 +1,89 @@ +/** + * fsck for trees: validate structure, not just hashes. + * + * A node can hash correctly and still break the rules: keys out of order, a + * boundary in the wrong place, wrong counts, wrong levels. Such a tree gives a + * valid-looking but different root for the same entries, which breaks + * convergence. `verifyTree` checks every node against its parent's pointer and + * then rebuilds the tree from its entries; the rebuilt root must be the same + * hash, which covers every chunking rule at once. + */ +import { compareUtf8 } from '../utf8.js' +import { TreeBuilder } from './builder.js' +import { type Chunking } from './chunking.js' +import { describeNode, type NodeDesc } from './node.js' +import type { NodeSource } from './source.js' + +export interface VerifyResult { + ok: boolean + errors: string[] + count: number + bytes: number + nodes: number +} + +export async function verifyTree( + source: NodeSource, + root: string | null, + opts: { chunking?: Chunking; maxErrors?: number } = {}, +): Promise { + const result: VerifyResult = { ok: true, errors: [], count: 0, bytes: 0, nodes: 0 } + if (root === null) return result + const maxErrors = opts.maxErrors ?? 20 + const fail = (msg: string) => { + result.ok = false + if (result.errors.length < maxErrors) result.errors.push(msg) + } + const spec = source.spec + const rebuilt = new TreeBuilder( + spec, + { leaf() {}, interior() {} }, + opts.chunking ? { chunking: opts.chunking } : {}, + ) + let prev: string | null = null + + const walk = async (hash: string, expect: NodeDesc | null): Promise => { + let node + try { + node = await source.node(hash) + } catch (err) { + fail(`${hash}: ${(err as Error).message}`) + return + } + result.nodes++ + const actual = describeNode(spec, node) + if (expect) { + if (actual.level !== expect.level) + fail(`${hash}: level ${actual.level}, parent says ${expect.level}`) + if (actual.lastKey !== expect.lastKey) fail(`${hash}: last key differs from parent pointer`) + if (actual.count !== expect.count) + fail(`${hash}: count ${actual.count}, parent says ${expect.count}`) + if (actual.bytes !== expect.bytes) + fail(`${hash}: bytes ${actual.bytes}, parent says ${expect.bytes}`) + } + if (node.kind === 'leaf') { + for (const e of node.entries) { + const k = spec.key(e) + if (prev !== null && compareUtf8(prev, k) >= 0) { + fail(`${hash}: key ${JSON.stringify(k)} out of order`) + continue + } + prev = k + result.count++ + result.bytes += spec.bytes(e) + rebuilt.addEntry(e) + } + return + } + for (const child of node.children) await walk(child.hash, child) + } + + await walk(root, null) + if (result.ok) { + const { root: again } = rebuilt.finish() + if (again?.hash !== root) { + fail(`Tree is not canonical: rebuilding its entries gives root ${again?.hash ?? 'null'}`) + } + } + return result +} diff --git a/packages/core/src/utf8.ts b/packages/core/src/utf8.ts new file mode 100644 index 0000000..3dd54b2 --- /dev/null +++ b/packages/core/src/utf8.ts @@ -0,0 +1,50 @@ +const encoder = new TextEncoder() + +export function utf8(s: string): Uint8Array { + return encoder.encode(s) +} + +/** UTF-8 byte length of a well-formed string, without encoding it. */ +export function utf8ByteLength(s: string): number { + let bytes = 0 + for (let i = 0; i < s.length; i++) { + const u = s.charCodeAt(i) + if (u < 0x80) bytes += 1 + else if (u < 0x800) bytes += 2 + else if (u >= 0xd800 && u <= 0xdbff && i + 1 < s.length) { + // A surrogate pair is one 4-byte code point. + bytes += 4 + i++ + } else bytes += 3 + } + return bytes +} + +/** + * Compare two strings by their UTF-8 bytes, which is Unicode code point order. + * + * JavaScript's `<` and default `sort()` compare UTF-16 code units, and those + * disagree with code point order exactly when one side has a surrogate + * (U+10000 and up, stored as 0xD800–0xDFFF) and the other a code unit in + * 0xE000–0xFFFF. At the first differing unit, remap so surrogates sort above the + * rest of the BMP; everything else compares as is. + */ +export function compareUtf8(a: string, b: string): number { + if (a === b) return 0 + const n = Math.min(a.length, b.length) + for (let i = 0; i < n; i++) { + const x = a.charCodeAt(i) + const y = b.charCodeAt(i) + if (x !== y) { + if (x >= 0xd800 && y >= 0xd800) return fixup(x) - fixup(y) + return x - y + } + } + return a.length - b.length +} + +function fixup(u: number): number { + if (u >= 0xe000) return u - 0x800 + if (u >= 0xd800) return u + 0x2000 + return u +} diff --git a/packages/core/test/input-rules.test.ts b/packages/core/test/input-rules.test.ts new file mode 100644 index 0000000..4f871ed --- /dev/null +++ b/packages/core/test/input-rules.test.ts @@ -0,0 +1,141 @@ +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' + +import { + compareUtf8, + InputRuleError, + parseRecordLine, + parseStrict, + scanJson, +} from '../src/index.js' + +const code = (f: () => unknown): string | null => { + try { + f() + return null + } catch (err) { + return err instanceof InputRuleError ? err.code : `other: ${(err as Error).message}` + } +} + +describe('scanJson', () => { + it('accepts ordinary JSON', () => { + expect( + code(() => scanJson('{"a":[1,2.5,-3,"x",true,false,null,{"b":{}}],"c":"\\"q\\""}')), + ).toBe(null) + }) + + it('rejects duplicate keys, compared after unescaping', () => { + expect(code(() => scanJson('{"a":1,"a":2}'))).toBe('duplicate_key') + expect(code(() => scanJson('{"a":1,"\\u0061":2}'))).toBe('duplicate_key') + expect(code(() => scanJson('{"x":{"a":1},"y":{"a":1}}'))).toBe(null) + expect(code(() => scanJson('[{"a":1},{"a":2}]'))).toBe(null) + expect(code(() => scanJson('{"a":{"a":{"a":1}}}'))).toBe(null) + // A key inside a nested value doesn't count toward the outer object. + expect(code(() => scanJson('{"a":{"b":1},"b":2}'))).toBe(null) + }) + + it('rejects integer literals outside ±(2^53−1) and keeps floats', () => { + expect(code(() => scanJson('[9007199254740991,-9007199254740991]'))).toBe(null) + expect(code(() => scanJson('[9007199254740992]'))).toBe('unsafe_integer') + expect(code(() => scanJson('{"n":-9007199254740992}'))).toBe('unsafe_integer') + expect(code(() => scanJson('[123456789012345678901234567890]'))).toBe('unsafe_integer') + expect(code(() => scanJson('[1e20,6.02e23,1.5,-0.0,1E+400]'))).toBe(null) + expect(code(() => scanJson('[9007199254740993.0]'))).toBe(null) + }) + + it('rejects lone surrogates, raw or escaped', () => { + expect(code(() => scanJson('["\\ud83d\\ude00", "😀"]'))).toBe(null) + expect(code(() => scanJson('["\\ud83d"]'))).toBe('lone_surrogate') + expect(code(() => scanJson('["\\ude00"]'))).toBe('lone_surrogate') + expect(code(() => scanJson('["\\ud83dx"]'))).toBe('lone_surrogate') + expect(code(() => scanJson('["\ud83d"]'))).toBe('lone_surrogate') + expect(code(() => scanJson('{"\\ud83d":1}'))).toBe('lone_surrogate') + // A raw high surrogate followed by an escaped low one decodes to a valid pair. + expect(code(() => scanJson('["\ud83d\\ude00"]'))).toBe(null) + }) + + it('limits nesting depth', () => { + expect(code(() => scanJson('[[[]]]', 3))).toBe(null) + expect(code(() => scanJson('[[[[]]]]', 3))).toBe('too_deep') + expect(code(() => scanJson('{"a":{"b":{"c":{}}}}', 3))).toBe('too_deep') + expect(code(() => scanJson('['.repeat(100_000)))).toBe('too_deep') + }) + + it('leaves syntax to JSON.parse', () => { + expect(code(() => parseStrict('{"a":'))).toBe('syntax') + expect(code(() => parseStrict('{"a":1}'))).toBe(null) + }) + + it('never throws anything but InputRuleError, on any input', () => { + fc.assert( + fc.property(fc.string({ unit: 'binary' }), (s) => { + const c = code(() => parseStrict(s)) + return c === null || !c.startsWith('other') + }), + { numRuns: 2000 }, + ) + }) + + it('accepts every JSON value without unsafe integers, duplicates or lone surrogates', () => { + const value = fc.jsonValue({ maxDepth: 6 }).filter((v) => { + const text = JSON.stringify(v) + return !/\d{16,}/.test(text) && (text ?? '').isWellFormed() + }) + fc.assert( + fc.property(value, (v) => code(() => scanJson(JSON.stringify(v))) === null), + { numRuns: 1000 }, + ) + }) +}) + +describe('parseRecordLine', () => { + it('parses and canonicalizes a record', () => { + const r = parseRecordLine( + '{"type":"Author","id":"r1","data":{"year":1815,"name":"Ada"},"private":true}', + ) + expect(r).toMatchObject({ id: 'r1', type: 'Author', private: true }) + expect(r.canonical).toBe('{"id":"r1","type":"Author","data":{"name":"Ada","year":1815}}') + }) + + it('rejects bad envelopes', () => { + expect(code(() => parseRecordLine('[1]'))).toBe('bad_envelope') + expect(code(() => parseRecordLine('{"id":"r","type":"t"}'))).toBe('bad_envelope') + expect(code(() => parseRecordLine('{"id":"","type":"t","data":1}'))).toBe('bad_id') + expect(code(() => parseRecordLine('{"id":1,"type":"t","data":1}'))).toBe('bad_id') + expect(code(() => parseRecordLine(`{"id":"${'x'.repeat(1025)}","type":"t","data":1}`))).toBe( + 'bad_id', + ) + expect(code(() => parseRecordLine('{"id":"r","type":"a/b","data":1}'))).toBe('bad_type') + expect(code(() => parseRecordLine('{"id":"r","type":".x","data":1}'))).toBe('bad_type') + expect(code(() => parseRecordLine('{"id":"r","type":"t","data":1,"private":"yes"}'))).toBe( + 'bad_envelope', + ) + }) + + it('counts the envelope as one level of nesting', () => { + const deep = (n: number) => '['.repeat(n) + ']'.repeat(n) + expect(code(() => parseRecordLine(`{"id":"r","type":"t","data":${deep(64)}}`))).toBe(null) + expect(code(() => parseRecordLine(`{"id":"r","type":"t","data":${deep(65)}}`))).toBe('too_deep') + }) +}) + +describe('compareUtf8', () => { + it('orders like UTF-8 bytes', () => { + const str = fc.string({ unit: 'grapheme', maxLength: 6 }) + fc.assert( + fc.property(str, str, (a, b) => { + const want = Math.sign(Buffer.compare(Buffer.from(a), Buffer.from(b))) + return Math.sign(compareUtf8(a, b)) === want + }), + { numRuns: 5000 }, + ) + }) + + it('puts astral characters after U+E000–U+FFFF, unlike JS sort', () => { + const bmp = String.fromCharCode(0xffff) + const astral = String.fromCodePoint(0x1f600) + expect(compareUtf8(bmp, astral)).toBeLessThan(0) + expect(bmp < astral).toBe(false) + }) +}) diff --git a/packages/core/test/jcs.test.ts b/packages/core/test/jcs.test.ts new file mode 100644 index 0000000..b9dd98d --- /dev/null +++ b/packages/core/test/jcs.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from 'vitest' + +import { + hashRecord, + hashSchema, + hasArrayIndexKey, + jcs, + legacyRecordHash, + legacySchemaHash, +} from '../src/index.js' + +describe('jcs', () => { + it('matches RFC 8785 §3.2.2 (primitives, numbers, escaping)', () => { + const input = JSON.parse( + '{"numbers":[333333333.33333329,1E30,4.50,2e-3,0.000000000000000000000000001],' + + '"string":"\\u20ac$\\u000F\\u000aA\'\\u0042\\u0022\\u005c\\\\\\"\\/",' + + '"literals":[null,true,false]}', + ) + expect(jcs(input)).toBe( + '{"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27],' + + '"string":"€$\\u000f\\nA\'B\\"\\\\\\\\\\"/"}', + ) + }) + + it('matches RFC 8785 §3.2.3 (key sorting by UTF-16 code units)', () => { + const input = JSON.parse( + '{"\\u20ac":"Euro Sign","\\r":"Carriage Return","\\ufb33":"Hebrew Letter Dalet With Dagesh",' + + '"1":"One","\\ud83d\\ude00":"Emoji: Grinning Face","\\u0080":"Control",' + + '"\\u00f6":"Latin Small Letter O With Diaeresis"}', + ) + expect(jcs(input)).toBe( + '{"\\r":"Carriage Return","1":"One","\u0080":"Control","ö":"Latin Small Letter O With Diaeresis",' + + '"€":"Euro Sign","😀":"Emoji: Grinning Face","דּ":"Hebrew Letter Dalet With Dagesh"}', + ) + }) + + it('matches RFC 8785 Appendix B number serialization', () => { + const f = (hex: string) => + new DataView(new Uint8Array(Buffer.from(hex, 'hex')).buffer).getFloat64(0) + const cases: [string, string][] = [ + ['0000000000000000', '0'], + ['8000000000000000', '0'], + ['0000000000000001', '5e-324'], + ['8000000000000001', '-5e-324'], + ['7fefffffffffffff', '1.7976931348623157e+308'], + ['4340000000000000', '9007199254740992'], + ['c340000000000000', '-9007199254740992'], + ['4430000000000000', '295147905179352830000'], + ['44b52d02c7e14af5', '9.999999999999997e+22'], + ['44b52d02c7e14af6', '1e+23'], + ['3eb0c6f7a0b5ed8d', '0.000001'], + ['3eb0c6f7a0b5ed8c', '9.999999999999997e-7'], + ] + for (const [hex, want] of cases) expect(jcs(f(hex))).toBe(want) + }) + + it('sorts integer-like keys as strings, unlike JS enumeration', () => { + expect(jcs({ a: 1, 10: 2, 9: 3 })).toBe('{"10":2,"9":3,"a":1}') + expect(jcs({ x: { 2: 'b', 10: 'a' } })).toBe('{"x":{"10":"a","2":"b"}}') + }) + + it('refuses values JSON cannot hold', () => { + expect(() => jcs(Number.NaN)).toThrow() + expect(() => jcs(Infinity)).toThrow() + expect(() => jcs(undefined)).toThrow() + expect(() => jcs(10n)).toThrow() + }) +}) + +describe('record and schema hashes', () => { + it('keeps v1 hashes for records without integer-like keys', () => { + // Golden value from v1's hash.test.ts. + const { hash, canonical } = hashRecord('r1', 'Author', { name: 'Ada', year: 1815 }) + expect(canonical).toBe('{"id":"r1","type":"Author","data":{"name":"Ada","year":1815}}') + expect(hash).toBe('adefbd10aa438f0c6ed1627817f391ac6cc0441737ee09b4ebcc30fbd8386c63') + expect(legacyRecordHash('r1', 'Author', { name: 'Ada', year: 1815 })).toBe(hash) + }) + + it('keeps v1 schema hashes without integer-like keys', () => { + const schema = { type: 'object', properties: { name: { type: 'string' } } } + expect(hashSchema(schema)).toBe( + '2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2', + ) + expect(legacySchemaHash(schema)).toBe(hashSchema(schema)) + }) + + it('re-hashes data with integer-like keys, and only that data', () => { + const data = { 9: 'x', 10: 'y' } + expect(hasArrayIndexKey(data)).toBe(true) + expect(hashRecord('r', 't', data).hash).not.toBe(legacyRecordHash('r', 't', data)) + expect(hasArrayIndexKey({ a: [{ b: 1 }], c: '9' })).toBe(false) + expect(hasArrayIndexKey({ a: [{ '01': 1 }] })).toBe(false) // "01" is not an array index + expect(hasArrayIndexKey({ a: [{ 4294967295: 1 }] })).toBe(false) // 2^32−1 is not either + expect(hasArrayIndexKey({ a: [{ 4294967294: 1 }] })).toBe(true) + }) +}) diff --git a/packages/core/test/tree.test.ts b/packages/core/test/tree.test.ts new file mode 100644 index 0000000..c50fd8c --- /dev/null +++ b/packages/core/test/tree.test.ts @@ -0,0 +1,308 @@ +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' + +import { + buildTree, + type Change, + compareUtf8, + diffTrees, + getEntry, + iterate, + MapSource, + MemorySink, + mergeTree, + protocolChunking, + type RecordEntry, + recordTree, + sha256Hex, + verifyTree, +} from '../src/index.js' +import { fixedChunking as fixed } from '../src/tree/chunking.js' + +// Tiny nodes so a few hundred entries make deep trees with forced splits: +// mean leaf 4 entries (forced at 8), mean fanout 2 (forced at 4). +const RUNS = Number(process.env.PROPERTY_RUNS ?? 1) +const tiny = fixed({ leafBits: 2, stepBits: 1, leafMax: 8, interiorMax: 4 }, 'tiny') + +const entry = (key: string, v = 0): RecordEntry => ({ + key, + hash: sha256Hex(`${key}#${v}`), + size: key.length + v, +}) + +const sorted = (es: RecordEntry[]) => es.slice().sort((a, b) => compareUtf8(a.key, b.key)) + +/** A store shared across builds, the way R2 is shared across versions. */ +function store() { + const sink = new MemorySink() + const source = new MapSource(recordTree, sink.nodes, sink.leaves) + return { sink, source } +} + +async function collect(it: AsyncIterable): Promise { + const out: T[] = [] + for await (const x of it) out.push(x) + return out +} + +const keyArb = fc.string({ unit: 'grapheme-ascii', minLength: 1, maxLength: 6 }) +const keySet = (max: number) => fc.uniqueArray(keyArb, { maxLength: max }) + +describe('build', () => { + it('gives an empty tree a null root and one leaf as its own root', () => { + const { sink } = store() + expect(buildTree(recordTree, sink, [])).toBe(null) + const root = buildTree(recordTree, sink, [entry('a'), entry('b')])! + expect(root.level).toBe(0) + expect(root.count).toBe(2) + }) + + it('builds deep trees that verify, with counts and bytes summed', async () => { + const { sink, source } = store() + const es = sorted(Array.from({ length: 2000 }, (_, i) => entry(`k${i}`, i % 7))) + const root = buildTree(recordTree, sink, es, { chunking: tiny })! + expect(root.level).toBeGreaterThan(3) + expect(root.count).toBe(2000) + expect(root.bytes).toBe(es.reduce((s, e) => s + e.size, 0)) + const v = await verifyTree(source, root.hash, { chunking: tiny }) + expect(v.errors).toEqual([]) + expect(v.count).toBe(2000) + }) + + it('refuses keys out of order', () => { + const { sink } = store() + expect(() => buildTree(recordTree, sink, [entry('b'), entry('a')])).toThrow(/out of order/) + expect(() => buildTree(recordTree, sink, [entry('a'), entry('a')])).toThrow(/out of order/) + }) + + it('verify rejects a tree built under different chunking', async () => { + const { sink, source } = store() + const es = sorted(Array.from({ length: 300 }, (_, i) => entry(`k${i}`))) + const root = buildTree(recordTree, sink, es, { chunking: tiny })! + const v = await verifyTree(source, root.hash, { chunking: protocolChunking }) + expect(v.ok).toBe(false) + expect(v.errors.join()).toMatch(/not canonical/) + }) +}) + +describe('merge', () => { + /** Apply changes to a key→entry map: the expected result of a merge. */ + const apply = (base: Map, changes: Change[]) => { + const m = new Map(base) + for (const c of changes) { + if (c.entry) m.set(c.key, c.entry) + else m.delete(c.key) + } + return sorted([...m.values()]) + } + + const changeArb = (pool: string[]) => + fc.uniqueArray( + fc.record({ + key: pool.length > 0 ? fc.oneof(fc.constantFrom(...pool), keyArb) : keyArb, + del: fc.boolean(), + v: fc.integer({ min: 0, max: 3 }), + }), + { selector: (c) => c.key, maxLength: 80 }, + ) + + it('an incremental merge equals a full rebuild (property)', async () => { + await fc.assert( + fc.asyncProperty( + keySet(400).chain((keys) => fc.tuple(fc.constant(keys), changeArb(keys))), + async ([keys, raw]) => { + const { sink, source } = store() + const base = sorted(keys.map((k) => entry(k))) + const baseRoot = buildTree(recordTree, sink, base, { chunking: tiny }) + const changes = raw + .map((c) => ({ key: c.key, entry: c.del ? null : entry(c.key, c.v) })) + .sort((a, b) => compareUtf8(a.key, b.key)) + const merged = await mergeTree(source, sink, baseRoot?.hash ?? null, changes, { + chunking: tiny, + }) + const want = buildTree( + recordTree, + new MemorySink(), + apply(new Map(base.map((e) => [e.key, e])), changes), + { + chunking: tiny, + }, + ) + expect(merged.root?.hash ?? null).toBe(want?.hash ?? null) + expect(merged.root?.count ?? 0).toBe(want?.count ?? 0) + if (merged.root) { + const v = await verifyTree(source, merged.root.hash, { chunking: tiny }) + expect(v.errors).toEqual([]) + } + }, + ), + { numRuns: 300 * RUNS }, + ) + }) + + it('random insertion orders converge on one root (property)', async () => { + await fc.assert( + fc.asyncProperty( + keySet(300), + fc.integer({ min: 1, max: 6 }), + fc.integer(), + async (keys, parts, seed) => { + const { sink, source } = store() + // Split the keys into `parts` batches pseudo-randomly and merge them in one by one. + const batches: string[][] = Array.from({ length: parts }, () => []) + keys.forEach((k, i) => batches[Math.abs((seed ^ (i * 2654435761)) % parts)]!.push(k)) + let root: string | null = null + for (const b of batches) { + const changes = b + .map((k) => ({ key: k, entry: entry(k) })) + .sort((x, y) => compareUtf8(x.key, y.key)) + root = + (await mergeTree(source, sink, root, changes, { chunking: tiny })).root?.hash ?? null + } + const want = buildTree(recordTree, new MemorySink(), sorted(keys.map((k) => entry(k))), { + chunking: tiny, + }) + expect(root).toBe(want?.hash ?? null) + }, + ), + { numRuns: 200 * RUNS }, + ) + }) + + it('deleting everything gives null; a no-op merge returns the base', async () => { + const { sink, source } = store() + const es = sorted(Array.from({ length: 200 }, (_, i) => entry(`k${i}`))) + const root = buildTree(recordTree, sink, es, { chunking: tiny })! + const none = await mergeTree(source, sink, root.hash, [], { chunking: tiny }) + expect(none.root?.hash).toBe(root.hash) + const same = await mergeTree( + source, + sink, + root.hash, + es.slice(0, 50).map((e) => ({ key: e.key, entry: e })), + { + chunking: tiny, + }, + ) + expect(same.root?.hash).toBe(root.hash) + expect(same.stats.unchanged).toBe(50) + const all = await mergeTree( + source, + sink, + root.hash, + es.map((e) => ({ key: e.key, entry: null })), + { + chunking: tiny, + }, + ) + expect(all.root).toBe(null) + expect(all.stats.removed).toBe(200) + }) + + it('collapses to a reused single-child subtree correctly', async () => { + // Keep only the keys under one base subtree; the new root must be found by + // walking down single-child chains inside reused nodes. + for (let n = 50; n < 400; n += 37) { + const { sink, source } = store() + const es = sorted(Array.from({ length: n }, (_, i) => entry(`k${i}`))) + const root = buildTree(recordTree, sink, es, { chunking: tiny })! + for (const keep of [1, 3, 10, n >> 1]) { + const drop = es.slice(keep).map((e) => ({ key: e.key, entry: null })) + const merged = await mergeTree(source, sink, root.hash, drop, { chunking: tiny }) + const want = buildTree(recordTree, new MemorySink(), es.slice(0, keep), { chunking: tiny }) + expect(merged.root?.hash).toBe(want?.hash) + } + } + }) + + it('costs O(changes): one update in a large tree reads one path', async () => { + const { sink, source } = store() + const es = sorted( + Array.from({ length: 200_000 }, (_, i) => entry(`id${String(i).padStart(7, '0')}`)), + ) + const root = buildTree(recordTree, sink, es, { chunking: protocolChunking })! + expect(root.level).toBeGreaterThanOrEqual(1) + const target = es[123_456]! + let written = 0 + const counting = { + leaf: (...a: Parameters) => { + written++ + sink.leaf(...a) + }, + interior: (...a: Parameters) => { + written++ + sink.interior(...a) + }, + } + const merged = await mergeTree(source, counting, root.hash, [ + { key: target.key, entry: entry(target.key, 9) }, + ]) + expect(merged.stats.updated).toBe(1) + // Root, maybe one interior level, one leaf (plus at most a neighbour to re-align). + expect(merged.stats.readNodes).toBeLessThanOrEqual(2 * (root.level + 1) + 1) + expect(written).toBeLessThanOrEqual(2 * (root.level + 1)) + expect(merged.stats.reusedNodes).toBeGreaterThan(0) + }) + + it('reports changes in key order through onChange', async () => { + const { sink, source } = store() + const root = buildTree(recordTree, sink, sorted(['a', 'b', 'c'].map((k) => entry(k))), { + chunking: tiny, + })! + const seen: string[] = [] + await mergeTree( + source, + sink, + root.hash, + [ + { key: 'a', entry: null }, + { key: 'b', entry: entry('b', 1) }, + { key: 'c', entry: entry('c') }, + { key: 'd', entry: entry('d') }, + { key: 'e', entry: null }, + ], + { + chunking: tiny, + onChange: (before, after) => seen.push(`${before?.key ?? '-'}>${after?.key ?? '-'}`), + }, + ) + expect(seen).toEqual(['a>-', 'b>b', '->d']) + }) +}) + +describe('read', () => { + it('looks up, iterates from a key or an offset, and diffs (property)', async () => { + await fc.assert( + fc.asyncProperty(keySet(300), fc.nat(), fc.nat(), async (keys, at, edits) => { + const { sink, source } = store() + const es = sorted(keys.map((k) => entry(k))) + const root = buildTree(recordTree, sink, es, { chunking: tiny })?.hash ?? null + // Point lookups. + for (const e of es.slice(0, 20)) + expect((await getEntry(source, root, e.key))?.hash).toBe(e.hash) + expect(await getEntry(source, root, '\u{10FFFF}missing')).toBe(null) + // Offset and keyset pagination agree with the sorted array. + const offset = es.length === 0 ? 0 : at % (es.length + 1) + const fromOffset = await collect(iterate(source, root, { offset, prefetch: 3 })) + expect(fromOffset.map((e) => e.key)).toEqual(es.slice(offset).map((e) => e.key)) + if (offset > 0) { + const after = es[offset - 1]!.key + const fromKey = await collect(iterate(source, root, { after })) + expect(fromKey.map((e) => e.key)).toEqual(es.slice(offset).map((e) => e.key)) + } + // Diff against an edited copy reports exactly the edits. + const changes: Change[] = [] + es.forEach((e, i) => { + if ((i * 7 + edits) % 11 === 0) changes.push({ key: e.key, entry: null }) + else if ((i * 5 + edits) % 13 === 0) changes.push({ key: e.key, entry: entry(e.key, 1) }) + }) + const merged = await mergeTree(source, sink, root, changes, { chunking: tiny }) + const diff = await collect(diffTrees(source, root, merged.root?.hash ?? null)) + expect(diff.map((d) => `${d.key}:${d.before ? 1 : 0}${d.after ? 1 : 0}`)).toEqual( + changes.map((c) => `${c.key}:1${c.entry ? 1 : 0}`), + ) + }), + { numRuns: 200 * RUNS }, + ) + }) +}) diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json new file mode 100644 index 0000000..2bea4f3 --- /dev/null +++ b/packages/core/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "target": "ES2024", + "lib": ["ES2024", "DOM"], + "module": "ESNext", + "moduleResolution": "bundler", + "skipLibCheck": true, + "isolatedModules": true, + "resolveJsonModule": true, + "types": ["node"], + "noEmit": true + }, + "include": ["src", "test", "scripts"] +} diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts new file mode 100644 index 0000000..3e42797 --- /dev/null +++ b/packages/core/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + include: ['test/**/*.test.ts'], + }, +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7762c85..b33f653 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -169,6 +169,24 @@ importers: specifier: ^0.25.0 version: 0.25.12 + packages/core: + devDependencies: + '@types/node': + specifier: ^25.0.0 + version: 25.6.2 + fast-check: + specifier: ^4.3.0 + version: 4.10.2 + tsx: + specifier: ^4.19.0 + version: 4.21.0 + typescript: + specifier: ^6.0.0 + version: 6.0.3 + vitest: + specifier: ^4.1.6 + version: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + packages: '@asamuzakjp/css-color@5.1.11': @@ -2354,6 +2372,10 @@ packages: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} + fast-check@4.10.2: + resolution: {integrity: sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2711,6 +2733,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -5254,6 +5279,10 @@ snapshots: expect-type@1.3.0: {} + fast-check@4.10.2: + dependencies: + pure-rand: 8.4.2 + fast-deep-equal@3.1.3: {} fast-fifo@1.3.2: {} @@ -5598,6 +5627,8 @@ snapshots: punycode@2.3.1: {} + pure-rand@8.4.2: {} + rc@1.2.8: dependencies: deep-extend: 0.6.0 diff --git a/vitest.config.ts b/vitest.config.ts index d3737d5..21648a2 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,6 +10,6 @@ export default defineConfig({ environment: 'happy-dom', // Agent worktrees are full checkouts inside the repo; without this, running // tests from the main checkout also runs every worktree's copy. - exclude: [...configDefaults.exclude, '.claude/**'], + exclude: [...configDefaults.exclude, '.claude/**', 'packages/**'], }, }) From f8e1869d2694cbf49353982b5d1e1064e6042003 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 16:19:35 -0400 Subject: [PATCH 002/178] v2: protocol spec draft, test vectors, chunking experiments docs/protocol-v2.md specifies format 2 (canonical JSON, input rules, records, trees, access sets, version roots, format 1 aliases). test/vectors/v2.json is generated by scripts/gen-vectors.ts and checked in CI (content comparison, so reformatting is harmless). The chunking experiment kept the fixed-probability rule and lowered the forced leaf split to 8,192 entries. --- docs/protocol-v2.md | 272 ++++++++++ packages/core/scripts/experiment-chunking.ts | 184 +++++++ packages/core/scripts/gen-vectors.ts | 391 ++++++++++++++ packages/core/src/constants.ts | 2 +- packages/core/test/jcs.test.ts | 7 +- packages/core/test/vectors.test.ts | 19 + packages/core/test/vectors/v2.json | 525 +++++++++++++++++++ 7 files changed, 1397 insertions(+), 3 deletions(-) create mode 100644 docs/protocol-v2.md create mode 100644 packages/core/scripts/experiment-chunking.ts create mode 100644 packages/core/scripts/gen-vectors.ts create mode 100644 packages/core/test/vectors.test.ts create mode 100644 packages/core/test/vectors/v2.json diff --git a/docs/protocol-v2.md b/docs/protocol-v2.md new file mode 100644 index 0000000..bfc65bc --- /dev/null +++ b/docs/protocol-v2.md @@ -0,0 +1,272 @@ +# Underlay protocol, format 2 + +**Status: draft.** Every value marked _provisional_ can still change until the format is frozen. +After the freeze, changing any of them needs a new format number. The reference implementation is +`packages/core` (`@underlay/core`). The test vectors are in `packages/core/test/vectors/v2.json` +(see [Test vectors](#test-vectors)). + +This document is normative. Another implementation has to reproduce everything here byte for +byte: it must accept and reject the same inputs, build the same trees, and compute the same hashes. + +Design background: the edge redesign plan and its build notes (`planning/kf/underlay/edge-redesign*.md` +in the KF meta repo). + +## 1. Conventions + +- **hash(x)** is SHA-256 of `x`, written as 64 lowercase hex characters. A string is hashed as its + UTF-8 bytes. +- **JCS(v)** is the RFC 8785 canonical JSON of a value (section 2). +- **Key order** means comparing the UTF-8 bytes of two strings, which is Unicode code point order + (section 7). This is _not_ the UTF-16 order that JCS uses to sort object keys. +- Sizes are in bytes. Counts are non-negative integers no larger than 2⁵³ − 1. + +## 2. Canonical JSON + +Every hashed JSON document is serialized with [RFC 8785 (JCS)](https://www.rfc-editor.org/rfc/rfc8785): + +- no whitespace; +- strings escaped as ECMAScript `JSON.stringify` does; +- numbers serialized as ECMAScript `Number.prototype.toString` does, with `-0` written as `0`; +- object members sorted by their keys' UTF-16 code units. + +Implementation note for JavaScript: objects enumerate integer-like keys (`"9"`, `"10"`) first, in +numeric order, whatever order they were inserted in. So you can't canonicalize by sorting keys into +a new object and calling `JSON.stringify`. Write objects out as strings instead (see +`packages/core/src/jcs.ts`). + +## 3. Input rules + +These rules apply to every record line and schema a client pushes. The CLI and the server check +them on the **source text**, before or alongside parsing, because a parsed value has already lost +the information they need. + +| Rule | Rejected | Error code | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | +| Duplicate keys | Two members of one object whose keys are equal after unescaping (`"a"` and `"a"` are duplicates) | `duplicate_key` | +| Unsafe integers | An **integer literal** (no fraction, no exponent) whose magnitude exceeds 2⁵³ − 1 (`9007199254740991`). Literals with a fraction or exponent, such as `1e20`, `6.02e23` or `9007199254740993.0`, are accepted and take their IEEE 754 double value | `unsafe_integer` | +| Lone surrogates | A UTF-16 surrogate code unit, raw or `\u`-escaped, that is not part of a valid pair, in any string or key | `lone_surrogate` | +| Depth | Nesting deeper than `MAX_JSON_DEPTH` = 64 levels within `data`. Each object or array is one level, and the record envelope adds one more | `too_deep` | +| Record size | A canonical record (section 4) longer than `MAX_RECORD_BYTES` = 8,388,608 bytes | `record_too_large` | +| Record id | Not a string, empty, or longer than `MAX_ID_BYTES` = 1,024 UTF-8 bytes | `bad_id` | +| Type slug | Not a string, empty, longer than 128 UTF-8 bytes, starting with `.`, or containing `/`, `\`, U+0000–U+001F or U+007F | `bad_type` | +| Envelope | A record line that isn't an object with `id`, `type` and `data`, or whose optional `private` isn't a boolean | `bad_envelope` | +| Syntax | Anything that isn't JSON (RFC 8259) | `syntax` | + +Unicode is **not** normalized. `é` as U+00E9 and as U+0065 U+0301 are different ids with different +hashes. The test vectors include both. + +The limits are provisional. A sample of production data (206,862 records) found a largest record +of 12 KB and a longest id of 145 bytes. + +## 4. Records + +A record has an `id` (string), a `type` (type slug) and `data` (any JSON value). Its **canonical +form** is a fixed envelope with only `data` canonicalized: + +``` +'{"id":' + JCS(id) + ',"type":' + JCS(type) + ',"data":' + JCS(data) + '}' +``` + +- **Record hash** = hash(canonical form). +- **Record size** = the length in bytes of the canonical form. +- The envelope keeps the field order of format 1, so a record without integer-like keys has the + same hash in both formats (see [Format 1 hashes](#12-format-1-hashes)). +- Record ids are unique per type within a version, across both access sets (section 9). + +**File references.** A record references a file through any object, at any depth of `data`, whose +`$file` member is a string of the form `sha256:` followed by 64 lowercase hex characters. The +referenced file hash is the hex part. A reference object is not searched further for nested +references. A `$file` value of any other form is not a reference, and the object is searched as +usual. + +## 5. Schemas + +A type's schema is a JSON Schema document. **Schema hash** = hash(JCS(schema)). + +- A schema with `"private": true` at its root makes the type private (section 9). +- `"private": true` on a property (field-level privacy) is **rejected** in format 2. +- The validation dialect, and the exact behaviour required of a validator, is **to be specified** + in phase 2 of the build, after the differential test against production data. Until then, the + reference validator is the server's. + +## 6. Files + +**File hash** = hash(file bytes). A file's size is its length in bytes. + +## 7. Key order and boundary hash + +- **Key order** compares keys by their UTF-8 bytes, lexicographically, with a shorter prefix + first. Implementations must not use UTF-16 code-unit comparison (JavaScript's default `<` and + `sort()`). The two differ when one string has a character at U+10000 or above where the other has + one in U+E000–U+FFFF. +- **Boundary hash**: u(k) = the first 8 bytes of hash(k), read as a big-endian unsigned 64-bit + integer. +- **tz(k)** = the number of trailing zero bits of u(k), from 0 to 64. + +## 8. Trees + +A tree is a sorted set of entries with unique keys, split into nodes. There are two kinds: + +| Kind | Key | Entry tuple | Entry size | +| ----------- | --------------- | ------------------------------ | ---------- | +| Record tree | record id | `[id, recordHash, recordSize]` | recordSize | +| File tree | file hash (hex) | `[fileHash, fileSize]` | fileSize | + +### 8.1 Shape + +Parameters (provisional): + +| Name | Value | +| ----------------------------- | ----------------------------- | +| `LEAF_BOUNDARY_BITS` | 10 (mean leaf: 1,024 entries) | +| `INTERIOR_BOUNDARY_BITS_STEP` | 6 (mean fanout: 64) | +| `LEAF_MAX_ENTRIES` | 8,192 | +| `INTERIOR_MAX_CHILDREN` | 1,024 | + +- **Leaves (level 0).** Walk the entries in key order. The current leaf ends after entry `k` when + any of these holds: + - tz(k) ≥ 10; + - the leaf now holds `LEAF_MAX_ENTRIES` entries; + - `k` is the last entry. +- **Interior level i ≥ 1.** Walk the nodes of level i − 1 in order. The current level-i node ends + after a child `c` when any of these holds: + - tz(last key of `c`) ≥ 10 + 6i, and 10 + 6i ≤ 64; + - the node now has `INTERIOR_MAX_CHILDREN` children; + - `c` is the last node of level i − 1. +- **Root.** Build level after level. The root is the node of the **lowest level that has exactly one + node**. A tree with one leaf has that leaf as its root. An empty tree has no nodes, and its root is + `null`. + +These rules give the following properties: + +- The same entry set always produces the same tree, whatever order it was built in. +- A natural boundary (a key with tz(k) ≥ 10) depends only on the key. A forced split depends only + on the position since the previous boundary. +- Any range between two natural boundaries can therefore be rebuilt on its own. + +### 8.2 Node encoding + +``` +leaf: {"e":[entry, entry, ...],"t":"leaf"} +interior: {"e":[[lastKey, childHash, count, bytes], ...],"l":level,"t":"node"} +``` + +- Both are JCS documents: no whitespace, members in the order shown. +- `lastKey` is the last key under the child. `count` is the number of entries under it. `bytes` is + the sum of the entry sizes under it. +- The **node hash** is hash(encoded node). + +### 8.3 Validity + +A tree is valid exactly when rebuilding its entries under section 8.1 produces the same root hash. +A node received from outside (tree sync, mirrors) must also satisfy all of the following: + +- its bytes hash to the expected hash, and are the canonical encoding; +- keys are strictly increasing within the node and across the whole tree; +- every interior entry's `lastKey`, `count` and `bytes` match its child; +- every child is exactly one level below its parent; +- `count` ≥ 1 for every child. + +A node that hashes correctly but breaks a structural rule is invalid. Accepting one would give two +different roots for one entry set. The reference `fsck` is `verifyTree` in +`packages/core/src/tree/verify.ts`. + +Record JSON (`{"id",…}`) and node JSON (`{"e",…}`) have disjoint member names, so one can never +be read as the other. + +## 9. Access sets + +A version has two sets of content: `public` and `private`. + +- **Records.** A record pushed with `"private": true` belongs to the private set; any other record + belongs to the public set. Every record of a private type belongs to the private set. +- **Types.** Each set lists, per type, the schema hash and the tree of that set's records of the + type. + - A private type appears only in the private set. + - A public type appears in the public set, and also in the private set if it has private + records. + - A public type with no records still appears in the public set, with a `null` root. +- **Files.** A file belongs to every set that has a record referencing it. A file that is declared + in the push but referenced by no record belongs to the private set, unless the push marks it + public. +- **Readers.** Owners may read both sets; everyone else, the public set only. Whether the collection + itself is public is mutable collection metadata. It is not part of the version. + +## 10. Versions + +``` +SetObject = { + "types": { slug: { "schema": schemaHash, "root": treeHash|null, "count": n, "bytes": b }, ... }, + "files": { "root": treeHash|null, "count": n, "bytes": b } +} +PrivateSetObject = SetObject + { "salt": 64 hex chars } +root = { "underlay": 2, "metadata": object|null, "public": SetObject, "private": commitment|null } +``` + +- `count` and `bytes` are the root node's totals, or 0 for a `null` root. +- **Commitment** = hash(JCS(PrivateSetObject)). +- The salt is 32 random bytes, chosen once per collection and reused across its versions. That way + an unchanged private set keeps its commitment. +- `private` is `null` when the private set is **empty**, meaning it has no types and no files. +- **Version hash** = `"ulv2:"` + hash(JCS(root)). + +A version hash commits to content only. There is no parent pointer, so the same content gives the +same hash in any collection (unless it has a private set, whose salt differs). Lineage, semver, +messages and authorship are mutable server state. + +What each reader can check: + +- Public readers get the root and can verify the version hash and everything in the public set. + From the root they learn only whether a private set exists. +- Owners also get the private set object, salt included, and can check it against the commitment. + +## 11. Limits and constants + +All protocol constants are in `packages/core/src/constants.ts`, and the vectors file repeats them. + +## 12. Format 1 hashes + +Format 1 canonicalized `data` and schemas by sorting keys into a new object and then calling +`JSON.stringify`. That puts array-index keys (canonical decimal integers below 2³² − 1) first, in +numeric order. + +- The two formats agree whenever no object at any depth has an array-index key. +- When one does, the format 1 hash differs. Servers keep `(format-1 hash → format-2 hash)` aliases, + and the compatibility push API accepts format 1 hashes from older clients. +- Format 1 version hashes (`private:`, `public:`) are kept as aliases of the versions + they name. + +## Test vectors + +`packages/core/test/vectors/v2.json` holds: + +- the constants; +- JCS input and output pairs; +- input-rule verdicts per record line; +- record and schema canonical forms and hashes; +- boundary hashes; +- a key-order list; +- one leaf node and one interior node, spelled out; +- tree roots for several entry sets (empty; one entry; 1,000; 100,000; Unicode keys; a key set with + no natural boundaries, so every leaf split is forced); +- a file tree; +- two version roots, one with a private set and its commitment; +- file-reference extraction cases. + +Tree vectors give a recipe for generating their entries rather than listing them. +`scripts/gen-vectors.ts --check` runs in CI. A failure there means a protocol change, which has to +be deliberate and recorded here. + +## Changes from the design plan + +These were made during implementation and recorded with their reasons in `edge-redesign-build.md`: + +1. Interior entries carry each child's **last** key, not its first. The boundary rule is defined + on last keys, and a merge needs them to reuse unchanged subtrees without reading them. +2. Record-tree entries carry the record's size, so `bytes` can be verified from nodes alone. +3. Record ids are limited to 1,024 UTF-8 bytes, which bounds node size. +4. The unsafe-integer rule applies to integer literals in the source text. +5. File references have one definition (section 4). Format 1 used two. +6. `LEAF_MAX_ENTRIES` is 8,192 (the plan had 16,384). The chunking rule stays fixed-probability + rather than size-aware, because size-aware boundaries depend on position and that rules out + parallel commit units. diff --git a/packages/core/scripts/experiment-chunking.ts b/packages/core/scripts/experiment-chunking.ts new file mode 100644 index 0000000..71a3044 --- /dev/null +++ b/packages/core/scripts/experiment-chunking.ts @@ -0,0 +1,184 @@ +/** + * Tree-parameter experiment: the protocol's fixed-probability chunker against a + * size-aware one (Dolt-style, Weibull-shaped split hazard). + * + * pnpm tsx scripts/experiment-chunking.ts [N] + * + * Boundaries depend on SHA-256 of keys, so leaf sizes are distributed the same + * way for any key set; synthetic ids are enough for shape. What real collections + * add (id lengths, record sizes) is measured separately. + * + * Reports, per chunker: leaf size spread, node counts per level, height, forced + * splits, and write amplification (nodes written) for single updates, single + * inserts, random batches and appends. + */ +import { createHash } from 'node:crypto' + +import { + buildTree, + type Change, + type Chunking, + compareUtf8, + MapSource, + mergeTree, + type NodeDesc, + protocolChunking, + type RecordEntry, + recordTree, + sha256Hex, + type TreeSink, +} from '../src/index.js' + +const N = Number(process.argv[2] ?? 1_000_000) + +/** Size-aware: split hazard from a Weibull(k=4) target with the given mean. */ +function sizeAware( + leafMean: number, + fanoutMean: number, + leafMax: number, + interiorMax: number, +): Chunking { + const k = 4 + const gamma = 0.9064024770554771 // Γ(1 + 1/4) + const hazard = (mean: number) => { + const lambda = mean / gamma + const table: number[] = [0] + for (let s = 1; s <= 8 * mean; s++) { + const a = Math.pow((s - 1) / lambda, k) + const b = Math.pow(s / lambda, k) + table.push(1 - Math.exp(-(b - a))) + } + return (s: number) => table[s] ?? 1 + } + const leafH = hazard(leafMean) + const nodeH = hazard(fanoutMean) + return { + name: `size-aware(${leafMean},${fanoutMean})`, + ends(level, u, size) { + if (level === 0 && size >= leafMax) return true + if (level > 0 && size >= interiorMax) return true + // A uniform draw per level from the key's hash (u is its first 8 bytes; + // derive more bits per level from it). + const d = createHash('sha256').update(u).update(String(level)).digest() + const r = d.readUInt32BE(0) / 2 ** 32 + return r < (level === 0 ? leafH(size) : nodeH(size)) + }, + } +} + +const idOf = (i: number) => `W${(i * 2654435761) % 4294967296}`.padEnd(12, 'x') + i +const entry = (key: string, v = 0): RecordEntry => ({ + key, + hash: sha256Hex(`${key}:${v}`), + size: 600, +}) + +class StatsSink implements TreeSink { + nodes = new Map() + leaves = new Map() + written = 0 + leafSizes: number[] = [] + perLevel: number[] = [] + forced = 0 + constructor(private readonly leafMax: number) {} + leaf(desc: NodeDesc, json: string, entries: readonly RecordEntry[]) { + this.written++ + this.nodes.set(desc.hash, json) + this.leaves.set(desc.hash, entries) + this.leafSizes.push(entries.length) + this.perLevel[0] = (this.perLevel[0] ?? 0) + 1 + if (entries.length >= this.leafMax) this.forced++ + } + interior(desc: NodeDesc, json: string) { + this.written++ + this.nodes.set(desc.hash, json) + this.perLevel[desc.level] = (this.perLevel[desc.level] ?? 0) + 1 + } +} + +const pct = (xs: number[], p: number) => + xs[Math.min(xs.length - 1, Math.floor((p / 100) * xs.length))]! + +async function run(chunking: Chunking, leafMax: number) { + const keys = Array.from({ length: N }, (_, i) => idOf(i)).sort(compareUtf8) + const sink = new StatsSink(leafMax) + const t0 = performance.now() + const root = buildTree( + recordTree, + sink, + keys.map((k) => entry(k)), + { chunking }, + )! + const buildMs = performance.now() - t0 + const sizes = sink.leafSizes.slice().sort((a, b) => a - b) + const perLevel = sink.perLevel.slice() + const forced = sink.forced + const source = new MapSource(recordTree, sink.nodes, sink.leaves) + + const amp = async (label: string, makeChanges: () => Change[], trials: number) => { + let total = 0 + for (let t = 0; t < trials; t++) { + const before = sink.written + await mergeTree(source, sink, root.hash, makeChanges(), { chunking }) + total += sink.written - before + } + return `${label}: ${(total / trials).toFixed(1)}` + } + let seed = 12345 + const rand = () => (seed = (seed * 1103515245 + 12345) % 2147483648) / 2147483648 + const pick = () => keys[Math.floor(rand() * keys.length)]! + const sortC = (cs: Change[]) => cs.sort((a, b) => compareUtf8(a.key, b.key)) + const results = [ + await amp( + '1 update', + () => + [{ key: pick(), entry: entry(pick(), 1) }].map((c) => ({ + key: c.key, + entry: entry(c.key, 1), + })), + 50, + ), + await amp( + '1 insert', + () => { + const k = `${pick()}~new` + return [{ key: k, entry: entry(k) }] + }, + 50, + ), + await amp('1 delete', () => [{ key: pick(), entry: null }], 50), + await amp( + '1k random updates', + () => { + const ks = new Set() + while (ks.size < 1000) ks.add(pick()) + return sortC([...ks].map((k) => ({ key: k, entry: entry(k, 2) }))) + }, + 5, + ), + await amp( + '10k appends', + () => + Array.from({ length: 10_000 }, (_, i) => `zzzz${String(i).padStart(6, '0')}`).map((k) => ({ + key: k, + entry: entry(k), + })), + 3, + ), + ] + const leafNodeBytes = [...sink.leaves.keys()].slice(0, 2000).map((h) => sink.nodes.get(h)!.length) + const meanLeafBytes = leafNodeBytes.reduce((a, b) => a + b, 0) / leafNodeBytes.length + console.log(`\n## ${chunking.name}, N=${N.toLocaleString()}`) + console.log( + `build: ${(buildMs / 1000).toFixed(1)} s (${((buildMs * 1000) / N).toFixed(2)} µs/entry), height ${root.level + 1}`, + ) + console.log(`nodes per level: ${perLevel.map((n, l) => `L${l}=${n}`).join(' ')}`) + console.log( + `leaf entries: mean ${(N / sizes.length).toFixed(0)}, p1 ${pct(sizes, 1)}, p10 ${pct(sizes, 10)}, p50 ${pct(sizes, 50)}, p90 ${pct(sizes, 90)}, p99 ${pct(sizes, 99)}, max ${sizes[sizes.length - 1]}; <100: ${((100 * sizes.filter((s) => s < 100).length) / sizes.length).toFixed(1)}%, >3×mean: ${((100 * sizes.filter((s) => s > 3 * (N / sizes.length)).length) / sizes.length).toFixed(1)}%; forced ${forced}`, + ) + console.log(`leaf node JSON: mean ${(meanLeafBytes / 1024).toFixed(0)} KB`) + console.log(`nodes written per commit (mean): ${results.join(' · ')}`) +} + +await run(protocolChunking, 8_192) +await run(sizeAware(1024, 64, 8_192, 1024), 8_192) diff --git a/packages/core/scripts/gen-vectors.ts b/packages/core/scripts/gen-vectors.ts new file mode 100644 index 0000000..00115a0 --- /dev/null +++ b/packages/core/scripts/gen-vectors.ts @@ -0,0 +1,391 @@ +/** + * Generate the protocol test vectors: test/vectors/v2.json. + * + * pnpm --filter @underlay/core vectors # write + * pnpm --filter @underlay/core vectors --check # compare, exit 1 on any difference + * + * Tree vectors are given as a recipe (how to generate the entries) plus the + * expected results, so a second implementation can regenerate the inputs without + * a multi-megabyte fixture. test/vectors.test.ts runs --check in CI. + */ +import { readFileSync, writeFileSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { + boundaryBytes, + buildTree, + compareUtf8, + type FileEntry, + fileRefs, + fileTree, + hashRecord, + hashSchema, + InputRuleError, + jcs, + makeRoot, + MemorySink, + type NodeDesc, + parseRecordLine, + privateCommitment, + type PrivateSetObject, + type RecordEntry, + recordTree, + type SetObject, + sha256Hex, + trailingZeros, + versionHash, +} from '../src/index.js' + +// Non-ASCII test strings are built from code points: the formatter rewrites +// \u escapes into literal characters, which an editor could silently normalize. +const cp = (...points: number[]) => String.fromCodePoint(...points) +const EMOJI = cp(0x1f600) +const E_ACUTE = cp(0xe9) // é, one code point +const E_COMBINING = cp(0x65, 0x301) // é, e + combining acute + +const here = dirname(fileURLToPath(import.meta.url)) +const out = resolve(here, '../test/vectors/v2.json') + +// --- Canonical JSON ----------------------------------------------------------- + +const jcsCases = [ + '{"b":1,"a":2}', + '{"9":3,"10":2,"a":1}', + '{"numbers":[333333333.33333329,1E30,4.50,2e-3,0.000000000000000000000000001,-0,1e21,1e-7]}', + '{"\\u20ac":1,"\\r":2,"\\ufb33":3,"1":4,"\\ud83d\\ude00":5,"\\u0080":6,"\\u00f6":7}', + '{"s":"\\u0000\\u001f\\"\\\\\\/\\b\\f\\n\\r\\t\\u007f\\u2028é😀"}', + '[[],{},[{}],null,true,false,0,"",[1,[2,[3]]]]', + '{"e\\u0301":1,"\\u00e9":2}', +].map((input) => ({ input, output: jcs(JSON.parse(input)) })) + +// --- Input rules ---------------------------------------------------------------- + +const ruleCase = (line: string) => { + try { + const r = parseRecordLine(line) + return { line, ok: true, canonical: r.canonical, hash: sha256Hex(r.canonical) } + } catch (err) { + if (!(err instanceof InputRuleError)) throw err + return { line, ok: false, error: err.code } + } +} + +const deep = (n: number) => '['.repeat(n) + ']'.repeat(n) +const inputRuleCases = [ + '{"id":"r1","type":"Author","data":{"name":"Ada","year":1815}}', + '{"type":"Author","data":{"year":1815,"name":"Ada"},"id":"r1","private":true}', + '{"id":"r","type":"t","data":{"a":1,"a":2}}', + '{"id":"r","type":"t","data":{"a":1,"\\u0061":2}}', + '{"id":"r","type":"t","data":{"x":{"a":1},"y":{"a":1}}}', + '{"id":"r","type":"t","data":9007199254740991}', + '{"id":"r","type":"t","data":9007199254740992}', + '{"id":"r","type":"t","data":-9007199254740992}', + '{"id":"r","type":"t","data":[1e20,6.02e23,9007199254740993.0]}', + '{"id":"r","type":"t","data":"\\ud83d\\ude00"}', + '{"id":"r","type":"t","data":"\\ud83d"}', + '{"id":"r","type":"t","data":"\\ude00"}', + '{"id":"\\u00e9","type":"t","data":1}', + '{"id":"e\\u0301","type":"t","data":1}', + `{"id":"r","type":"t","data":${deep(64)}}`, + `{"id":"r","type":"t","data":${deep(65)}}`, + `{"id":"${'x'.repeat(1024)}","type":"t","data":1}`, + `{"id":"${'x'.repeat(1025)}","type":"t","data":1}`, + '{"id":"","type":"t","data":1}', + '{"id":"r","type":"a/b","data":1}', + '{"id":"r","type":"t"}', + '{"id":"r","type":"t","data":{"9":3,"10":2}}', +].map(ruleCase) + +// --- Hashes --------------------------------------------------------------------- + +const recordHashes = [ + { id: 'r1', type: 'Author', data: { name: 'Ada', year: 1815 } }, + { + id: 'r2', + type: 'Pub', + data: { title: 'On Computable Numbers', refs: [{ $file: `sha256:${'a'.repeat(64)}` }] }, + }, + { id: '9', type: 'T', data: { 10: 'x', 9: 'y', z: [1.5, null, true] } }, + { id: '😀', type: 'Émoji', data: 'plain string data' }, +].map((r) => ({ ...r, ...hashRecord(r.id, r.type, r.data) })) + +const schemaHashes = [ + { type: 'object', properties: { name: { type: 'string' } } }, + { type: 'object', properties: { 1: { type: 'string' }, 10: { type: 'integer' } }, private: true }, +].map((schema) => ({ schema, canonical: jcs(schema), hash: hashSchema(schema) })) + +const boundaryCases = ['', 'a', 'r0', 'k12345', EMOJI, E_COMBINING].map((key) => { + const u = boundaryBytes(key) + return { key, u: Buffer.from(u).toString('hex'), trailingZeros: trailingZeros(u) } +}) +// Keys whose boundary hash ends in ≥10 and ≥16 zero bits, found by search. +for (const want of [10, 16]) { + for (let i = 0; ; i++) { + const key = `b${i}` + const u = boundaryBytes(key) + if (trailingZeros(u) >= want) { + boundaryCases.push({ + key, + u: Buffer.from(u).toString('hex'), + trailingZeros: trailingZeros(u), + }) + break + } + } +} + +const keyOrder = [ + 'a', + 'b', + 'aa', + '', + 'Z', + cp(0x7f), + cp(0x80), + cp(0xffff), + EMOJI, + cp(0xe000), + E_ACUTE, + E_COMBINING, +] + .slice() + .sort(compareUtf8) + +// --- Trees ---------------------------------------------------------------------- + +interface TreeVector { + name: string + recipe: string + entries: number + root: string | null + count: number + bytes: number + height: number + nodesPerLevel: number[] + leafSizes?: number[] +} + +function recordEntries(keys: string[]): RecordEntry[] { + return keys + .slice() + .sort(compareUtf8) + .map((key) => { + const { hash, canonical } = hashRecord(key, 'T', { k: key }) + return { key, hash, size: Buffer.byteLength(canonical) } + }) +} + +function treeVector( + name: string, + recipe: string, + entries: RecordEntry[], + withLeafSizes = false, +): TreeVector { + const levels: number[] = [] + const leafSizes: number[] = [] + const sink = { + leaf(_d: NodeDesc, _j: string, es: readonly RecordEntry[]) { + levels[0] = (levels[0] ?? 0) + 1 + leafSizes.push(es.length) + }, + interior(d: NodeDesc) { + levels[d.level] = (levels[d.level] ?? 0) + 1 + }, + } + const root = buildTree(recordTree, sink, entries) + const v: TreeVector = { + name, + recipe, + entries: entries.length, + root: root?.hash ?? null, + count: root?.count ?? 0, + bytes: root?.bytes ?? 0, + height: root ? root.level + 1 : 0, + nodesPerLevel: levels.slice(0, root ? root.level + 1 : 0), + } + if (withLeafSizes) v.leafSizes = leafSizes + return v +} + +const RECIPE = + 'entries: for each key, hashRecord(key, "T", {"k": key}) → [key, hash, UTF-8 length of the canonical record]; sorted by key' +const range = (n: number, f: (i: number) => T): T[] => Array.from({ length: n }, (_, i) => f(i)) + +const trees: TreeVector[] = [ + treeVector('empty', `${RECIPE}; no keys`, []), + treeVector('one', `${RECIPE}; keys ["only"]`, recordEntries(['only'])), + treeVector( + 'small', + `${RECIPE}; keys "r0".."r999"`, + recordEntries(range(1000, (i) => `r${i}`)), + true, + ), + treeVector( + 'medium', + `${RECIPE}; keys "r0".."r99999"`, + recordEntries(range(100_000, (i) => `r${i}`)), + ), + treeVector( + 'unicode', + `${RECIPE}; keys "é", "e\\u0301", "😀", "\\uffff" for i in 0..2499`, + recordEntries( + range(2500, (i) => [ + `${E_ACUTE}${i}`, + `${E_COMBINING}${i}`, + `${EMOJI}${i}`, + `${cp(0xffff)}${i}`, + ]).flat(), + ), + true, + ), + treeVector( + 'forced-leaf-splits', + `${RECIPE}; keys "f" for i in 0..19999 whose boundary hash has fewer than 10 trailing zero bits (no natural leaf boundary at all)`, + recordEntries( + range(20_000, (i) => `f${i}`).filter((k) => trailingZeros(boundaryBytes(k)) < 10), + ), + true, + ), +] + +// File trees: key = file hash, value = size. +const fileEntries: FileEntry[] = range(3000, (i) => ({ + key: sha256Hex(`file${i}`), + size: i * 1000 + 1, +})).sort((a, b) => compareUtf8(a.key, b.key)) +const fileRoot = buildTree(fileTree, new MemorySink(), fileEntries) + +// One leaf, spelled out, so the node encoding can be checked by eye. +const tinyLeafSink = new MemorySink() +const tinyLeaf = buildTree(recordTree, tinyLeafSink, recordEntries(['a', 'b', 'c']))! +const tinyInteriorSink = new MemorySink() +const tinyInteriorRoot = buildTree( + recordTree, + tinyInteriorSink, + recordEntries(range(3000, (i) => `r${i}`)), +)! + +// --- Roots ---------------------------------------------------------------------- + +const smallTree = trees.find((t) => t.name === 'small')! +const publicSet: SetObject = { + types: { + Author: { + schema: schemaHashes[0]!.hash, + root: smallTree.root, + count: smallTree.count, + bytes: smallTree.bytes, + }, + Empty: { schema: schemaHashes[0]!.hash, root: null, count: 0, bytes: 0 }, + }, + files: { root: fileRoot!.hash, count: fileRoot!.count, bytes: fileRoot!.bytes }, +} +const privateSet: PrivateSetObject = { + types: { + Secret: { + schema: schemaHashes[1]!.hash, + root: tinyLeaf.hash, + count: tinyLeaf.count, + bytes: tinyLeaf.bytes, + }, + }, + files: { root: null, count: 0, bytes: 0 }, + salt: sha256Hex('vector salt'), +} +const metadata = { name: 'Vector collection', readme: '# Hi\n', tags: ['a', 'b'], 10: 'x', 9: 'y' } +const rootPublicOnly = makeRoot(metadata, publicSet, null) +const rootWithPrivate = makeRoot(metadata, publicSet, privateSet) +const roots = [ + { + name: 'public-only', + root: rootPublicOnly, + canonical: jcs(rootPublicOnly), + versionHash: versionHash(rootPublicOnly), + }, + { + name: 'with-private', + privateSet, + privateCanonical: jcs(privateSet), + commitment: privateCommitment(privateSet), + root: rootWithPrivate, + canonical: jcs(rootWithPrivate), + versionHash: versionHash(rootWithPrivate), + }, +] + +// --- File references -------------------------------------------------------------- + +const H = (c: string) => c.repeat(64) +const fileRefCases = [ + { data: { f: { $file: `sha256:${H('a')}` } } }, + { + data: { + list: [ + { $file: `sha256:${H('b')}` }, + { nested: { $file: `sha256:${H('c')}`, extra: { $file: `sha256:${H('d')}` } } }, + ], + }, + }, + { + data: { + f: { $file: `sha256:${H('A')}` }, + g: { $file: H('e') }, + h: { $file: 1, inner: { $file: `sha256:${H('f')}` } }, + }, + }, + { data: [{ $file: `sha256:${H('a')}` }, { $file: `sha256:${H('a')}` }] }, +].map((c) => ({ ...c, refs: fileRefs(c.data) })) + +const vectors = { + format: 2, + generatedBy: 'packages/core/scripts/gen-vectors.ts', + constants: await import('../src/constants.js').then((m) => ({ ...m })), + jcs: jcsCases, + inputRules: inputRuleCases, + recordHashes, + schemaHashes, + boundary: boundaryCases, + keyOrder, + nodes: { + leaf: { + keys: ['a', 'b', 'c'], + recipe: RECIPE, + hash: tinyLeaf.hash, + json: tinyLeafSink.nodes.get(tinyLeaf.hash), + }, + interior: { + recipe: `${RECIPE}; keys "r0".."r2999"; the root node`, + hash: tinyInteriorRoot.hash, + json: tinyInteriorSink.nodes.get(tinyInteriorRoot.hash), + }, + }, + trees, + fileTree: { + recipe: + 'entries: key = sha256Hex("file"), size = i × 1000 + 1, for i in 0..2999; sorted by key', + root: fileRoot!.hash, + count: fileRoot!.count, + bytes: fileRoot!.bytes, + }, + roots, + fileRefs: fileRefCases, +} + +const text = JSON.stringify(vectors, null, 1) + '\n' +if (process.argv.includes('--check')) { + const existing = readFileSync(out, 'utf8') + // Compare content, not layout: the formatter (and the pre-commit hook) may + // re-wrap the file. + const same = JSON.stringify(JSON.parse(existing)) === JSON.stringify(JSON.parse(text)) + if (!same) { + console.error( + 'test/vectors/v2.json is out of date with the implementation. If the change is intended, it is a protocol change: regenerate with `pnpm vectors` and say so in docs/protocol-v2.md.', + ) + process.exit(1) + } + console.log('vectors match') +} else { + writeFileSync(out, text) + console.log(`wrote ${out}`) +} diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index c3df3ad..41d8599 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -21,7 +21,7 @@ export const VERSION_HASH_PREFIX = 'ulv2:' /** A leaf ends after a key whose boundary hash has at least this many trailing zero bits (mean 1,024 entries). */ export const LEAF_BOUNDARY_BITS = 10 /** Forced leaf split: a leaf never holds more entries than this. */ -export const LEAF_MAX_ENTRIES = 16_384 +export const LEAF_MAX_ENTRIES = 8_192 /** Level i ≥ 1 needs LEAF_BOUNDARY_BITS + i × this many trailing zero bits (mean fanout 64). */ export const INTERIOR_BOUNDARY_BITS_STEP = 6 /** Forced interior split: an interior node never holds more children than this. */ diff --git a/packages/core/test/jcs.test.ts b/packages/core/test/jcs.test.ts index b9dd98d..2483a94 100644 --- a/packages/core/test/jcs.test.ts +++ b/packages/core/test/jcs.test.ts @@ -28,9 +28,12 @@ describe('jcs', () => { '"1":"One","\\ud83d\\ude00":"Emoji: Grinning Face","\\u0080":"Control",' + '"\\u00f6":"Latin Small Letter O With Diaeresis"}', ) + // Built from code points: the formatter turns \u escapes into literal + // characters, and U+FB33 would not survive an NFC-normalizing editor. + const k = (cp: number) => JSON.stringify(String.fromCodePoint(cp)) expect(jcs(input)).toBe( - '{"\\r":"Carriage Return","1":"One","\u0080":"Control","ö":"Latin Small Letter O With Diaeresis",' + - '"€":"Euro Sign","😀":"Emoji: Grinning Face","דּ":"Hebrew Letter Dalet With Dagesh"}', + `{"\\r":"Carriage Return","1":"One",${k(0x80)}:"Control",${k(0xf6)}:"Latin Small Letter O With Diaeresis",` + + `${k(0x20ac)}:"Euro Sign",${k(0x1f600)}:"Emoji: Grinning Face",${k(0xfb33)}:"Hebrew Letter Dalet With Dagesh"}`, ) }) diff --git a/packages/core/test/vectors.test.ts b/packages/core/test/vectors.test.ts new file mode 100644 index 0000000..2d56478 --- /dev/null +++ b/packages/core/test/vectors.test.ts @@ -0,0 +1,19 @@ +import { execFileSync } from 'node:child_process' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { describe, expect, it } from 'vitest' + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') + +describe('protocol test vectors', () => { + // The vectors are the protocol's contract with other implementations. If this + // fails, the implementation changed a hash, a tree shape or an input rule. + it('test/vectors/v2.json matches the implementation', () => { + const out = execFileSync('npx', ['tsx', 'scripts/gen-vectors.ts', '--check'], { + cwd: root, + encoding: 'utf8', + }) + expect(out).toContain('vectors match') + }, 60_000) +}) diff --git a/packages/core/test/vectors/v2.json b/packages/core/test/vectors/v2.json new file mode 100644 index 0000000..542781c --- /dev/null +++ b/packages/core/test/vectors/v2.json @@ -0,0 +1,525 @@ +{ + "format": 2, + "generatedBy": "packages/core/scripts/gen-vectors.ts", + "constants": { + "FORMAT_VERSION": 2, + "INTERIOR_BOUNDARY_BITS_STEP": 6, + "INTERIOR_MAX_CHILDREN": 1024, + "LEAF_BOUNDARY_BITS": 10, + "LEAF_MAX_ENTRIES": 8192, + "MAX_ID_BYTES": 1024, + "MAX_JSON_DEPTH": 64, + "MAX_RECORD_BYTES": 8388608, + "MAX_SAFE_INTEGER_LITERAL": "9007199254740991", + "MAX_SCHEMA_BYTES": 262144, + "MAX_TYPE_BYTES": 128, + "VERSION_HASH_PREFIX": "ulv2:" + }, + "jcs": [ + { + "input": "{\"b\":1,\"a\":2}", + "output": "{\"a\":2,\"b\":1}" + }, + { + "input": "{\"9\":3,\"10\":2,\"a\":1}", + "output": "{\"10\":2,\"9\":3,\"a\":1}" + }, + { + "input": "{\"numbers\":[333333333.33333329,1E30,4.50,2e-3,0.000000000000000000000000001,-0,1e21,1e-7]}", + "output": "{\"numbers\":[333333333.3333333,1e+30,4.5,0.002,1e-27,0,1e+21,1e-7]}" + }, + { + "input": "{\"\\u20ac\":1,\"\\r\":2,\"\\ufb33\":3,\"1\":4,\"\\ud83d\\ude00\":5,\"\\u0080\":6,\"\\u00f6\":7}", + "output": "{\"\\r\":2,\"1\":4,\"€\":6,\"ö\":7,\"€\":1,\"😀\":5,\"דּ\":3}" + }, + { + "input": "{\"s\":\"\\u0000\\u001f\\\"\\\\\\/\\b\\f\\n\\r\\t\\u007f\\u2028é😀\"}", + "output": "{\"s\":\"\\u0000\\u001f\\\"\\\\/\\b\\f\\n\\r\\t
é😀\"}" + }, + { + "input": "[[],{},[{}],null,true,false,0,\"\",[1,[2,[3]]]]", + "output": "[[],{},[{}],null,true,false,0,\"\",[1,[2,[3]]]]" + }, + { + "input": "{\"e\\u0301\":1,\"\\u00e9\":2}", + "output": "{\"é\":1,\"é\":2}" + } + ], + "inputRules": [ + { + "line": "{\"id\":\"r1\",\"type\":\"Author\",\"data\":{\"name\":\"Ada\",\"year\":1815}}", + "ok": true, + "canonical": "{\"id\":\"r1\",\"type\":\"Author\",\"data\":{\"name\":\"Ada\",\"year\":1815}}", + "hash": "adefbd10aa438f0c6ed1627817f391ac6cc0441737ee09b4ebcc30fbd8386c63" + }, + { + "line": "{\"type\":\"Author\",\"data\":{\"year\":1815,\"name\":\"Ada\"},\"id\":\"r1\",\"private\":true}", + "ok": true, + "canonical": "{\"id\":\"r1\",\"type\":\"Author\",\"data\":{\"name\":\"Ada\",\"year\":1815}}", + "hash": "adefbd10aa438f0c6ed1627817f391ac6cc0441737ee09b4ebcc30fbd8386c63" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":{\"a\":1,\"a\":2}}", + "ok": false, + "error": "duplicate_key" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":{\"a\":1,\"\\u0061\":2}}", + "ok": false, + "error": "duplicate_key" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":{\"x\":{\"a\":1},\"y\":{\"a\":1}}}", + "ok": true, + "canonical": "{\"id\":\"r\",\"type\":\"t\",\"data\":{\"x\":{\"a\":1},\"y\":{\"a\":1}}}", + "hash": "edec381ac18eef8cc201472a6a9c1900fcc423eda52191d6e95cf96aeb42e3c0" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":9007199254740991}", + "ok": true, + "canonical": "{\"id\":\"r\",\"type\":\"t\",\"data\":9007199254740991}", + "hash": "5ece22156b8f1998b32f1d55690ab678ed56ef26fb77efa3ca3e7cdeceea9014" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":9007199254740992}", + "ok": false, + "error": "unsafe_integer" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":-9007199254740992}", + "ok": false, + "error": "unsafe_integer" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":[1e20,6.02e23,9007199254740993.0]}", + "ok": true, + "canonical": "{\"id\":\"r\",\"type\":\"t\",\"data\":[100000000000000000000,6.02e+23,9007199254740992]}", + "hash": "3dbe19a933d5f58f671e585d69e9a2a8821e99b21b9251d500c69dcaef8d1b99" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":\"\\ud83d\\ude00\"}", + "ok": true, + "canonical": "{\"id\":\"r\",\"type\":\"t\",\"data\":\"😀\"}", + "hash": "954aa6984679f11f06cd47ba3cf7f0cfbb90025622f4bd3b950e307e3d6c42e6" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":\"\\ud83d\"}", + "ok": false, + "error": "lone_surrogate" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":\"\\ude00\"}", + "ok": false, + "error": "lone_surrogate" + }, + { + "line": "{\"id\":\"\\u00e9\",\"type\":\"t\",\"data\":1}", + "ok": true, + "canonical": "{\"id\":\"é\",\"type\":\"t\",\"data\":1}", + "hash": "727990d1c3826f31ce44381963232c96e401f5032a93340006aac42d5c828b3b" + }, + { + "line": "{\"id\":\"e\\u0301\",\"type\":\"t\",\"data\":1}", + "ok": true, + "canonical": "{\"id\":\"é\",\"type\":\"t\",\"data\":1}", + "hash": "18e9735e8c9dacb7b9fff7fa443fc2524850405c24affa7d5cbd03c9412d5b42" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]}", + "ok": true, + "canonical": "{\"id\":\"r\",\"type\":\"t\",\"data\":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]}", + "hash": "8ba61dc6000260fb0836c65cfdd3510a8124d231d9552e643aea595bc182c2a3" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]}", + "ok": false, + "error": "too_deep" + }, + { + "line": "{\"id\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",\"type\":\"t\",\"data\":1}", + "ok": true, + "canonical": "{\"id\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",\"type\":\"t\",\"data\":1}", + "hash": "678076cecc40fc4cdd0352e4b3ec9124c8fce4830526aa1c2f20e0b4970ba5fc" + }, + { + "line": "{\"id\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",\"type\":\"t\",\"data\":1}", + "ok": false, + "error": "bad_id" + }, + { + "line": "{\"id\":\"\",\"type\":\"t\",\"data\":1}", + "ok": false, + "error": "bad_id" + }, + { + "line": "{\"id\":\"r\",\"type\":\"a/b\",\"data\":1}", + "ok": false, + "error": "bad_type" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\"}", + "ok": false, + "error": "bad_envelope" + }, + { + "line": "{\"id\":\"r\",\"type\":\"t\",\"data\":{\"9\":3,\"10\":2}}", + "ok": true, + "canonical": "{\"id\":\"r\",\"type\":\"t\",\"data\":{\"10\":2,\"9\":3}}", + "hash": "b551945866116dcfa7737f4e687c4ffac0dcd4b2da992b1fb2a74d38a72bfe05" + } + ], + "recordHashes": [ + { + "id": "r1", + "type": "Author", + "data": { + "name": "Ada", + "year": 1815 + }, + "hash": "adefbd10aa438f0c6ed1627817f391ac6cc0441737ee09b4ebcc30fbd8386c63", + "canonical": "{\"id\":\"r1\",\"type\":\"Author\",\"data\":{\"name\":\"Ada\",\"year\":1815}}" + }, + { + "id": "r2", + "type": "Pub", + "data": { + "title": "On Computable Numbers", + "refs": [ + { + "$file": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + "hash": "0f96d6e03006818b0e48479a6e4317598c9e570e28b68cbff452bfb622d8d919", + "canonical": "{\"id\":\"r2\",\"type\":\"Pub\",\"data\":{\"refs\":[{\"$file\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}],\"title\":\"On Computable Numbers\"}}" + }, + { + "id": "9", + "type": "T", + "data": { + "9": "y", + "10": "x", + "z": [1.5, null, true] + }, + "hash": "ff47fdde5857af087fbb016abdc6b0983b78634e6eaab49b184cde1694e15c6d", + "canonical": "{\"id\":\"9\",\"type\":\"T\",\"data\":{\"10\":\"x\",\"9\":\"y\",\"z\":[1.5,null,true]}}" + }, + { + "id": "😀", + "type": "Émoji", + "data": "plain string data", + "hash": "3bbf6c464350187596dddc952fba294126ae3a62b962b078d7ba3a723f1f6582", + "canonical": "{\"id\":\"😀\",\"type\":\"Émoji\",\"data\":\"plain string data\"}" + } + ], + "schemaHashes": [ + { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + }, + "canonical": "{\"properties\":{\"name\":{\"type\":\"string\"}},\"type\":\"object\"}", + "hash": "2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2" + }, + { + "schema": { + "type": "object", + "properties": { + "1": { + "type": "string" + }, + "10": { + "type": "integer" + } + }, + "private": true + }, + "canonical": "{\"private\":true,\"properties\":{\"1\":{\"type\":\"string\"},\"10\":{\"type\":\"integer\"}},\"type\":\"object\"}", + "hash": "2e61ca37bf92bd289753852ae33a88ec3df6e6606b761d4b0199cfd30a850ded" + } + ], + "boundary": [ + { + "key": "", + "u": "e3b0c44298fc1c14", + "trailingZeros": 2 + }, + { + "key": "a", + "u": "ca978112ca1bbdca", + "trailingZeros": 1 + }, + { + "key": "r0", + "u": "dd191696e15e2ee2", + "trailingZeros": 1 + }, + { + "key": "k12345", + "u": "054b40ca4964a150", + "trailingZeros": 4 + }, + { + "key": "😀", + "u": "f0443a342c5ef547", + "trailingZeros": 0 + }, + { + "key": "é", + "u": "bf12767b0f2a56b2", + "trailingZeros": 1 + }, + { + "key": "b525", + "u": "9de5e591b68ce800", + "trailingZeros": 11 + }, + { + "key": "b54026", + "u": "e36ea7df953a0000", + "trailingZeros": 17 + } + ], + "keyOrder": ["", "Z", "a", "aa", "b", "é", "", "€", "é", "", "￿", "😀"], + "nodes": { + "leaf": { + "keys": ["a", "b", "c"], + "recipe": "entries: for each key, hashRecord(key, \"T\", {\"k\": key}) → [key, hash, UTF-8 length of the canonical record]; sorted by key", + "hash": "7453c994a2387a24e5074f003b2b23660f9c310c0da3142c61981976817bcafa", + "json": "{\"e\":[[\"a\",\"6d3f26485cae4e156436aba786fad1ba0a2f7777525ed0835ca983d3dcca34c9\",38],[\"b\",\"3676805073b7e99c07673ec1c85a9da7383d1dfe7f6723c4c0b18b4459973421\",38],[\"c\",\"e4d4edd2615ca27ec9a5d5dd9f7328e10f325aa9836d17955a236c44073828ee\",38]],\"t\":\"leaf\"}" + }, + "interior": { + "recipe": "entries: for each key, hashRecord(key, \"T\", {\"k\": key}) → [key, hash, UTF-8 length of the canonical record]; sorted by key; keys \"r0\"..\"r2999\"; the root node", + "hash": "07af5e7b6a4395a189fcad73dcd10f0d57381838e60dd6bd5dd4ef701cff3618", + "json": "{\"e\":[[\"r2936\",\"a79185f218e237eb4199fbcede4cfcf83a1ff219b6da1270b9a2936eb3757372\",2154,98598],[\"r633\",\"e52ef8ca8081b67f556066e7a8716aef26d6cec5a653cb680b54a4d739346941\",441,19446],[\"r95\",\"9b54ff0cf254dd6c04c156b344be82e4134be503cf6d6328d63509705e2617a8\",351,15368],[\"r999\",\"54c1584f0968be01c571de2fdd3133a4c1227d17e3302b1b84f5286d9e424901\",54,2368]],\"l\":1,\"t\":\"node\"}" + } + }, + "trees": [ + { + "name": "empty", + "recipe": "entries: for each key, hashRecord(key, \"T\", {\"k\": key}) → [key, hash, UTF-8 length of the canonical record]; sorted by key; no keys", + "entries": 0, + "root": null, + "count": 0, + "bytes": 0, + "height": 0, + "nodesPerLevel": [] + }, + { + "name": "one", + "recipe": "entries: for each key, hashRecord(key, \"T\", {\"k\": key}) → [key, hash, UTF-8 length of the canonical record]; sorted by key; keys [\"only\"]", + "entries": 1, + "root": "691bf271695bfcfa5bdf79182bbe47d51aef8bf68278b0cd3a2c1f006094fc34", + "count": 1, + "bytes": 44, + "height": 1, + "nodesPerLevel": [1] + }, + { + "name": "small", + "recipe": "entries: for each key, hashRecord(key, \"T\", {\"k\": key}) → [key, hash, UTF-8 length of the canonical record]; sorted by key; keys \"r0\"..\"r999\"", + "entries": 1000, + "root": "ea81aa2788dfedb958068212effe511ea75d4ce094a2ce66e9b7cd70f77a93ed", + "count": 1000, + "bytes": 43780, + "height": 2, + "nodesPerLevel": [3, 1], + "leafSizes": [595, 351, 54] + }, + { + "name": "medium", + "recipe": "entries: for each key, hashRecord(key, \"T\", {\"k\": key}) → [key, hash, UTF-8 length of the canonical record]; sorted by key; keys \"r0\"..\"r99999\"", + "entries": 100000, + "root": "0e8532a8b014b2c0ec4895d582f7469390e38ed011305dbef315b501a34dc701", + "count": 100000, + "bytes": 4777780, + "height": 3, + "nodesPerLevel": [122, 4, 1] + }, + { + "name": "unicode", + "recipe": "entries: for each key, hashRecord(key, \"T\", {\"k\": key}) → [key, hash, UTF-8 length of the canonical record]; sorted by key; keys \"é\", \"e\\u0301\", \"😀\", \"\\uffff\" for i in 0..2499", + "entries": 10000, + "root": "c167944c45a676e0355920757ebdf4c6308a3554f6cb8a36d3e5833636ad5ec4", + "count": 10000, + "bytes": 491120, + "height": 2, + "nodesPerLevel": [11, 1], + "leafSizes": [735, 555, 707, 275, 2250, 609, 217, 1649, 622, 1619, 762] + }, + { + "name": "forced-leaf-splits", + "recipe": "entries: for each key, hashRecord(key, \"T\", {\"k\": key}) → [key, hash, UTF-8 length of the canonical record]; sorted by key; keys \"f\" for i in 0..19999 whose boundary hash has fewer than 10 trailing zero bits (no natural leaf boundary at all)", + "entries": 19981, + "root": "368f46bae82edcb837de3e6e2fa612703b9a2cc3ad52ff9f826e6b126464afe9", + "count": 19981, + "bytes": 936900, + "height": 2, + "nodesPerLevel": [3, 1], + "leafSizes": [8192, 8192, 3597] + } + ], + "fileTree": { + "recipe": "entries: key = sha256Hex(\"file\"), size = i × 1000 + 1, for i in 0..2999; sorted by key", + "root": "8398cadc93b9896cea5bf7f08ccfdaf4e2420c29bf71d23fdaeadfb657307e39", + "count": 3000, + "bytes": 4498503000 + }, + "roots": [ + { + "name": "public-only", + "root": { + "underlay": 2, + "metadata": { + "9": "y", + "10": "x", + "name": "Vector collection", + "readme": "# Hi\n", + "tags": ["a", "b"] + }, + "public": { + "types": { + "Author": { + "schema": "2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2", + "root": "ea81aa2788dfedb958068212effe511ea75d4ce094a2ce66e9b7cd70f77a93ed", + "count": 1000, + "bytes": 43780 + }, + "Empty": { + "schema": "2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2", + "root": null, + "count": 0, + "bytes": 0 + } + }, + "files": { + "root": "8398cadc93b9896cea5bf7f08ccfdaf4e2420c29bf71d23fdaeadfb657307e39", + "count": 3000, + "bytes": 4498503000 + } + }, + "private": null + }, + "canonical": "{\"metadata\":{\"10\":\"x\",\"9\":\"y\",\"name\":\"Vector collection\",\"readme\":\"# Hi\\n\",\"tags\":[\"a\",\"b\"]},\"private\":null,\"public\":{\"files\":{\"bytes\":4498503000,\"count\":3000,\"root\":\"8398cadc93b9896cea5bf7f08ccfdaf4e2420c29bf71d23fdaeadfb657307e39\"},\"types\":{\"Author\":{\"bytes\":43780,\"count\":1000,\"root\":\"ea81aa2788dfedb958068212effe511ea75d4ce094a2ce66e9b7cd70f77a93ed\",\"schema\":\"2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2\"},\"Empty\":{\"bytes\":0,\"count\":0,\"root\":null,\"schema\":\"2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2\"}}},\"underlay\":2}", + "versionHash": "ulv2:7d96a42b7841767d577510d2d25591e40a3ff67ae2bbdb16e15a2d12e0c60ed1" + }, + { + "name": "with-private", + "privateSet": { + "types": { + "Secret": { + "schema": "2e61ca37bf92bd289753852ae33a88ec3df6e6606b761d4b0199cfd30a850ded", + "root": "7453c994a2387a24e5074f003b2b23660f9c310c0da3142c61981976817bcafa", + "count": 3, + "bytes": 114 + } + }, + "files": { + "root": null, + "count": 0, + "bytes": 0 + }, + "salt": "9bd47d892258c4edb74ef4824f8fc23bb9e09267623afce8bc6e6e098257d84c" + }, + "privateCanonical": "{\"files\":{\"bytes\":0,\"count\":0,\"root\":null},\"salt\":\"9bd47d892258c4edb74ef4824f8fc23bb9e09267623afce8bc6e6e098257d84c\",\"types\":{\"Secret\":{\"bytes\":114,\"count\":3,\"root\":\"7453c994a2387a24e5074f003b2b23660f9c310c0da3142c61981976817bcafa\",\"schema\":\"2e61ca37bf92bd289753852ae33a88ec3df6e6606b761d4b0199cfd30a850ded\"}}}", + "commitment": "77c3bd2fc048eae2b77d74d765f1e5699b4fc27662668dc84165d6cf597ea811", + "root": { + "underlay": 2, + "metadata": { + "9": "y", + "10": "x", + "name": "Vector collection", + "readme": "# Hi\n", + "tags": ["a", "b"] + }, + "public": { + "types": { + "Author": { + "schema": "2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2", + "root": "ea81aa2788dfedb958068212effe511ea75d4ce094a2ce66e9b7cd70f77a93ed", + "count": 1000, + "bytes": 43780 + }, + "Empty": { + "schema": "2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2", + "root": null, + "count": 0, + "bytes": 0 + } + }, + "files": { + "root": "8398cadc93b9896cea5bf7f08ccfdaf4e2420c29bf71d23fdaeadfb657307e39", + "count": 3000, + "bytes": 4498503000 + } + }, + "private": "77c3bd2fc048eae2b77d74d765f1e5699b4fc27662668dc84165d6cf597ea811" + }, + "canonical": "{\"metadata\":{\"10\":\"x\",\"9\":\"y\",\"name\":\"Vector collection\",\"readme\":\"# Hi\\n\",\"tags\":[\"a\",\"b\"]},\"private\":\"77c3bd2fc048eae2b77d74d765f1e5699b4fc27662668dc84165d6cf597ea811\",\"public\":{\"files\":{\"bytes\":4498503000,\"count\":3000,\"root\":\"8398cadc93b9896cea5bf7f08ccfdaf4e2420c29bf71d23fdaeadfb657307e39\"},\"types\":{\"Author\":{\"bytes\":43780,\"count\":1000,\"root\":\"ea81aa2788dfedb958068212effe511ea75d4ce094a2ce66e9b7cd70f77a93ed\",\"schema\":\"2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2\"},\"Empty\":{\"bytes\":0,\"count\":0,\"root\":null,\"schema\":\"2b7196d853bac7cea83330be9c2073848dedc10746eaf403bb5f73687531baf2\"}}},\"underlay\":2}", + "versionHash": "ulv2:c9998bb6ea5063e913226f6b612841dd02b87babf0d7826dce7635d02185f53c" + } + ], + "fileRefs": [ + { + "data": { + "f": { + "$file": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }, + "refs": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"] + }, + { + "data": { + "list": [ + { + "$file": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + { + "nested": { + "$file": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "extra": { + "$file": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + } + } + } + ] + }, + "refs": [ + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + ] + }, + { + "data": { + "f": { + "$file": "sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + }, + "g": { + "$file": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }, + "h": { + "$file": 1, + "inner": { + "$file": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + } + } + }, + "refs": ["ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"] + }, + { + "data": [ + { + "$file": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + { + "$file": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ], + "refs": ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"] + } + ] +} From 694af58aed06e4814089edc1caab06b4ffecfc27 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 16:30:30 -0400 Subject: [PATCH 003/178] v2 server: ports and adapters (phase 3) @underlay/server gets the four ports and their adapters: blob store (S3 API via aws4fetch, filesystem with HMAC-presigned URLs, memory), SQLite via Drizzle (libsql on Node, D1 on Workers; batches only, no interactive transactions), jobs (SQLite table + runner on Node, Queues on Workers) and cache (LRU, Cache API). The object layer stores nodes, bodies (split into parts past 8 MB), out-of-line records, roots, private sets and schemas, and reads them back as a NodeSource. Node and Worker entries serve a skeleton Hono app. Core: TreeSink gains drain() so merges apply write backpressure. --- packages/core/package.json | 6 + packages/core/src/tree/builder.ts | 5 + packages/core/src/tree/merge.ts | 9 +- packages/server/drizzle.config.ts | 9 + packages/server/drizzle/0000_init.sql | 389 +++ .../server/drizzle/meta/0000_snapshot.json | 2488 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 13 + packages/server/package.json | 32 + packages/server/src/app.ts | 59 + packages/server/src/blob/fs.ts | 232 ++ packages/server/src/blob/memory.ts | 107 + packages/server/src/blob/s3.ts | 245 ++ packages/server/src/cache.ts | 56 + packages/server/src/db/d1.ts | 12 + packages/server/src/db/node.ts | 22 + packages/server/src/db/schema.ts | 574 ++++ packages/server/src/jobs.ts | 130 + packages/server/src/lib/gzip.ts | 30 + packages/server/src/lib/lru.ts | 43 + packages/server/src/node/main.ts | 95 + packages/server/src/ports.ts | 103 + packages/server/src/storage/objects.ts | 375 +++ packages/server/src/worker.ts | 81 + packages/server/test/blob.test.ts | 132 + packages/server/test/db.test.ts | 98 + packages/server/test/fake-s3.ts | 136 + packages/server/test/storage.test.ts | 142 + packages/server/tsconfig.json | 17 + packages/server/vitest.config.ts | 7 + pnpm-lock.yaml | 356 ++- 30 files changed, 5972 insertions(+), 31 deletions(-) create mode 100644 packages/server/drizzle.config.ts create mode 100644 packages/server/drizzle/0000_init.sql create mode 100644 packages/server/drizzle/meta/0000_snapshot.json create mode 100644 packages/server/drizzle/meta/_journal.json create mode 100644 packages/server/package.json create mode 100644 packages/server/src/app.ts create mode 100644 packages/server/src/blob/fs.ts create mode 100644 packages/server/src/blob/memory.ts create mode 100644 packages/server/src/blob/s3.ts create mode 100644 packages/server/src/cache.ts create mode 100644 packages/server/src/db/d1.ts create mode 100644 packages/server/src/db/node.ts create mode 100644 packages/server/src/db/schema.ts create mode 100644 packages/server/src/jobs.ts create mode 100644 packages/server/src/lib/gzip.ts create mode 100644 packages/server/src/lib/lru.ts create mode 100644 packages/server/src/node/main.ts create mode 100644 packages/server/src/ports.ts create mode 100644 packages/server/src/storage/objects.ts create mode 100644 packages/server/src/worker.ts create mode 100644 packages/server/test/blob.test.ts create mode 100644 packages/server/test/db.test.ts create mode 100644 packages/server/test/fake-s3.ts create mode 100644 packages/server/test/storage.test.ts create mode 100644 packages/server/tsconfig.json create mode 100644 packages/server/vitest.config.ts diff --git a/packages/core/package.json b/packages/core/package.json index a62ca03..55ec2df 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -12,8 +12,14 @@ "typecheck": "tsc --noEmit", "vectors": "tsx scripts/gen-vectors.ts" }, + "dependencies": { + "@cfworker/json-schema": "^4.1.1" + }, "devDependencies": { + "@hyperjump/json-schema": "^1.17.9", "@types/node": "^25.0.0", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "fast-check": "^4.3.0", "tsx": "^4.19.0", "typescript": "^6.0.0", diff --git a/packages/core/src/tree/builder.ts b/packages/core/src/tree/builder.ts index d1bd963..e133e0d 100644 --- a/packages/core/src/tree/builder.ts +++ b/packages/core/src/tree/builder.ts @@ -37,6 +37,11 @@ export interface TreeSink { * instead of holding a whole leaf's bodies in memory. */ spill?(entries: readonly E[]): void + /** + * Backpressure for sinks that write asynchronously: async callers (mergeTree) + * await it between leaves so pending writes stay bounded. + */ + drain?(): Promise } export interface BuilderOptions { diff --git a/packages/core/src/tree/merge.ts b/packages/core/src/tree/merge.ts index 009f7bb..6864298 100644 --- a/packages/core/src/tree/merge.ts +++ b/packages/core/src/tree/merge.ts @@ -127,6 +127,8 @@ export async function mergeTree( const c = await pending.peek() if (!c || compareUtf8(c.key, oldKey) >= 0) break insert((await nextChange())!) + // A run of inserts inside one base leaf's range can be arbitrarily long. + if (stats.added % 1024 === 0) await sink.drain?.() } const c = await pending.peek() if (c && c.key === oldKey) { @@ -163,6 +165,7 @@ export async function mergeTree( } if (desc.level === 0) { await rewriteLeaf(desc) + await sink.drain?.() return } stats.readNodes++ @@ -182,7 +185,11 @@ export async function mergeTree( stats.readNodes++ await visit(await rootDesc(source, base), true) } - for (let c = await nextChange(); c; c = await nextChange()) insert(c) + let appended = 0 + for (let c = await nextChange(); c; c = await nextChange()) { + insert(c) + if (++appended % 1024 === 0) await sink.drain?.() + } const { root, unresolved } = builder.finish() return { root: root && unresolved ? await resolveRoot(source, root) : root, stats } diff --git a/packages/server/drizzle.config.ts b/packages/server/drizzle.config.ts new file mode 100644 index 0000000..8bc4c9a --- /dev/null +++ b/packages/server/drizzle.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'drizzle-kit' + +// One migration set for both runtimes: libsql applies it with drizzle's +// migrator, D1 with `wrangler d1 migrations apply` (migrations_dir = drizzle/). +export default defineConfig({ + dialect: 'sqlite', + schema: './src/db/schema.ts', + out: './drizzle', +}) diff --git a/packages/server/drizzle/0000_init.sql b/packages/server/drizzle/0000_init.sql new file mode 100644 index 0000000..0fdda7b --- /dev/null +++ b/packages/server/drizzle/0000_init.sql @@ -0,0 +1,389 @@ +CREATE TABLE `account` ( + `id` text PRIMARY KEY NOT NULL, + `account_id` text NOT NULL, + `provider_id` text NOT NULL, + `user_id` text NOT NULL, + `access_token` text, + `refresh_token` text, + `id_token` text, + `access_token_expires_at` integer, + `refresh_token_expires_at` integer, + `scope` text, + `password` text, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `updated_at` integer NOT NULL, + FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `account_user_id_idx` ON `account` (`user_id`);--> statement-breakpoint +CREATE TABLE `apikey` ( + `id` text PRIMARY KEY NOT NULL, + `config_id` text DEFAULT 'default' NOT NULL, + `name` text, + `start` text, + `reference_id` text NOT NULL, + `prefix` text, + `key` text NOT NULL, + `refill_interval` integer, + `refill_amount` integer, + `last_refill_at` integer, + `enabled` integer DEFAULT true, + `rate_limit_enabled` integer DEFAULT true, + `rate_limit_time_window` integer DEFAULT 86400000, + `rate_limit_max` integer DEFAULT 10, + `request_count` integer DEFAULT 0, + `remaining` integer, + `last_request` integer, + `expires_at` integer, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `updated_at` integer NOT NULL, + `permissions` text, + `metadata` text +); +--> statement-breakpoint +CREATE INDEX `apikey_key_idx` ON `apikey` (`key`);--> statement-breakpoint +CREATE INDEX `apikey_reference_id_idx` ON `apikey` (`reference_id`);--> statement-breakpoint +CREATE TABLE `ark_collections` ( + `collection_id` text PRIMARY KEY NOT NULL, + `ark_id` text NOT NULL, + `enabled` integer DEFAULT true NOT NULL, + `custom_url` text, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + FOREIGN KEY (`collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX `ark_collections_ark_id_unique` ON `ark_collections` (`ark_id`);--> statement-breakpoint +CREATE TABLE `ark_record_types` ( + `collection_id` text NOT NULL, + `record_type` text NOT NULL, + `redirect_url_field` text NOT NULL, + PRIMARY KEY(`collection_id`, `record_type`), + FOREIGN KEY (`collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `ark_shoulders` ( + `id` text PRIMARY KEY NOT NULL, + `organization_id` text NOT NULL, + `shoulder` text NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + FOREIGN KEY (`organization_id`) REFERENCES `organization`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX `ark_shoulders_shoulder_unique` ON `ark_shoulders` (`shoulder`);--> statement-breakpoint +CREATE TABLE `collection_webhooks` ( + `id` text PRIMARY KEY NOT NULL, + `collection_id` text NOT NULL, + `url` text NOT NULL, + `bump_filter` text DEFAULT 'all' NOT NULL, + `secret` text NOT NULL, + `enabled` integer DEFAULT true NOT NULL, + `created_by` text, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `updated_at` integer NOT NULL, + `last_delivery_at` integer, + FOREIGN KEY (`collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `collection_webhooks_collection_idx` ON `collection_webhooks` (`collection_id`);--> statement-breakpoint +CREATE TABLE `collections` ( + `id` text PRIMARY KEY NOT NULL, + `organization_id` text NOT NULL, + `slug` text NOT NULL, + `name` text NOT NULL, + `public` integer DEFAULT false NOT NULL, + `head_version_id` text, + `private_salt` text NOT NULL, + `public_files_root` text, + `summary` text, + `ref_events` integer DEFAULT 0 NOT NULL, + `ref_bytes` integer DEFAULT 0 NOT NULL, + `deleted_at` integer, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `updated_at` integer NOT NULL, + FOREIGN KEY (`organization_id`) REFERENCES `organization`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX `collections_org_slug_uq` ON `collections` (`organization_id`,`slug`);--> statement-breakpoint +CREATE INDEX `collections_public_updated_idx` ON `collections` (`public`,`updated_at`);--> statement-breakpoint +CREATE TABLE `denylist` ( + `hash` text PRIMARY KEY NOT NULL, + `kind` text NOT NULL, + `reason` text NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL +); +--> statement-breakpoint +CREATE TABLE `file_uploads` ( + `id` text PRIMARY KEY NOT NULL, + `collection_id` text NOT NULL, + `session_id` text, + `hash` text NOT NULL, + `size` integer NOT NULL, + `mime_type` text NOT NULL, + `storage_key` text NOT NULL, + `multipart_upload_id` text, + `status` text DEFAULT 'pending' NOT NULL, + `error` text, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL +); +--> statement-breakpoint +CREATE INDEX `file_uploads_hash_idx` ON `file_uploads` (`hash`);--> statement-breakpoint +CREATE TABLE `files` ( + `hash` text PRIMARY KEY NOT NULL, + `size` integer NOT NULL, + `mime_type` text NOT NULL, + `storage_key` text NOT NULL, + `verified_at` integer, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL +); +--> statement-breakpoint +CREATE TABLE `forks` ( + `child_collection_id` text PRIMARY KEY NOT NULL, + `parent_collection_id` text NOT NULL, + `parent_seq` integer NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + FOREIGN KEY (`child_collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `instance_settings` ( + `key` text PRIMARY KEY NOT NULL, + `value` text NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +CREATE TABLE `invitation` ( + `id` text PRIMARY KEY NOT NULL, + `organization_id` text NOT NULL, + `email` text NOT NULL, + `role` text, + `status` text DEFAULT 'pending' NOT NULL, + `expires_at` integer NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `inviter_id` text NOT NULL, + FOREIGN KEY (`organization_id`) REFERENCES `organization`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`inviter_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `invitation_organization_id_idx` ON `invitation` (`organization_id`);--> statement-breakpoint +CREATE TABLE `jobs` ( + `id` text PRIMARY KEY NOT NULL, + `type` text NOT NULL, + `payload` text NOT NULL, + `status` text DEFAULT 'queued' NOT NULL, + `attempts` integer DEFAULT 0 NOT NULL, + `run_at` integer NOT NULL, + `locked_until` integer, + `error` text, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL +); +--> statement-breakpoint +CREATE INDEX `jobs_ready_idx` ON `jobs` (`status`,`run_at`);--> statement-breakpoint +CREATE TABLE `legacy_hashes` ( + `legacy_hash` text PRIMARY KEY NOT NULL, + `kind` text NOT NULL, + `hash` text NOT NULL +); +--> statement-breakpoint +CREATE TABLE `member` ( + `id` text PRIMARY KEY NOT NULL, + `organization_id` text NOT NULL, + `user_id` text NOT NULL, + `role` text DEFAULT 'member' NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + FOREIGN KEY (`organization_id`) REFERENCES `organization`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `member_organization_id_idx` ON `member` (`organization_id`);--> statement-breakpoint +CREATE INDEX `member_user_id_idx` ON `member` (`user_id`);--> statement-breakpoint +CREATE TABLE `organization` ( + `id` text PRIMARY KEY NOT NULL, + `name` text NOT NULL, + `slug` text NOT NULL, + `logo` text, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `metadata` text, + `bio` text, + `website` text, + `avatar_url` text, + `ark_naan` text, + `kf_org_id` text, + `is_default` integer DEFAULT false +); +--> statement-breakpoint +CREATE UNIQUE INDEX `organization_slug_unique` ON `organization` (`slug`);--> statement-breakpoint +CREATE TABLE `page_comments` ( + `id` text PRIMARY KEY NOT NULL, + `page` text NOT NULL, + `anchor` text NOT NULL, + `quote` text, + `quote_context` text, + `parent_id` text, + `user_id` text NOT NULL, + `body` text NOT NULL, + `approved_at` integer, + `approved_by` text, + `status` text DEFAULT 'open' NOT NULL, + `resolution_note` text, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `edited_at` integer, + `deleted_at` integer +); +--> statement-breakpoint +CREATE INDEX `page_comments_page_idx` ON `page_comments` (`page`);--> statement-breakpoint +CREATE TABLE `push_runs` ( + `session_id` text NOT NULL, + `seq` integer NOT NULL, + `kind` text NOT NULL, + `object_key` text NOT NULL, + `count` integer NOT NULL, + `first_key` text NOT NULL, + `last_key` text NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + PRIMARY KEY(`session_id`, `seq`), + FOREIGN KEY (`session_id`) REFERENCES `push_sessions`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `push_sessions` ( + `id` text PRIMARY KEY NOT NULL, + `collection_id` text NOT NULL, + `user_id` text NOT NULL, + `kind` text NOT NULL, + `base_version_id` text, + `base_semver` text, + `message` text, + `app_id` text, + `actor_id` text, + `strip_unknown_fields` integer DEFAULT false NOT NULL, + `manifest_expected` integer, + `manifest_received` integer DEFAULT 0 NOT NULL, + `manifest_needed` integer DEFAULT 0 NOT NULL, + `records_received` integer DEFAULT 0 NOT NULL, + `runs` integer DEFAULT 0 NOT NULL, + `status` text DEFAULT 'open' NOT NULL, + `result` text, + `error` text, + `finalize_started_at` integer, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `expires_at` integer NOT NULL, + FOREIGN KEY (`collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `push_sessions_collection_idx` ON `push_sessions` (`collection_id`);--> statement-breakpoint +CREATE INDEX `push_sessions_expires_idx` ON `push_sessions` (`status`,`expires_at`);--> statement-breakpoint +CREATE TABLE `schema_labels` ( + `schema_hash` text NOT NULL, + `label` text NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + PRIMARY KEY(`schema_hash`, `label`) +); +--> statement-breakpoint +CREATE INDEX `schema_labels_label_idx` ON `schema_labels` (`label`);--> statement-breakpoint +CREATE TABLE `schema_usage` ( + `schema_hash` text NOT NULL, + `collection_id` text NOT NULL, + `type_slug` text NOT NULL, + `set` text NOT NULL, + `from_seq` integer NOT NULL, + `to_seq` integer, + PRIMARY KEY(`collection_id`, `type_slug`, `set`, `from_seq`), + FOREIGN KEY (`collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `schema_usage_hash_idx` ON `schema_usage` (`schema_hash`);--> statement-breakpoint +CREATE TABLE `schemas` ( + `hash` text PRIMARY KEY NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL +); +--> statement-breakpoint +CREATE TABLE `session` ( + `id` text PRIMARY KEY NOT NULL, + `expires_at` integer NOT NULL, + `token` text NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `updated_at` integer NOT NULL, + `ip_address` text, + `user_agent` text, + `user_id` text NOT NULL, + `active_organization_id` text, + FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX `session_token_unique` ON `session` (`token`);--> statement-breakpoint +CREATE INDEX `session_user_id_idx` ON `session` (`user_id`);--> statement-breakpoint +CREATE TABLE `user` ( + `id` text PRIMARY KEY NOT NULL, + `name` text NOT NULL, + `email` text NOT NULL, + `email_verified` integer DEFAULT false NOT NULL, + `image` text, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `user_email_unique` ON `user` (`email`);--> statement-breakpoint +CREATE TABLE `verification` ( + `id` text PRIMARY KEY NOT NULL, + `identifier` text NOT NULL, + `value` text NOT NULL, + `expires_at` integer NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +CREATE INDEX `verification_identifier_idx` ON `verification` (`identifier`);--> statement-breakpoint +CREATE TABLE `versions` ( + `id` text PRIMARY KEY NOT NULL, + `collection_id` text NOT NULL, + `seq` integer NOT NULL, + `semver` text NOT NULL, + `major` integer NOT NULL, + `minor` integer NOT NULL, + `patch` integer NOT NULL, + `hash` text NOT NULL, + `legacy_hash` text, + `legacy_public_hash` text, + `base_semver` text, + `message` text, + `pushed_by` text, + `app_id` text, + `actor_id` text, + `signature` text, + `record_count` integer NOT NULL, + `public_record_count` integer NOT NULL, + `file_count` integer NOT NULL, + `total_bytes` integer NOT NULL, + `type_counts` text NOT NULL, + `public_type_counts` text NOT NULL, + `has_private` integer DEFAULT false NOT NULL, + `changes` text, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + FOREIGN KEY (`collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX `versions_collection_seq_uq` ON `versions` (`collection_id`,`seq`);--> statement-breakpoint +CREATE UNIQUE INDEX `versions_collection_semver_uq` ON `versions` (`collection_id`,`semver`);--> statement-breakpoint +CREATE INDEX `versions_hash_idx` ON `versions` (`hash`);--> statement-breakpoint +CREATE INDEX `versions_legacy_hash_idx` ON `versions` (`legacy_hash`);--> statement-breakpoint +CREATE INDEX `versions_legacy_public_hash_idx` ON `versions` (`legacy_public_hash`);--> statement-breakpoint +CREATE TABLE `webhook_deliveries` ( + `id` text PRIMARY KEY NOT NULL, + `webhook_id` text NOT NULL, + `collection_id` text NOT NULL, + `version_id` text, + `semver` text, + `bump_type` text NOT NULL, + `event` text DEFAULT 'version.created' NOT NULL, + `payload` text NOT NULL, + `status` text DEFAULT 'pending' NOT NULL, + `attempts` integer DEFAULT 0 NOT NULL, + `response_code` integer, + `error` text, + `duration_ms` integer, + `next_attempt_at` integer, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + `delivered_at` integer, + FOREIGN KEY (`webhook_id`) REFERENCES `collection_webhooks`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `webhook_deliveries_webhook_idx` ON `webhook_deliveries` (`webhook_id`,`created_at`);--> statement-breakpoint +CREATE INDEX `webhook_deliveries_pending_idx` ON `webhook_deliveries` (`status`,`next_attempt_at`); \ No newline at end of file diff --git a/packages/server/drizzle/meta/0000_snapshot.json b/packages/server/drizzle/meta/0000_snapshot.json new file mode 100644 index 0000000..3dcb588 --- /dev/null +++ b/packages/server/drizzle/meta/0000_snapshot.json @@ -0,0 +1,2488 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "8af0a44f-14a2-4c79-8b3a-0976bbe54d57", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'all'" + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "legacy_hashes": { + "name": "legacy_hashes", + "columns": { + "legacy_hash": { + "name": "legacy_hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "page_comments": { + "name": "page_comments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "anchor": { + "name": "anchor", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "quote": { + "name": "quote", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "quote_context": { + "name": "quote_context", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolution_note": { + "name": "resolution_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "edited_at": { + "name": "edited_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "page_comments_page_idx": { + "name": "page_comments_page_idx", + "columns": ["page"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "manifest_expected": { + "name": "manifest_expected", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest_received": { + "name": "manifest_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "manifest_needed": { + "name": "manifest_needed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "legacy_hash": { + "name": "legacy_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "legacy_public_hash": { + "name": "legacy_public_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_legacy_hash_idx": { + "name": "versions_legacy_hash_idx", + "columns": ["legacy_hash"], + "isUnique": false + }, + "versions_legacy_public_hash_idx": { + "name": "versions_legacy_public_hash_idx", + "columns": ["legacy_public_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json new file mode 100644 index 0000000..b008607 --- /dev/null +++ b/packages/server/drizzle/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1791059262989, + "tag": "0000_init", + "breakpoints": true + } + ] +} diff --git a/packages/server/package.json b/packages/server/package.json new file mode 100644 index 0000000..e7206a3 --- /dev/null +++ b/packages/server/package.json @@ -0,0 +1,32 @@ +{ + "name": "@underlay/server", + "version": "0.0.0", + "private": true, + "description": "Underlay v2 server: one Hono app over blob, SQLite, jobs and cache ports. Runs on Cloudflare Workers and on Node.", + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit", + "db:generate": "drizzle-kit generate", + "dev:node": "tsx watch src/node/main.ts" + }, + "dependencies": { + "@hono/node-server": "^1.19.14", + "@libsql/client": "^0.18.0", + "@underlay/core": "workspace:*", + "aws4fetch": "^1.0.20", + "drizzle-orm": "^0.45.2", + "hono": "^4.12.18" + }, + "devDependencies": { + "@cloudflare/workers-types": "^5.20261003.1", + "@types/node": "^25.0.0", + "drizzle-kit": "^0.31.10", + "tsx": "^4.19.0", + "typescript": "^6.0.0", + "vitest": "^4.1.6" + } +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts new file mode 100644 index 0000000..6766858 --- /dev/null +++ b/packages/server/src/app.ts @@ -0,0 +1,59 @@ +/** + * The Hono app. Runtime-agnostic: each entry (Node, Worker) supplies a function + * that builds the ports and config for a request, and everything below reads + * them from the context. + */ +import { type Context, Hono } from 'hono' + +import type { Ports } from './ports.js' + +export interface AppConfig { + /** Public origin, e.g. https://staging.underlay.org */ + appUrl: string + /** "staging", "next", "production" or "dev": shown in /api/health. */ + deployment: string +} + +export type AppEnv = { + Bindings: Record + Variables: { + ports: Ports + config: AppConfig + } +} + +/** + * Builds a request's ports and config. Runs per request: on Workers, bindings and + * the execution context belong to the invocation. + */ +export type Setup = (c: Context) => { ports: Ports; config: AppConfig } + +export function createApp(setup: Setup) { + const app = new Hono() + + app.use('*', async (c, next) => { + const { ports, config } = setup(c) + c.set('ports', ports) + c.set('config', config) + await next() + }) + + app.get('/api/health', (c) => + c.json({ + ok: true, + version: 2, + deployment: c.var.config.deployment, + time: new Date().toISOString(), + }), + ) + + app.notFound((c) => c.json({ error: 'Not found', statusCode: 404 }, 404)) + app.onError((err, c) => { + console.error('[app]', err) + return c.json({ error: 'Internal error', statusCode: 500 }, 500) + }) + + return app +} + +export type App = ReturnType diff --git a/packages/server/src/blob/fs.ts b/packages/server/src/blob/fs.ts new file mode 100644 index 0000000..7a2c927 --- /dev/null +++ b/packages/server/src/blob/fs.ts @@ -0,0 +1,232 @@ +/** + * Filesystem blob store for development and small self-hosted Node setups. + * + * Presigned URLs point at the app itself (`/_blob/`), signed with HMAC; the + * Node entry serves them through `serveSignedBlob`. Bytes still never pass + * through the API routes, and the same client flow (presigned PUT, then GET by + * redirect) works as on R2. + */ +import { createHmac, timingSafeEqual } from 'node:crypto' +import { createReadStream } from 'node:fs' +import { mkdir, open, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join, relative, resolve, sep } from 'node:path' +import { Readable } from 'node:stream' + +import type { + BlobHead, + BlobObject, + BlobStore, + PresignGetOptions, + PresignPutOptions, + PutOptions, +} from '../ports.js' + +export interface FsBlobConfig { + root: string + /** Public base URL of the app, for presigned URLs. */ + publicUrl: string + /** HMAC secret for presigned URLs. */ + secret: string +} + +export class FsBlobStore implements BlobStore { + readonly #root: string + constructor(readonly cfg: FsBlobConfig) { + this.#root = resolve(cfg.root) + } + + #path(key: string): string { + const p = resolve(this.#root, key) + if (!p.startsWith(this.#root + sep)) throw new Error(`Bad blob key: ${key}`) + return p + } + + async get(key: string, range?: { offset: number; length?: number }): Promise { + const path = this.#path(key) + const st = await stat(path).catch(() => null) + if (!st?.isFile()) return null + const start = range?.offset ?? 0 + const end = + range?.length === undefined ? st.size - 1 : Math.min(st.size - 1, start + range.length - 1) + const size = Math.max(0, end - start + 1) + const head = { + size, + etag: `"${st.mtimeMs}-${st.size}"`, + contentType: await this.#contentType(key), + } + const read = async () => { + if (size === 0) return new Uint8Array() + const fh = await open(path) + try { + const buf = new Uint8Array(size) + await fh.read(buf, 0, size, start) + return buf + } finally { + await fh.close() + } + } + return { + ...head, + get body() { + return Readable.toWeb(createReadStream(path, { start, end })) as ReadableStream + }, + bytes: read, + text: async () => new TextDecoder().decode(await read()), + } + } + + async #contentType(key: string): Promise { + return readFile(this.#path(key) + '.__type', 'utf8').catch(() => null) + } + + async head(key: string): Promise { + const st = await stat(this.#path(key)).catch(() => null) + if (!st?.isFile()) return null + return { + size: st.size, + etag: `"${st.mtimeMs}-${st.size}"`, + contentType: await this.#contentType(key), + } + } + + async put(key: string, body: Uint8Array | string, opts: PutOptions = {}): Promise { + const path = this.#path(key) + if (opts.ifAbsent && (await stat(path).catch(() => null))) return + await mkdir(dirname(path), { recursive: true }) + // Write then rename, so a reader never sees a partial object. + const tmp = `${path}.${crypto.randomUUID()}.tmp` + await writeFile(tmp, body) + await rename(tmp, path) + if (opts.contentType) await writeFile(path + '.__type', opts.contentType) + } + + async delete(key: string): Promise { + await rm(this.#path(key), { force: true }) + await rm(this.#path(key) + '.__type', { force: true }) + } + + async list(prefix: string, cursor?: string): Promise<{ keys: string[]; cursor?: string }> { + const out: string[] = [] + const walk = async (dir: string) => { + const entries = await readdir(dir, { withFileTypes: true }).catch(() => []) + for (const e of entries) { + const p = join(dir, e.name) + if (e.isDirectory()) await walk(p) + else if (!e.name.endsWith('.__type') && !e.name.endsWith('.tmp')) { + const key = relative(this.#root, p).split(sep).join('/') + if (key.startsWith(prefix) && (!cursor || key > cursor)) out.push(key) + } + } + } + await walk(this.#root) + out.sort() + const page = out.slice(0, 1000) + return page.length === 1000 ? { keys: page, cursor: page[page.length - 1]! } : { keys: page } + } + + sign(method: string, key: string, exp: number, extra = ''): string { + return createHmac('sha256', this.cfg.secret) + .update(`${method}\n${key}\n${exp}\n${extra}`) + .digest('hex') + } + + #signedUrl(method: string, key: string, expiresIn: number, params: Record = {}) { + const exp = Math.floor(Date.now() / 1000) + expiresIn + const extra = new URLSearchParams(params).toString() + const u = new URL( + `${this.cfg.publicUrl.replace(/\/$/, '')}/_blob/${key.split('/').map(encodeURIComponent).join('/')}`, + ) + for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v) + u.searchParams.set('m', method) + u.searchParams.set('exp', String(exp)) + u.searchParams.set('sig', this.sign(method, key, exp, extra)) + return u.toString() + } + + async presignGet(key: string, opts: PresignGetOptions): Promise { + const params: Record = {} + if (opts.disposition) params.disposition = opts.disposition + if (opts.contentType) params.type = opts.contentType + return this.#signedUrl('GET', key, opts.expiresIn, params) + } + + async presignPut(key: string, opts: PresignPutOptions): Promise { + return this.#signedUrl('PUT', key, opts.expiresIn) + } + + async createMultipart(key: string): Promise { + const id = crypto.randomUUID() + await mkdir(this.#path(`_multipart/${id}`), { recursive: true }) + await writeFile(this.#path(`_multipart/${id}/.key`), key) + return id + } + + async presignPart( + key: string, + uploadId: string, + partNumber: number, + expiresIn: number, + ): Promise { + return this.#signedUrl('PUT', `_multipart/${uploadId}/${partNumber}`, expiresIn) + } + + async completeMultipart( + key: string, + uploadId: string, + parts: { partNumber: number; etag: string }[], + ) { + const path = this.#path(key) + await mkdir(dirname(path), { recursive: true }) + const tmp = `${path}.${crypto.randomUUID()}.tmp` + const fh = await open(tmp, 'w') + try { + for (const p of parts) + await fh.write(await readFile(this.#path(`_multipart/${uploadId}/${p.partNumber}`))) + } finally { + await fh.close() + } + await rename(tmp, path) + await rm(this.#path(`_multipart/${uploadId}`), { recursive: true, force: true }) + } + + async abortMultipart(_key: string, uploadId: string): Promise { + await rm(this.#path(`_multipart/${uploadId}`), { recursive: true, force: true }) + } + + /** Verify a presigned `/_blob/…` request. Returns the key, or null if the signature is bad or expired. */ + verify(method: string, url: URL): string | null { + const key = decodeURIComponent(url.pathname.replace(/^\/_blob\//, '')) + const exp = Number(url.searchParams.get('exp')) + const sig = url.searchParams.get('sig') ?? '' + if (url.searchParams.get('m') !== method || !Number.isFinite(exp) || exp < Date.now() / 1000) + return null + const params: Record = {} + for (const p of ['disposition', 'type']) { + const v = url.searchParams.get(p) + if (v !== null) params[p] = v + } + const want = this.sign(method, key, exp, new URLSearchParams(params).toString()) + const a = Buffer.from(sig, 'hex') + const b = Buffer.from(want, 'hex') + return a.length === b.length && timingSafeEqual(a, b) ? key : null + } +} + +/** Serve a presigned `/_blob/…` GET or PUT for an FsBlobStore (Node entry only). */ +export async function serveSignedBlob(store: FsBlobStore, req: Request): Promise { + const url = new URL(req.url) + const key = store.verify(req.method, url) + if (!key) return new Response('Forbidden', { status: 403 }) + if (req.method === 'PUT') { + await store.put(key, new Uint8Array(await req.arrayBuffer())) + return new Response(null, { status: 200, headers: { etag: `"${key.split('/').pop()}"` } }) + } + const obj = await store.get(key) + if (!obj) return new Response('Not found', { status: 404 }) + const headers: Record = { 'content-length': String(obj.size) } + const type = url.searchParams.get('type') ?? obj.contentType + if (type) headers['content-type'] = type + const disposition = url.searchParams.get('disposition') + if (disposition) headers['content-disposition'] = disposition + return new Response(obj.body, { headers }) +} diff --git a/packages/server/src/blob/memory.ts b/packages/server/src/blob/memory.ts new file mode 100644 index 0000000..195dbac --- /dev/null +++ b/packages/server/src/blob/memory.ts @@ -0,0 +1,107 @@ +import type { BlobHead, BlobObject, BlobStore, PutOptions } from '../ports.js' + +const enc = new TextEncoder() + +export function blobObject(bytes: Uint8Array, head: Omit): BlobObject { + return { + ...head, + size: bytes.byteLength, + body: new Blob([bytes as Uint8Array]).stream(), + bytes: async () => bytes, + text: async () => new TextDecoder().decode(bytes), + } +} + +/** In-memory blob store for tests. Presigned URLs are `memory://` placeholders. */ +export class MemoryBlobStore implements BlobStore { + readonly objects = new Map() + readonly multipart = new Map>() + puts = 0 + gets = 0 + + async get(key: string, range?: { offset: number; length?: number }) { + this.gets++ + const o = this.objects.get(key) + if (!o) return null + const bytes = range + ? o.bytes.subarray( + range.offset, + range.length === undefined ? undefined : range.offset + range.length, + ) + : o.bytes + return blobObject(bytes, { etag: `"${key}"`, contentType: o.contentType }) + } + + async head(key: string) { + const o = this.objects.get(key) + return o ? { size: o.bytes.byteLength, etag: `"${key}"`, contentType: o.contentType } : null + } + + async put(key: string, body: Uint8Array | string, opts: PutOptions = {}) { + if (opts.ifAbsent && this.objects.has(key)) return + this.puts++ + this.objects.set(key, { + bytes: typeof body === 'string' ? enc.encode(body) : body.slice(), + contentType: opts.contentType ?? null, + }) + } + + async delete(key: string) { + this.objects.delete(key) + } + + async list(prefix: string, cursor?: string) { + const keys = [...this.objects.keys()] + .filter((k) => k.startsWith(prefix) && (!cursor || k > cursor)) + .sort() + const page = keys.slice(0, 1000) + return page.length === 1000 ? { keys: page, cursor: page[page.length - 1]! } : { keys: page } + } + + async presignGet(key: string) { + return `memory://get/${key}` + } + + async presignPut(key: string) { + return `memory://put/${key}` + } + + async createMultipart(key: string) { + const id = crypto.randomUUID() + this.multipart.set(`${key}#${id}`, new Map()) + return id + } + + async presignPart(key: string, uploadId: string, partNumber: number) { + return `memory://part/${key}?uploadId=${uploadId}&partNumber=${partNumber}` + } + + /** Test helper standing in for a client PUT to a presigned part URL. */ + uploadPart(key: string, uploadId: string, partNumber: number, bytes: Uint8Array): string { + this.multipart.get(`${key}#${uploadId}`)!.set(partNumber, bytes) + return `"part-${partNumber}"` + } + + async completeMultipart( + key: string, + uploadId: string, + parts: { partNumber: number; etag: string }[], + ) { + const stored = this.multipart.get(`${key}#${uploadId}`) + if (!stored) throw new Error('No such upload') + const chunks = parts.map((p) => stored.get(p.partNumber)!) + const total = chunks.reduce((n, c) => n + c.byteLength, 0) + const out = new Uint8Array(total) + let off = 0 + for (const c of chunks) { + out.set(c, off) + off += c.byteLength + } + this.objects.set(key, { bytes: out, contentType: null }) + this.multipart.delete(`${key}#${uploadId}`) + } + + async abortMultipart(key: string, uploadId: string) { + this.multipart.delete(`${key}#${uploadId}`) + } +} diff --git a/packages/server/src/blob/s3.ts b/packages/server/src/blob/s3.ts new file mode 100644 index 0000000..4eba042 --- /dev/null +++ b/packages/server/src/blob/s3.ts @@ -0,0 +1,245 @@ +/** + * BlobStore over the S3 API, signed with aws4fetch. Works against R2, S3 and + * MinIO, from Workers and Node alike, and can presign (which the R2 binding + * can't). Path-style URLs: `${endpoint}/${bucket}/${key}`. + */ +import { AwsClient } from 'aws4fetch' + +import type { + BlobHead, + BlobObject, + BlobStore, + PresignGetOptions, + PresignPutOptions, + PutOptions, +} from '../ports.js' + +export interface S3Config { + endpoint: string + bucket: string + accessKeyId: string + secretAccessKey: string + /** "auto" for R2. */ + region?: string +} + +export class S3Error extends Error { + constructor( + readonly status: number, + message: string, + ) { + super(message) + this.name = 'S3Error' + } +} + +/** Keys are path segments; encode each one but keep the slashes. */ +const encodeKey = (key: string) => key.split('/').map(encodeURIComponent).join('/') + +const xmlValues = (xml: string, tag: string) => + [...xml.matchAll(new RegExp(`<${tag}>([\\s\\S]*?)`, 'g'))].map((m) => + m[1]! + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'"), + ) + +const escapeXml = (s: string) => + s.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"') + +export class S3BlobStore implements BlobStore { + readonly #aws: AwsClient + readonly #base: string + + constructor(cfg: S3Config) { + this.#aws = new AwsClient({ + accessKeyId: cfg.accessKeyId, + secretAccessKey: cfg.secretAccessKey, + service: 's3', + region: cfg.region ?? 'auto', + retries: 3, + }) + this.#base = `${cfg.endpoint.replace(/\/$/, '')}/${cfg.bucket}` + } + + #url(key: string, query?: Record) { + const u = new URL(`${this.#base}/${encodeKey(key)}`) + for (const [k, v] of Object.entries(query ?? {})) u.searchParams.set(k, v) + return u.toString() + } + + async #fail(res: Response, what: string): Promise { + const body = await res.text().catch(() => '') + throw new S3Error( + res.status, + `S3 ${what}: ${res.status} ${xmlValues(body, 'Code')[0] ?? body.slice(0, 200)}`, + ) + } + + #head(res: Response): BlobHead { + return { + size: Number(res.headers.get('content-length') ?? 0), + etag: res.headers.get('etag') ?? '', + contentType: res.headers.get('content-type'), + } + } + + async get(key: string, range?: { offset: number; length?: number }): Promise { + const headers: Record = {} + if (range) { + headers.range = + range.length === undefined + ? `bytes=${range.offset}-` + : `bytes=${range.offset}-${range.offset + range.length - 1}` + } + const res = await this.#aws.fetch(this.#url(key), { headers }) + if (res.status === 404) { + await res.body?.cancel() + return null + } + if (!res.ok) return this.#fail(res, `GET ${key}`) + const head = this.#head(res) + let used = false + const take = () => { + if (used) throw new Error('Blob body already consumed') + used = true + return res + } + return { + ...head, + get body() { + return take().body! + }, + bytes: async () => new Uint8Array(await take().arrayBuffer()), + text: async () => take().text(), + } + } + + async head(key: string): Promise { + const res = await this.#aws.fetch(this.#url(key), { method: 'HEAD' }) + if (res.status === 404) return null + if (!res.ok) return this.#fail(res, `HEAD ${key}`) + return this.#head(res) + } + + async put(key: string, body: Uint8Array | string, opts: PutOptions = {}): Promise { + const headers: Record = {} + if (opts.contentType) headers['content-type'] = opts.contentType + if (opts.ifAbsent) headers['if-none-match'] = '*' + const res = await this.#aws.fetch(this.#url(key), { + method: 'PUT', + headers, + body: body as BodyInit, + }) + // 412: the key exists and ifAbsent was asked for. Keys are immutable, so that's success. + if (res.status === 412 && opts.ifAbsent) { + await res.body?.cancel() + return + } + if (!res.ok) return this.#fail(res, `PUT ${key}`) + await res.body?.cancel() + } + + async delete(key: string): Promise { + const res = await this.#aws.fetch(this.#url(key), { method: 'DELETE' }) + if (!res.ok && res.status !== 404) return this.#fail(res, `DELETE ${key}`) + await res.body?.cancel() + } + + async list(prefix: string, cursor?: string): Promise<{ keys: string[]; cursor?: string }> { + const u = new URL(this.#base) + u.searchParams.set('list-type', '2') + u.searchParams.set('prefix', prefix) + if (cursor) u.searchParams.set('continuation-token', cursor) + const res = await this.#aws.fetch(u.toString()) + if (!res.ok) return this.#fail(res, `LIST ${prefix}`) + const xml = await res.text() + const next = xmlValues(xml, 'NextContinuationToken')[0] + const keys = xmlValues(xml, 'Key') + return next ? { keys, cursor: next } : { keys } + } + + async #presign(url: string, method: string, expiresIn: number, headers?: Record) { + const u = new URL(url) + u.searchParams.set('X-Amz-Expires', String(expiresIn)) + const signed = await this.#aws.sign(u.toString(), { + method, + ...(headers ? { headers } : {}), + aws: { signQuery: true }, + }) + return signed.url + } + + presignGet(key: string, opts: PresignGetOptions): Promise { + const q: Record = {} + if (opts.disposition) q['response-content-disposition'] = opts.disposition + if (opts.contentType) q['response-content-type'] = opts.contentType + return this.#presign(this.#url(key, q), 'GET', opts.expiresIn) + } + + presignPut(key: string, opts: PresignPutOptions): Promise { + return this.#presign( + this.#url(key), + 'PUT', + opts.expiresIn, + opts.contentType ? { 'content-type': opts.contentType } : undefined, + ) + } + + async createMultipart(key: string, contentType?: string): Promise { + const res = await this.#aws.fetch(`${this.#url(key)}?uploads`, { + method: 'POST', + headers: contentType ? { 'content-type': contentType } : {}, + }) + if (!res.ok) return this.#fail(res, `CreateMultipartUpload ${key}`) + const id = xmlValues(await res.text(), 'UploadId')[0] + if (!id) throw new S3Error(500, 'CreateMultipartUpload: no UploadId') + return id + } + + presignPart( + key: string, + uploadId: string, + partNumber: number, + expiresIn: number, + ): Promise { + return this.#presign( + this.#url(key, { partNumber: String(partNumber), uploadId }), + 'PUT', + expiresIn, + ) + } + + async completeMultipart( + key: string, + uploadId: string, + parts: { partNumber: number; etag: string }[], + ): Promise { + const xml = + '' + + parts + .map( + (p) => + `${p.partNumber}${escapeXml(p.etag)}`, + ) + .join('') + + '' + const res = await this.#aws.fetch(this.#url(key, { uploadId }), { method: 'POST', body: xml }) + const text = await res.text() + // S3 can answer 200 with an body for a failed completion. + if (!res.ok || text.includes('')) { + throw new S3Error( + res.ok ? 500 : res.status, + `CompleteMultipartUpload ${key}: ${text.slice(0, 200)}`, + ) + } + } + + async abortMultipart(key: string, uploadId: string): Promise { + const res = await this.#aws.fetch(this.#url(key, { uploadId }), { method: 'DELETE' }) + if (!res.ok && res.status !== 404) return this.#fail(res, `AbortMultipartUpload ${key}`) + await res.body?.cancel() + } +} diff --git a/packages/server/src/cache.ts b/packages/server/src/cache.ts new file mode 100644 index 0000000..c370693 --- /dev/null +++ b/packages/server/src/cache.ts @@ -0,0 +1,56 @@ +import { Lru } from './lib/lru.js' +import type { Cache } from './ports.js' + +/** Node: an in-process LRU (the isolate LRU in Objects sits in front of it). */ +export class MemoryCache implements Cache { + readonly #lru: Lru + constructor(budgetBytes = 256 * 1024 * 1024) { + this.#lru = new Lru(budgetBytes, (v) => v.byteLength) + } + async get(key: string) { + return this.#lru.get(key) ?? null + } + async put(key: string, value: Uint8Array) { + this.#lru.set(key, value) + } +} + +/** A cache that holds nothing (tests that count blob reads). */ +export const noCache: Cache = { + get: async () => null, + put: async () => {}, +} + +interface CfCacheStorage { + default: { + match(req: Request): Promise + put(req: Request, res: Response): Promise + } +} + +/** + * Workers: the per-colo Cache API. Keys are synthetic URLs; everything cached is + * immutable and keyed by content, so it's cached for a year. + */ +export class CfCache implements Cache { + constructor( + readonly caches: CfCacheStorage, + readonly namespace: string, + ) {} + #req(key: string) { + return new Request(`https://cache.underlay.internal/${this.namespace}/${key}`) + } + async get(key: string) { + const res = await this.caches.default.match(this.#req(key)) + return res ? new Uint8Array(await res.arrayBuffer()) : null + } + async put(key: string, value: Uint8Array, opts?: { ttlSeconds?: number }) { + const ttl = opts?.ttlSeconds ?? 31_536_000 + await this.caches.default.put( + this.#req(key), + new Response(value as Uint8Array, { + headers: { 'cache-control': `public, max-age=${ttl}, immutable` }, + }), + ) + } +} diff --git a/packages/server/src/db/d1.ts b/packages/server/src/db/d1.ts new file mode 100644 index 0000000..9a98876 --- /dev/null +++ b/packages/server/src/db/d1.ts @@ -0,0 +1,12 @@ +/** SQLite on Cloudflare: D1. Migrations are applied by wrangler, not at runtime. */ +import type { D1Database } from '@cloudflare/workers-types' +import { drizzle } from 'drizzle-orm/d1' + +import type { Db } from '../ports.js' +import * as schema from './schema.js' + +export function openD1(binding: D1Database): Db { + // The D1 and libsql drivers build the same queries and both support batch(); + // the app is typed against one of them. + return drizzle(binding, { schema }) as unknown as Db +} diff --git a/packages/server/src/db/node.ts b/packages/server/src/db/node.ts new file mode 100644 index 0000000..6b84ebd --- /dev/null +++ b/packages/server/src/db/node.ts @@ -0,0 +1,22 @@ +/** SQLite on Node, through libsql: async with atomic batches, the same shape as D1. */ +import { fileURLToPath } from 'node:url' + +import { createClient } from '@libsql/client' +import { drizzle } from 'drizzle-orm/libsql' +import { migrate } from 'drizzle-orm/libsql/migrator' + +import type { Db } from '../ports.js' +import * as schema from './schema.js' + +const migrationsFolder = fileURLToPath(new URL('../../drizzle', import.meta.url)) + +/** Open (and migrate) a database. `url` is `file:path/to/db.sqlite`, or `:memory:`. */ +export async function openNodeDb(url: string): Promise { + const client = createClient({ url }) + if (url !== ':memory:') await client.execute('PRAGMA journal_mode = WAL') + await client.execute('PRAGMA foreign_keys = ON') + await client.execute('PRAGMA busy_timeout = 5000') + const db = drizzle(client, { schema }) + await migrate(db, { migrationsFolder }) + return db +} diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts new file mode 100644 index 0000000..2ec49db --- /dev/null +++ b/packages/server/src/db/schema.ts @@ -0,0 +1,574 @@ +/** + * Mutable state, in SQLite (D1 on Cloudflare, libsql on Node). + * + * Rule: rows are small and bounded. Anything whose size the user controls — + * version roots, metadata, READMEs, schemas, file lists — lives in the blob store + * and is referenced by hash. Anything that grows with data (records, version + * membership, provenance) lives in the blob store too. See edge-redesign.md, + * "SQLite (mutable state)". + * + * D1 has no interactive transactions, only atomic batches, so every + * read-modify-write is written as a batch whose conditions are in SQL (see + * publishVersion in ../versions/publish.ts). + */ +import { sql } from 'drizzle-orm' +import { index, integer, primaryKey, sqliteTable, text, uniqueIndex } from 'drizzle-orm/sqlite-core' + +const id = () => + text('id') + .primaryKey() + .$defaultFn(() => crypto.randomUUID()) +const createdAt = () => + integer('created_at', { mode: 'timestamp_ms' }) + .notNull() + .default(sql`(unixepoch('subsec') * 1000)`) +const ts = (name: string) => integer(name, { mode: 'timestamp_ms' }) +const bool = (name: string) => integer(name, { mode: 'boolean' }) +const json = (name: string) => text(name, { mode: 'json' }).$type() + +// --- better-auth (same fields as v1, so sessions and keys port across) ------- + +export const user = sqliteTable('user', { + id: id(), + name: text('name').notNull(), + email: text('email').notNull().unique(), + emailVerified: bool('email_verified').notNull().default(false), + image: text('image'), + createdAt: createdAt(), + updatedAt: ts('updated_at') + .notNull() + .$defaultFn(() => new Date()), +}) + +export const session = sqliteTable( + 'session', + { + id: id(), + expiresAt: ts('expires_at').notNull(), + token: text('token').notNull().unique(), + createdAt: createdAt(), + updatedAt: ts('updated_at') + .notNull() + .$defaultFn(() => new Date()), + ipAddress: text('ip_address'), + userAgent: text('user_agent'), + userId: text('user_id') + .notNull() + .references(() => user.id, { onDelete: 'cascade' }), + activeOrganizationId: text('active_organization_id'), + }, + (t) => [index('session_user_id_idx').on(t.userId)], +) + +export const account = sqliteTable( + 'account', + { + id: id(), + accountId: text('account_id').notNull(), + providerId: text('provider_id').notNull(), + userId: text('user_id') + .notNull() + .references(() => user.id, { onDelete: 'cascade' }), + accessToken: text('access_token'), + refreshToken: text('refresh_token'), + idToken: text('id_token'), + accessTokenExpiresAt: ts('access_token_expires_at'), + refreshTokenExpiresAt: ts('refresh_token_expires_at'), + scope: text('scope'), + password: text('password'), + createdAt: createdAt(), + updatedAt: ts('updated_at') + .notNull() + .$defaultFn(() => new Date()), + }, + (t) => [index('account_user_id_idx').on(t.userId)], +) + +export const verification = sqliteTable( + 'verification', + { + id: id(), + identifier: text('identifier').notNull(), + value: text('value').notNull(), + expiresAt: ts('expires_at').notNull(), + createdAt: createdAt(), + updatedAt: ts('updated_at') + .notNull() + .$defaultFn(() => new Date()), + }, + (t) => [index('verification_identifier_idx').on(t.identifier)], +) + +export const organization = sqliteTable('organization', { + id: id(), + name: text('name').notNull(), + slug: text('slug').notNull().unique(), + logo: text('logo'), + createdAt: createdAt(), + metadata: text('metadata'), + bio: text('bio'), + website: text('website'), + avatarUrl: text('avatar_url'), + arkNaan: text('ark_naan'), + kfOrgId: text('kf_org_id'), + isDefault: bool('is_default').default(false), +}) + +export const member = sqliteTable( + 'member', + { + id: id(), + organizationId: text('organization_id') + .notNull() + .references(() => organization.id, { onDelete: 'cascade' }), + userId: text('user_id') + .notNull() + .references(() => user.id, { onDelete: 'cascade' }), + role: text('role').notNull().default('member'), + createdAt: createdAt(), + }, + (t) => [ + index('member_organization_id_idx').on(t.organizationId), + index('member_user_id_idx').on(t.userId), + ], +) + +export const invitation = sqliteTable( + 'invitation', + { + id: id(), + organizationId: text('organization_id') + .notNull() + .references(() => organization.id, { onDelete: 'cascade' }), + email: text('email').notNull(), + role: text('role'), + status: text('status').notNull().default('pending'), + expiresAt: ts('expires_at').notNull(), + createdAt: createdAt(), + inviterId: text('inviter_id') + .notNull() + .references(() => user.id, { onDelete: 'cascade' }), + }, + (t) => [index('invitation_organization_id_idx').on(t.organizationId)], +) + +export const apikey = sqliteTable( + 'apikey', + { + id: id(), + configId: text('config_id').notNull().default('default'), + name: text('name'), + start: text('start'), + referenceId: text('reference_id').notNull(), + prefix: text('prefix'), + key: text('key').notNull(), + refillInterval: integer('refill_interval'), + refillAmount: integer('refill_amount'), + lastRefillAt: ts('last_refill_at'), + enabled: bool('enabled').default(true), + rateLimitEnabled: bool('rate_limit_enabled').default(true), + rateLimitTimeWindow: integer('rate_limit_time_window').default(86_400_000), + rateLimitMax: integer('rate_limit_max').default(10), + requestCount: integer('request_count').default(0), + remaining: integer('remaining'), + lastRequest: ts('last_request'), + expiresAt: ts('expires_at'), + createdAt: createdAt(), + updatedAt: ts('updated_at') + .notNull() + .$defaultFn(() => new Date()), + permissions: text('permissions'), + metadata: text('metadata'), + }, + (t) => [index('apikey_key_idx').on(t.key), index('apikey_reference_id_idx').on(t.referenceId)], +) + +// --- Collections and versions ------------------------------------------------- + +/** What lists and explore show without reading a root: bounded, refreshed at publish. */ +export interface CollectionSummary { + title?: string + description?: string + tags?: string[] + license?: string +} + +export const collections = sqliteTable( + 'collections', + { + id: id(), + organizationId: text('organization_id') + .notNull() + .references(() => organization.id, { onDelete: 'cascade' }), + slug: text('slug').notNull(), + name: text('name').notNull(), + public: bool('public').notNull().default(false), + /** The head: latest published version. Advanced only by compare-and-swap. */ + headVersionId: text('head_version_id'), + /** Per-collection salt for the private-set commitment (32 bytes hex). */ + privateSalt: text('private_salt').notNull(), + /** Cumulative public file tree over every published version (file access checks). */ + publicFilesRoot: text('public_files_root'), + summary: json('summary'), + /** Billing counters for the reference log (decision 18); rebuildable from version diffs. */ + refEvents: integer('ref_events').notNull().default(0), + refBytes: integer('ref_bytes').notNull().default(0), + deletedAt: ts('deleted_at'), + createdAt: createdAt(), + updatedAt: ts('updated_at') + .notNull() + .$defaultFn(() => new Date()), + }, + (t) => [ + uniqueIndex('collections_org_slug_uq').on(t.organizationId, t.slug), + index('collections_public_updated_idx').on(t.public, t.updatedAt), + ], +) + +export const versions = sqliteTable( + 'versions', + { + id: id(), + collectionId: text('collection_id') + .notNull() + .references(() => collections.id, { onDelete: 'cascade' }), + /** 1, 2, 3… per collection: the order of publication. */ + seq: integer('seq').notNull(), + semver: text('semver').notNull(), + major: integer('major').notNull(), + minor: integer('minor').notNull(), + patch: integer('patch').notNull(), + /** `ulv2:`; the root document is roots/.json in the blob store. */ + hash: text('hash').notNull(), + /** Format 1 hashes of migrated versions (`private:…`, `public:…`). */ + legacyHash: text('legacy_hash'), + legacyPublicHash: text('legacy_public_hash'), + baseSemver: text('base_semver'), + message: text('message'), + pushedBy: text('pushed_by'), + appId: text('app_id'), + actorId: text('actor_id'), + signature: text('signature'), + /** Cached from the root: totals over both sets, and per type. */ + recordCount: integer('record_count').notNull(), + publicRecordCount: integer('public_record_count').notNull(), + fileCount: integer('file_count').notNull(), + totalBytes: integer('total_bytes').notNull(), + typeCounts: json>('type_counts').notNull(), + publicTypeCounts: json>('public_type_counts').notNull(), + hasPrivate: bool('has_private').notNull().default(false), + /** Change counts against the previous version (drive semver and webhooks). */ + changes: json<{ added: number; removed: number; updated: number }>('changes'), + createdAt: createdAt(), + }, + (t) => [ + uniqueIndex('versions_collection_seq_uq').on(t.collectionId, t.seq), + uniqueIndex('versions_collection_semver_uq').on(t.collectionId, t.semver), + index('versions_hash_idx').on(t.hash), + index('versions_legacy_hash_idx').on(t.legacyHash), + index('versions_legacy_public_hash_idx').on(t.legacyPublicHash), + ], +) + +export const forks = sqliteTable('forks', { + childCollectionId: text('child_collection_id') + .primaryKey() + .references(() => collections.id, { onDelete: 'cascade' }), + parentCollectionId: text('parent_collection_id').notNull(), + parentSeq: integer('parent_seq').notNull(), + createdAt: createdAt(), +}) + +// --- Schemas --------------------------------------------------------------------- + +/** Known schemas. The body is schemas/.json in the blob store. */ +export const schemas = sqliteTable('schemas', { + hash: text('hash').primaryKey(), + createdAt: createdAt(), +}) + +export const schemaLabels = sqliteTable( + 'schema_labels', + { + schemaHash: text('schema_hash').notNull(), + label: text('label').notNull(), + createdAt: createdAt(), + }, + (t) => [ + primaryKey({ columns: [t.schemaHash, t.label] }), + index('schema_labels_label_idx').on(t.label), + ], +) + +/** + * Where each schema is used: one row per run of versions in which a collection's + * type uses the schema in a set. Opened when a type starts using a schema, closed + * (`toSeq`) when it stops, so writes are O(changes). Rebuildable from the roots. + */ +export const schemaUsage = sqliteTable( + 'schema_usage', + { + schemaHash: text('schema_hash').notNull(), + collectionId: text('collection_id') + .notNull() + .references(() => collections.id, { onDelete: 'cascade' }), + typeSlug: text('type_slug').notNull(), + set: text('set', { enum: ['public', 'private'] }).notNull(), + fromSeq: integer('from_seq').notNull(), + /** First seq that no longer uses it; null while current. */ + toSeq: integer('to_seq'), + }, + (t) => [ + primaryKey({ columns: [t.collectionId, t.typeSlug, t.set, t.fromSeq] }), + index('schema_usage_hash_idx').on(t.schemaHash), + ], +) + +/** Format 1 → format 2 aliases for records and schemas re-hashed by JCS. */ +export const legacyHashes = sqliteTable('legacy_hashes', { + legacyHash: text('legacy_hash').primaryKey(), + kind: text('kind', { enum: ['record', 'schema'] }).notNull(), + hash: text('hash').notNull(), +}) + +// --- Files ------------------------------------------------------------------------- + +export const files = sqliteTable('files', { + hash: text('hash').primaryKey(), + size: integer('size').notNull(), + mimeType: text('mime_type').notNull(), + storageKey: text('storage_key').notNull(), + verifiedAt: ts('verified_at'), + createdAt: createdAt(), +}) + +/** Pending direct uploads: verified by a job before the file row exists. */ +export const fileUploads = sqliteTable( + 'file_uploads', + { + id: id(), + collectionId: text('collection_id').notNull(), + sessionId: text('session_id'), + hash: text('hash').notNull(), + size: integer('size').notNull(), + mimeType: text('mime_type').notNull(), + storageKey: text('storage_key').notNull(), + multipartUploadId: text('multipart_upload_id'), + status: text('status', { enum: ['pending', 'verifying', 'verified', 'failed'] }) + .notNull() + .default('pending'), + error: text('error'), + createdAt: createdAt(), + }, + (t) => [index('file_uploads_hash_idx').on(t.hash)], +) + +/** Hashes that are never served (abuse). Checked on file redirects and record reads. */ +export const denylist = sqliteTable('denylist', { + hash: text('hash').primaryKey(), + kind: text('kind', { enum: ['file', 'record'] }).notNull(), + reason: text('reason').notNull(), + createdAt: createdAt(), +}) + +// --- Push sessions ------------------------------------------------------------------- + +export type SessionStatus = 'open' | 'committing' | 'committed' | 'failed' | 'expired' + +/** + * A push in progress. Session inputs whose size the user controls (schemas, + * metadata, the declared file list) are in the blob store under sessions//; + * runs of uploaded manifest entries and records are sessions//runs/…. + */ +export const pushSessions = sqliteTable( + 'push_sessions', + { + id: id(), + collectionId: text('collection_id') + .notNull() + .references(() => collections.id, { onDelete: 'cascade' }), + userId: text('user_id').notNull(), + /** negotiate: full-snapshot compatibility API; delta: upserts and deletes against a base. */ + kind: text('kind', { enum: ['negotiate', 'delta'] }).notNull(), + baseVersionId: text('base_version_id'), + baseSemver: text('base_semver'), + message: text('message'), + appId: text('app_id'), + actorId: text('actor_id'), + stripUnknownFields: bool('strip_unknown_fields').notNull().default(false), + manifestExpected: integer('manifest_expected'), + manifestReceived: integer('manifest_received').notNull().default(0), + manifestNeeded: integer('manifest_needed').notNull().default(0), + recordsReceived: integer('records_received').notNull().default(0), + runs: integer('runs').notNull().default(0), + status: text('status').$type().notNull().default('open'), + result: json>('result'), + error: json<{ statusCode: number; error: string; [k: string]: unknown }>('error'), + finalizeStartedAt: ts('finalize_started_at'), + createdAt: createdAt(), + expiresAt: ts('expires_at').notNull(), + }, + (t) => [ + index('push_sessions_collection_idx').on(t.collectionId), + index('push_sessions_expires_idx').on(t.status, t.expiresAt), + ], +) + +/** One uploaded batch, stored as a sorted run object. */ +export const pushRuns = sqliteTable( + 'push_runs', + { + sessionId: text('session_id') + .notNull() + .references(() => pushSessions.id, { onDelete: 'cascade' }), + seq: integer('seq').notNull(), + kind: text('kind', { enum: ['manifest', 'records', 'deletes'] }).notNull(), + objectKey: text('object_key').notNull(), + count: integer('count').notNull(), + /** Run key range, `type\u0000id`, for planning commit units. */ + firstKey: text('first_key').notNull(), + lastKey: text('last_key').notNull(), + createdAt: createdAt(), + }, + (t) => [primaryKey({ columns: [t.sessionId, t.seq] })], +) + +// --- Jobs (Node only; Cloudflare uses Queues) ---------------------------------------- + +export const jobs = sqliteTable( + 'jobs', + { + id: id(), + type: text('type').notNull(), + payload: json>('payload').notNull(), + status: text('status', { enum: ['queued', 'running', 'done', 'failed'] }) + .notNull() + .default('queued'), + attempts: integer('attempts').notNull().default(0), + runAt: ts('run_at').notNull(), + lockedUntil: ts('locked_until'), + error: text('error'), + createdAt: createdAt(), + }, + (t) => [index('jobs_ready_idx').on(t.status, t.runAt)], +) + +// --- Webhooks -------------------------------------------------------------------------- + +export const collectionWebhooks = sqliteTable( + 'collection_webhooks', + { + id: id(), + collectionId: text('collection_id') + .notNull() + .references(() => collections.id, { onDelete: 'cascade' }), + url: text('url').notNull(), + bumpFilter: text('bump_filter', { enum: ['all', 'major', 'minor', 'patch'] }) + .notNull() + .default('all'), + secret: text('secret').notNull(), + enabled: bool('enabled').notNull().default(true), + createdBy: text('created_by'), + createdAt: createdAt(), + updatedAt: ts('updated_at') + .notNull() + .$defaultFn(() => new Date()), + lastDeliveryAt: ts('last_delivery_at'), + }, + (t) => [index('collection_webhooks_collection_idx').on(t.collectionId)], +) + +export const webhookDeliveries = sqliteTable( + 'webhook_deliveries', + { + id: id(), + webhookId: text('webhook_id') + .notNull() + .references(() => collectionWebhooks.id, { onDelete: 'cascade' }), + collectionId: text('collection_id').notNull(), + versionId: text('version_id'), + semver: text('semver'), + bumpType: text('bump_type', { enum: ['major', 'minor', 'patch'] }).notNull(), + event: text('event').notNull().default('version.created'), + payload: json>('payload').notNull(), + status: text('status', { enum: ['pending', 'success', 'failed'] }) + .notNull() + .default('pending'), + attempts: integer('attempts').notNull().default(0), + responseCode: integer('response_code'), + error: text('error'), + durationMs: integer('duration_ms'), + nextAttemptAt: ts('next_attempt_at'), + createdAt: createdAt(), + deliveredAt: ts('delivered_at'), + }, + (t) => [ + index('webhook_deliveries_webhook_idx').on(t.webhookId, t.createdAt), + index('webhook_deliveries_pending_idx').on(t.status, t.nextAttemptAt), + ], +) + +// --- ARKs, comments, settings ---------------------------------------------------------------- + +export const arkShoulders = sqliteTable('ark_shoulders', { + id: id(), + organizationId: text('organization_id') + .notNull() + .references(() => organization.id, { onDelete: 'cascade' }), + shoulder: text('shoulder').notNull().unique(), + createdAt: createdAt(), +}) + +export const arkCollections = sqliteTable('ark_collections', { + collectionId: text('collection_id') + .primaryKey() + .references(() => collections.id, { onDelete: 'cascade' }), + arkId: text('ark_id').notNull().unique(), + enabled: bool('enabled').notNull().default(true), + customUrl: text('custom_url'), + createdAt: createdAt(), +}) + +export const arkRecordTypes = sqliteTable( + 'ark_record_types', + { + collectionId: text('collection_id') + .notNull() + .references(() => collections.id, { onDelete: 'cascade' }), + recordType: text('record_type').notNull(), + redirectUrlField: text('redirect_url_field').notNull(), + }, + (t) => [primaryKey({ columns: [t.collectionId, t.recordType] })], +) + +export const pageComments = sqliteTable( + 'page_comments', + { + id: id(), + page: text('page').notNull(), + anchor: text('anchor').notNull(), + quote: text('quote'), + quoteContext: json<{ prefix: string; suffix: string }>('quote_context'), + parentId: text('parent_id'), + userId: text('user_id').notNull(), + body: text('body').notNull(), + approvedAt: ts('approved_at'), + approvedBy: text('approved_by'), + status: text('status', { enum: ['open', 'answered', 'decided', 'changed'] }) + .notNull() + .default('open'), + resolutionNote: text('resolution_note'), + createdAt: createdAt(), + editedAt: ts('edited_at'), + deletedAt: ts('deleted_at'), + }, + (t) => [index('page_comments_page_idx').on(t.page)], +) + +export const instanceSettings = sqliteTable('instance_settings', { + key: text('key').primaryKey(), + value: json('value').notNull(), + updatedAt: ts('updated_at') + .notNull() + .$defaultFn(() => new Date()), +}) diff --git a/packages/server/src/jobs.ts b/packages/server/src/jobs.ts new file mode 100644 index 0000000..d7a8fe1 --- /dev/null +++ b/packages/server/src/jobs.ts @@ -0,0 +1,130 @@ +/** + * Background work. Handlers are idempotent: a job may run more than once + * (Queues delivers at least once; the Node runner retries after a crash), so + * every handler either writes content-addressed objects or makes a conditional + * state change. + * + * Cloudflare: Cloudflare Queues; the Worker's `queue` handler calls runJob. + * Node: a `jobs` table polled by an in-process runner. + */ +import { and, asc, eq, lte, or, sql } from 'drizzle-orm' + +import * as schema from './db/schema.js' +import type { Db, JobMessage, Jobs, Ports } from './ports.js' + +export type JobHandler = (job: JobMessage, ports: Ports) => Promise + +const handlers = new Map() + +export function registerJob(type: string, handler: JobHandler): void { + handlers.set(type, handler) +} + +export async function runJob(job: JobMessage, ports: Ports): Promise { + const handler = handlers.get(job.type) + if (!handler) throw new Error(`No handler for job type ${job.type}`) + await handler(job, ports) +} + +// --- Cloudflare Queues -------------------------------------------------------------- + +interface CfQueue { + send(body: unknown, opts?: { delaySeconds?: number }): Promise + sendBatch(messages: { body: unknown; delaySeconds?: number }[]): Promise +} + +export class QueueJobs implements Jobs { + constructor(readonly queue: CfQueue) {} + async enqueue(job: JobMessage, opts?: { delaySeconds?: number }) { + await this.queue.send(job, opts?.delaySeconds ? { delaySeconds: opts.delaySeconds } : undefined) + } + async enqueueBatch(jobs: JobMessage[]) { + // sendBatch takes at most 100 messages. + for (let i = 0; i < jobs.length; i += 100) { + await this.queue.sendBatch(jobs.slice(i, i + 100).map((body) => ({ body }))) + } + } +} + +// --- Node: SQLite jobs table ----------------------------------------------------------- + +const LOCK_MS = 15 * 60 * 1000 +const MAX_ATTEMPTS = 8 + +export class SqliteJobs implements Jobs { + constructor(readonly db: Db) {} + async enqueue(job: JobMessage, opts?: { delaySeconds?: number }) { + await this.db.insert(schema.jobs).values({ + type: job.type, + payload: job, + runAt: new Date(Date.now() + (opts?.delaySeconds ?? 0) * 1000), + }) + } + async enqueueBatch(jobs: JobMessage[]) { + // D1-safe chunking of bound parameters (3 per row). + for (let i = 0; i < jobs.length; i += 30) { + await this.db + .insert(schema.jobs) + .values( + jobs.slice(i, i + 30).map((job) => ({ type: job.type, payload: job, runAt: new Date() })), + ) + } + } +} + +/** Claim one ready job, atomically: a single UPDATE … RETURNING. */ +async function claim(db: Db): Promise { + const now = Date.now() + const ready = db + .select({ id: schema.jobs.id }) + .from(schema.jobs) + .where( + or( + and(eq(schema.jobs.status, 'queued'), lte(schema.jobs.runAt, new Date(now))), + and(eq(schema.jobs.status, 'running'), lte(schema.jobs.lockedUntil, new Date(now))), + ), + ) + .orderBy(asc(schema.jobs.runAt)) + .limit(1) + const [job] = await db + .update(schema.jobs) + .set({ + status: 'running', + lockedUntil: new Date(now + LOCK_MS), + attempts: sql`${schema.jobs.attempts} + 1`, + }) + .where(eq(schema.jobs.id, sql`(${ready})`)) + .returning() + return job ?? null +} + +/** + * Run ready jobs until none are left. Returns how many ran. The Node entry calls + * this on an interval and right after enqueueing; tests call it directly. + */ +export async function drainSqliteJobs(ports: Ports, opts: { max?: number } = {}): Promise { + let ran = 0 + for (;;) { + if (opts.max !== undefined && ran >= opts.max) return ran + const job = await claim(ports.db) + if (!job) return ran + ran++ + try { + await runJob(job.payload as JobMessage, ports) + await ports.db.delete(schema.jobs).where(eq(schema.jobs.id, job.id)) + } catch (err) { + const failed = job.attempts >= MAX_ATTEMPTS + const backoff = Math.min(3600, 2 ** job.attempts) * 1000 + await ports.db + .update(schema.jobs) + .set({ + status: failed ? 'failed' : 'queued', + runAt: new Date(Date.now() + backoff), + lockedUntil: null, + error: String((err as Error)?.stack ?? err).slice(0, 4000), + }) + .where(eq(schema.jobs.id, job.id)) + console.error(`[jobs] ${job.type} attempt ${job.attempts} failed:`, err) + } + } +} diff --git a/packages/server/src/lib/gzip.ts b/packages/server/src/lib/gzip.ts new file mode 100644 index 0000000..688aee5 --- /dev/null +++ b/packages/server/src/lib/gzip.ts @@ -0,0 +1,30 @@ +/** Gzip with the Web Streams compression API (Workers and Node alike). */ + +async function pipe( + bytes: Uint8Array, + t: CompressionStream | DecompressionStream, +): Promise { + const stream = new Blob([bytes as Uint8Array]).stream().pipeThrough(t) + return new Uint8Array(await new Response(stream).arrayBuffer()) +} + +const enc = new TextEncoder() +const dec = new TextDecoder() + +export const gzip = (data: Uint8Array | string) => + pipe(typeof data === 'string' ? enc.encode(data) : data, new CompressionStream('gzip')) + +export const gunzip = (bytes: Uint8Array) => pipe(bytes, new DecompressionStream('gzip')) + +export const gunzipText = async (bytes: Uint8Array) => dec.decode(await gunzip(bytes)) + +export const isGzip = (bytes: Uint8Array) => + bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b + +/** Split decompressed NDJSON into lines (a trailing newline ends the last line). */ +export function splitLines(text: string): string[] { + if (text.length === 0) return [] + const lines = text.split('\n') + if (lines[lines.length - 1] === '') lines.pop() + return lines +} diff --git a/packages/server/src/lib/lru.ts b/packages/server/src/lib/lru.ts new file mode 100644 index 0000000..0adc503 --- /dev/null +++ b/packages/server/src/lib/lru.ts @@ -0,0 +1,43 @@ +/** + * A byte-budgeted LRU for the isolate (or process). Holds immutable, hash-keyed + * things — roots, interior nodes, decoded leaves — so a busy collection's upper + * tree stays in memory across requests. + */ +export class Lru { + readonly #map = new Map() + #bytes = 0 + + constructor( + readonly budget: number, + readonly sizeOf: (v: V) => number, + ) {} + + get(key: string): V | undefined { + const hit = this.#map.get(key) + if (!hit) return undefined + this.#map.delete(key) + this.#map.set(key, hit) + return hit.value + } + + set(key: string, value: V): void { + const size = this.sizeOf(value) + if (size > this.budget / 4) return // never let one item flush the cache + const old = this.#map.get(key) + if (old) { + this.#bytes -= old.size + this.#map.delete(key) + } + this.#map.set(key, { value, size }) + this.#bytes += size + while (this.#bytes > this.budget) { + const oldest = this.#map.keys().next().value as string + this.#bytes -= this.#map.get(oldest)!.size + this.#map.delete(oldest) + } + } + + get bytes(): number { + return this.#bytes + } +} diff --git a/packages/server/src/node/main.ts b/packages/server/src/node/main.ts new file mode 100644 index 0000000..f576397 --- /dev/null +++ b/packages/server/src/node/main.ts @@ -0,0 +1,95 @@ +/** + * Node entry: one process serving the app, running jobs from the SQLite jobs + * table, and serving presigned `/_blob/…` URLs when blobs are on the filesystem. + * + * Env: + * PORT (4200), APP_URL, DEPLOYMENT + * DB_URL file:./data/underlay.sqlite + * BLOB_DIR use the filesystem blob store under this directory, or + * S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY, S3_REGION + * BLOB_URL_SECRET HMAC key for filesystem presigned URLs + */ +import { serve } from '@hono/node-server' + +import { createApp } from '../app.js' +import { FsBlobStore, serveSignedBlob } from '../blob/fs.js' +import { S3BlobStore } from '../blob/s3.js' +import { MemoryCache } from '../cache.js' +import { openNodeDb } from '../db/node.js' +import { drainSqliteJobs, SqliteJobs } from '../jobs.js' +import type { BlobStore, Ports } from '../ports.js' + +const env = process.env +const port = Number(env.PORT ?? 4200) +const appUrl = env.APP_URL ?? `http://localhost:${port}` + +const db = await openNodeDb(env.DB_URL ?? 'file:./data/underlay.sqlite') + +let blobs: BlobStore +let fsBlobs: FsBlobStore | null = null +if (env.S3_ENDPOINT) { + blobs = new S3BlobStore({ + endpoint: env.S3_ENDPOINT, + bucket: env.S3_BUCKET ?? 'underlay', + accessKeyId: env.S3_ACCESS_KEY ?? '', + secretAccessKey: env.S3_SECRET_KEY ?? '', + region: env.S3_REGION ?? 'auto', + }) +} else { + fsBlobs = new FsBlobStore({ + root: env.BLOB_DIR ?? './data/blobs', + publicUrl: appUrl, + secret: env.BLOB_URL_SECRET ?? 'dev-blob-secret', + }) + blobs = fsBlobs +} + +let kick: () => void = () => {} +const ports: Ports = { + db, + blobs, + cache: new MemoryCache(), + jobs: { + async enqueue(job, opts) { + await jobsTable.enqueue(job, opts) + kick() + }, + async enqueueBatch(js) { + await jobsTable.enqueueBatch(js) + kick() + }, + }, + waitUntil: (p) => { + p.catch((err) => console.error('[waitUntil]', err)) + }, +} +const jobsTable = new SqliteJobs(db) + +// One runner loop: wakes on enqueue and every few seconds for delayed jobs. +let running = false +const runJobs = async () => { + if (running) return + running = true + try { + await drainSqliteJobs(ports) + } catch (err) { + console.error('[jobs]', err) + } finally { + running = false + } +} +kick = () => void runJobs() +setInterval(kick, 5000).unref() + +const config = { appUrl, deployment: env.DEPLOYMENT ?? 'dev' } +const app = createApp(() => ({ ports, config })) + +serve({ + port, + fetch: (req) => { + if (fsBlobs && new URL(req.url).pathname.startsWith('/_blob/')) + return serveSignedBlob(fsBlobs, req) + return app.fetch(req) + }, +}) +console.log(`underlay v2 (node) listening on ${appUrl}`) diff --git a/packages/server/src/ports.ts b/packages/server/src/ports.ts new file mode 100644 index 0000000..f3d2af8 --- /dev/null +++ b/packages/server/src/ports.ts @@ -0,0 +1,103 @@ +/** + * The four ports the server is written against. Cloudflare and Node each supply + * adapters; nothing outside the adapters knows which runtime it's on. + * + * BlobStore S3 API via aws4fetch (R2, S3, MinIO), filesystem and memory for dev/tests + * Db Drizzle sqlite-core over D1 or libsql — async, batches only (no interactive transactions) + * Jobs Cloudflare Queues, or a SQLite jobs table polled by the Node process + * Cache Cache API on Workers, in-memory LRU on Node + */ +import type { LibSQLDatabase } from 'drizzle-orm/libsql' + +import type * as schema from './db/schema.js' + +// --- Blob store ------------------------------------------------------------------ + +export interface BlobHead { + size: number + etag: string + contentType: string | null +} + +export interface BlobObject extends BlobHead { + body: ReadableStream + bytes(): Promise + text(): Promise +} + +export interface PutOptions { + contentType?: string + /** Only write if the key doesn't exist. Immutable keys make this an optimization. */ + ifAbsent?: boolean +} + +export interface PresignGetOptions { + expiresIn: number + /** Content-Disposition for the response. */ + disposition?: string + contentType?: string +} + +export interface PresignPutOptions { + expiresIn: number + contentType?: string +} + +export interface BlobStore { + get(key: string, range?: { offset: number; length?: number }): Promise + head(key: string): Promise + put(key: string, body: Uint8Array | string, opts?: PutOptions): Promise + delete(key: string): Promise + list(prefix: string, cursor?: string): Promise<{ keys: string[]; cursor?: string }> + presignGet(key: string, opts: PresignGetOptions): Promise + presignPut(key: string, opts: PresignPutOptions): Promise + createMultipart(key: string, contentType?: string): Promise + presignPart(key: string, uploadId: string, partNumber: number, expiresIn: number): Promise + completeMultipart( + key: string, + uploadId: string, + parts: { partNumber: number; etag: string }[], + ): Promise + abortMultipart(key: string, uploadId: string): Promise +} + +// --- Database ---------------------------------------------------------------------- + +/** + * Drizzle over SQLite. Both adapters are async and support `db.batch([...])`, + * which runs statements atomically. Don't use `db.transaction`: D1 doesn't have + * interactive transactions, so code that works on Node would break on Workers. + */ +export type Db = LibSQLDatabase + +// --- Jobs -------------------------------------------------------------------------------- + +/** Messages carry ids only (Queues caps messages at 128 KB); data is in SQLite and blobs. */ +export interface JobMessage { + type: string + [k: string]: string | number | boolean | null +} + +export interface Jobs { + enqueue(job: JobMessage, opts?: { delaySeconds?: number }): Promise + enqueueBatch(jobs: JobMessage[]): Promise +} + +// --- Cache -------------------------------------------------------------------------------- + +/** A shared cache for immutable, hash-keyed bytes (nodes, roots, schemas). */ +export interface Cache { + get(key: string): Promise + put(key: string, value: Uint8Array, opts?: { ttlSeconds?: number }): Promise +} + +// --- Everything the app needs ------------------------------------------------------- + +export interface Ports { + blobs: BlobStore + db: Db + jobs: Jobs + cache: Cache + /** Run work after the response (Workers: ctx.waitUntil; Node: fire and forget with logging). */ + waitUntil(p: Promise): void +} diff --git a/packages/server/src/storage/objects.ts b/packages/server/src/storage/objects.ts new file mode 100644 index 0000000..5c5e298 --- /dev/null +++ b/packages/server/src/storage/objects.ts @@ -0,0 +1,375 @@ +/** + * The object layout in the blob store, and typed access to it. + * + * nodes/ tree node JSON, gzip (the hash is of the uncompressed bytes) + * bodies/ the leaf's records, one canonical record per line, one gzip member; + * or, for a leaf over BODY_PART_BYTES, a JSON part list {"parts":[…]} + * bodyparts/ one part of a large leaf body, gzip (hash of the uncompressed part) + * records/ an out-of-line record over OUT_OF_LINE_BYTES, gzip; + * its body line is {"$ref":""} + * roots/.json version roots + * private/.json private set objects + * schemas/.json schemas, as canonical JSON + * sessions//… push session inputs and runs (expire by lifecycle rule) + * files/…, uploads/ file bytes (unchanged from v1) + * + * Everything except sessions/ and uploads/ is immutable and keyed by content, so + * it is cached by key forever: the isolate LRU first, then the shared cache, then + * the bucket. Storage layout (part sizes, out-of-line records) is not protocol. + */ +import { + type DecodedNode, + decodeNode, + hashSchema, + jcs, + type NodeDesc, + type NodeSource, + type PrivateSetObject, + privateCommitment, + type RecordEntry, + recordTree, + sha256Hex, + type TreeSink, + type TreeSpec, + type VersionRoot, + versionDigest, + versionHash, +} from '@underlay/core' + +import { gunzip, gunzipText, gzip, isGzip, splitLines } from '../lib/gzip.js' +import { Lru } from '../lib/lru.js' +import type { BlobStore, Cache } from '../ports.js' + +export const OUT_OF_LINE_BYTES = 64 * 1024 +export const BODY_PART_BYTES = 8 * 1024 * 1024 + +export const keys = { + node: (h: string) => `nodes/${h}`, + body: (h: string) => `bodies/${h}`, + part: (h: string) => `bodyparts/${h}`, + record: (h: string) => `records/${h}`, + root: (versionHashOrDigest: string) => + `roots/${versionHashOrDigest.includes(':') ? versionDigest(versionHashOrDigest) : versionHashOrDigest}.json`, + privateSet: (commitment: string) => `private/${commitment}.json`, + schema: (h: string) => `schemas/${h}.json`, + session: (id: string, name: string) => `sessions/${id}/${name}`, +} + +const REF_PREFIX = '{"$ref":"' +const enc = new TextEncoder() +const dec = new TextDecoder() + +/** Per-isolate memory: decoded nodes, roots, schemas, bodies. Shared by every request. */ +const isolateLru = new Lru(32 * 1024 * 1024, (v) => { + if (typeof v === 'string') return v.length * 2 + if (v && typeof v === 'object' && 'approxBytes' in v) + return (v as { approxBytes: number }).approxBytes + return 1024 +}) + +export class Objects { + constructor( + readonly blobs: BlobStore, + readonly cache: Cache, + readonly lru: Lru = isolateLru, + ) {} + + /** An immutable object's bytes: shared cache, then the bucket. */ + async #immutable(key: string): Promise { + const cached = await this.cache.get(key) + if (cached) return cached + const obj = await this.blobs.get(key) + if (!obj) return null + const bytes = await obj.bytes() + await this.cache.put(key, bytes) + return bytes + } + + // --- Nodes --- + + async nodeJson(hash: string): Promise { + const key = keys.node(hash) + const hit = this.lru.get(key) + if (typeof hit === 'string') return hit + const bytes = await this.#immutable(key) + if (!bytes) throw new Error(`Missing node ${hash}`) + const json = await gunzipText(bytes) + this.lru.set(key, json) + return json + } + + async putNode(hash: string, json: string): Promise { + await this.blobs.put(keys.node(hash), await gzip(json), { + contentType: 'application/gzip', + ifAbsent: true, + }) + } + + // --- Bodies --- + + /** A leaf's body lines, in entry order, with out-of-line records resolved. */ + async bodyLines(leafHash: string): Promise { + const key = keys.body(leafHash) + const hit = this.lru.get(key) + if (hit) return (hit as { lines: string[] }).lines + const bytes = await this.#immutable(key) + if (!bytes) throw new Error(`Missing body for leaf ${leafHash}`) + let lines: string[] + if (isGzip(bytes)) { + lines = splitLines(await gunzipText(bytes)) + } else { + const { parts } = JSON.parse(dec.decode(bytes)) as { parts: string[] } + const texts = await Promise.all( + parts.map(async (h) => { + const b = await this.#immutable(keys.part(h)) + if (!b) throw new Error(`Missing body part ${h}`) + return splitLines(await gunzipText(b)) + }), + ) + lines = texts.flat() + } + if (lines.some((l) => l.startsWith(REF_PREFIX))) { + lines = await Promise.all( + lines.map((l) => (l.startsWith(REF_PREFIX) ? this.#outOfLine(l) : l)), + ) + } + const approxBytes = lines.reduce((n, l) => n + l.length * 2, 0) + this.lru.set(key, { lines, approxBytes }) + return lines + } + + async #outOfLine(pointer: string): Promise { + const hash = (JSON.parse(pointer) as { $ref: string }).$ref + const b = await this.#immutable(keys.record(hash)) + if (!b) throw new Error(`Missing out-of-line record ${hash}`) + return gunzipText(b) + } + + async putBody(leafHash: string, lines: readonly string[]): Promise { + await this.blobs.put(keys.body(leafHash), await gzip(lines.join('\n') + '\n'), { + contentType: 'application/gzip', + ifAbsent: true, + }) + } + + async putBodyParts(leafHash: string, parts: readonly string[]): Promise { + await this.blobs.put(keys.body(leafHash), JSON.stringify({ parts }), { + contentType: 'application/json', + ifAbsent: true, + }) + } + + /** Write one body part; returns its hash (of the uncompressed text). */ + partOf(lines: readonly string[]): { hash: string; write: () => Promise } { + const text = lines.join('\n') + '\n' + const hash = sha256Hex(text) + return { + hash, + write: async () => + this.blobs.put(keys.part(hash), await gzip(text), { + contentType: 'application/gzip', + ifAbsent: true, + }), + } + } + + /** Store a large record out of line; returns the pointer line that goes in the body. */ + async putOutOfLine(recordHash: string, canonical: string): Promise { + await this.blobs.put(keys.record(recordHash), await gzip(canonical), { + contentType: 'application/gzip', + ifAbsent: true, + }) + return outOfLinePointer(recordHash) + } + + // --- Roots, private sets, schemas --- + + async root(hash: string): Promise { + const key = keys.root(hash) + const hit = this.lru.get(key) + if (hit) return (hit as { root: VersionRoot }).root + const bytes = await this.#immutable(key) + if (!bytes) throw new Error(`Missing root ${hash}`) + const text = dec.decode(bytes) + const root = JSON.parse(text) as VersionRoot + this.lru.set(key, { root, approxBytes: text.length * 3 }) + return root + } + + async putRoot(root: VersionRoot): Promise { + const hash = versionHash(root) + await this.blobs.put(keys.root(hash), jcs(root), { + contentType: 'application/json', + ifAbsent: true, + }) + return hash + } + + async privateSet(commitment: string): Promise { + const key = keys.privateSet(commitment) + const hit = this.lru.get(key) + if (hit) return (hit as { set: PrivateSetObject }).set + const bytes = await this.#immutable(key) + if (!bytes) throw new Error(`Missing private set ${commitment}`) + const text = dec.decode(bytes) + const set = JSON.parse(text) as PrivateSetObject + this.lru.set(key, { set, approxBytes: text.length * 3 }) + return set + } + + async putPrivateSet(set: PrivateSetObject): Promise { + const commitment = privateCommitment(set) + await this.blobs.put(keys.privateSet(commitment), jcs(set), { + contentType: 'application/json', + ifAbsent: true, + }) + return commitment + } + + async schema(hash: string): Promise> { + const key = keys.schema(hash) + const hit = this.lru.get(key) + if (hit) return (hit as { schema: Record }).schema + const bytes = await this.#immutable(key) + if (!bytes) throw new Error(`Missing schema ${hash}`) + const text = dec.decode(bytes) + const schema = JSON.parse(text) as Record + this.lru.set(key, { schema, approxBytes: text.length * 3 }) + return schema + } + + async putSchema(schema: unknown): Promise { + const hash = hashSchema(schema) + await this.blobs.put(keys.schema(hash), jcs(schema), { + contentType: 'application/json', + ifAbsent: true, + }) + return hash + } + + // --- Decoded nodes (for tree reads) --- + + async decoded(spec: TreeSpec, hash: string): Promise> { + const key = `decoded:${spec.name}:${hash}` + const hit = this.lru.get(key) + if (hit) return (hit as { node: DecodedNode }).node + const json = await this.nodeJson(hash) + const node = decodeNode(spec, json, hash) + this.lru.set(key, { node, approxBytes: json.length * 3 }) + return node + } +} + +export const outOfLinePointer = (recordHash: string) => `${REF_PREFIX}${recordHash}"}` + +/** NodeSource over the blob store. Record trees get bodies through `leafEntries`. */ +export class BlobSource implements NodeSource { + constructor( + readonly spec: TreeSpec, + readonly objects: Objects, + ) {} + + node(hash: string): Promise> { + return this.objects.decoded(this.spec, hash) + } + + async leafEntries(hash: string): Promise { + const node = await this.node(hash) + if (node.kind !== 'leaf') throw new Error(`Node ${hash} is not a leaf`) + if (this.spec !== (recordTree as TreeSpec)) return node.entries.slice() + const lines = await this.objects.bodyLines(hash) + if (lines.length !== node.entries.length) { + throw new Error( + `Body of leaf ${hash} has ${lines.length} lines for ${node.entries.length} entries`, + ) + } + return (node.entries as RecordEntry[]).map((e, i) => ({ ...e, body: lines[i]! })) as E[] + } +} + +/** + * TreeSink that writes nodes (and, for record trees, bodies) to the blob store. + * The builder is synchronous; writes run in the background with bounded + * concurrency. Call `drain()` between units of work to apply backpressure and + * `flush()` before using the root. + */ +export class BlobSink implements TreeSink { + readonly #pending = new Set>() + #error: unknown = null + #parts: string[] = [] + written = 0 + + constructor( + readonly objects: Objects, + readonly opts: { bodyOf?: (e: E) => string; concurrency?: number } = {}, + ) {} + + #run(work: () => Promise) { + const p = work() + .catch((err) => { + this.#error ??= err + }) + .finally(() => this.#pending.delete(p)) + this.#pending.add(p) + this.written++ + } + + spill(entries: readonly E[]): void { + const { bodyOf } = this.opts + if (!bodyOf) return + const part = this.objects.partOf(entries.map(bodyOf)) + this.#parts.push(part.hash) + this.#run(part.write) + } + + leaf(desc: NodeDesc, json: string, entries: readonly E[], spilled: number): void { + this.#run(() => this.objects.putNode(desc.hash, json)) + const { bodyOf } = this.opts + if (!bodyOf) return + if (this.#parts.length === 0) { + const lines = entries.map(bodyOf) + this.#run(() => this.objects.putBody(desc.hash, lines)) + return + } + const tail = entries.slice(spilled) + if (tail.length > 0) { + const part = this.objects.partOf(tail.map(bodyOf)) + this.#parts.push(part.hash) + this.#run(part.write) + } + const parts = this.#parts + this.#parts = [] + this.#run(() => this.objects.putBodyParts(desc.hash, parts)) + } + + interior(desc: NodeDesc, json: string): void { + this.#run(() => this.objects.putNode(desc.hash, json)) + } + + async drain(): Promise { + const limit = this.opts.concurrency ?? 16 + while (this.#pending.size >= limit) await Promise.race(this.#pending) + if (this.#error) throw this.#error + } + + async flush(): Promise { + while (this.#pending.size > 0) await Promise.race(this.#pending) + if (this.#error) throw this.#error + } +} + +export const bodyOfRecord = (e: RecordEntry): string => { + if (e.body === undefined) throw new Error(`Record ${e.key} has no body`) + return e.body +} + +/** Payload size for spilling: the body line's bytes. */ +export const recordPayloadBytes = (e: RecordEntry) => e.body?.length ?? 0 + +/** Clear a record entry's body after it was written in a part. */ +export const dropRecordBody = (e: RecordEntry) => { + delete e.body +} + +export const textBytes = (s: string) => enc.encode(s) +export { gunzip } diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts new file mode 100644 index 0000000..cadbbd4 --- /dev/null +++ b/packages/server/src/worker.ts @@ -0,0 +1,81 @@ +/** + * Cloudflare Workers entry: HTTP (fetch), background jobs (queue) and + * schedules (scheduled). Bindings and secrets are in wrangler.jsonc. + */ +import type { + D1Database, + ExecutionContext, + MessageBatch, + Queue, + ScheduledController, +} from '@cloudflare/workers-types' + +import { createApp } from './app.js' +import { S3BlobStore } from './blob/s3.js' +import { CfCache } from './cache.js' +import { openD1 } from './db/d1.js' +import { QueueJobs, runJob } from './jobs.js' +import type { JobMessage, Ports } from './ports.js' + +export interface Env { + DB: D1Database + JOBS: Queue + APP_URL: string + DEPLOYMENT: string + R2_ENDPOINT: string + R2_BUCKET: string + R2_ACCESS_KEY_ID: string + R2_SECRET_ACCESS_KEY: string +} + +function makePorts(env: Env, ctx: ExecutionContext): Ports { + return { + db: openD1(env.DB), + blobs: new S3BlobStore({ + endpoint: env.R2_ENDPOINT, + bucket: env.R2_BUCKET, + accessKeyId: env.R2_ACCESS_KEY_ID, + secretAccessKey: env.R2_SECRET_ACCESS_KEY, + region: 'auto', + }), + cache: new CfCache(caches as never, env.DEPLOYMENT), + jobs: new QueueJobs(env.JOBS as never), + waitUntil: (p) => ctx.waitUntil(p), + } +} + +const app = createApp((c) => { + const env = c.env as unknown as Env + return { + ports: makePorts(env, c.executionCtx as unknown as ExecutionContext), + config: { appUrl: env.APP_URL, deployment: env.DEPLOYMENT }, + } +}) + +export default { + fetch(req: Request, env: Env, ctx: ExecutionContext): Response | Promise { + return app.fetch(req, env as never, ctx as never) + }, + + async queue(batch: MessageBatch, env: Env, ctx: ExecutionContext): Promise { + const ports = makePorts(env, ctx) + for (const msg of batch.messages) { + try { + await runJob(msg.body, ports) + msg.ack() + } catch (err) { + console.error(`[jobs] ${msg.body.type} failed (attempt ${msg.attempts}):`, err) + msg.retry({ delaySeconds: Math.min(3600, 2 ** msg.attempts) }) + } + } + }, + + async scheduled( + _controller: ScheduledController, + env: Env, + ctx: ExecutionContext, + ): Promise { + const ports = makePorts(env, ctx) + await ports.jobs.enqueue({ type: 'maintenance.sweep' }) + }, +} diff --git a/packages/server/test/blob.test.ts b/packages/server/test/blob.test.ts new file mode 100644 index 0000000..b262c72 --- /dev/null +++ b/packages/server/test/blob.test.ts @@ -0,0 +1,132 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { FsBlobStore, serveSignedBlob } from '../src/blob/fs.js' +import { MemoryBlobStore } from '../src/blob/memory.js' +import { S3BlobStore } from '../src/blob/s3.js' +import type { BlobStore } from '../src/ports.js' +import { type FakeS3, startFakeS3 } from './fake-s3.js' + +const enc = new TextEncoder() + +/** The contract every BlobStore satisfies. */ +function contract(name: string, make: () => Promise) { + describe(`${name} blob store`, () => { + let store: BlobStore + beforeAll(async () => { + store = await make() + }) + + it('puts, gets, heads, ranges and deletes', async () => { + await store.put('a/b c/obj', 'hello world', { contentType: 'text/plain' }) + const obj = await store.get('a/b c/obj') + expect(obj?.size).toBe(11) + expect(await obj!.text()).toBe('hello world') + expect((await store.head('a/b c/obj'))?.size).toBe(11) + expect(await (await store.get('a/b c/obj', { offset: 6, length: 3 }))!.text()).toBe('wor') + expect(await (await store.get('a/b c/obj', { offset: 6 }))!.text()).toBe('world') + await store.delete('a/b c/obj') + expect(await store.get('a/b c/obj')).toBe(null) + expect(await store.head('a/b c/obj')).toBe(null) + }) + + it('treats ifAbsent on an existing key as success without overwriting', async () => { + await store.put('immutable', 'first') + await store.put('immutable', 'second', { ifAbsent: true }) + expect(await (await store.get('immutable'))!.text()).toBe('first') + }) + + it('streams bodies', async () => { + await store.put('stream', enc.encode('x'.repeat(100_000))) + const obj = await store.get('stream') + const text = await new Response(obj!.body).text() + expect(text.length).toBe(100_000) + }) + + it('lists by prefix', async () => { + await store.put('list/1', '1') + await store.put('list/2', '2') + await store.put('other/3', '3') + expect((await store.list('list/')).keys).toEqual(['list/1', 'list/2']) + }) + + it('presigns and completes multipart uploads', async () => { + expect(await store.presignGet('x', { expiresIn: 60 })).toBeTruthy() + expect(await store.presignPut('x', { expiresIn: 60 })).toBeTruthy() + const id = await store.createMultipart('big') + expect(await store.presignPart('big', id, 1, 60)).toBeTruthy() + await store.abortMultipart('big', id) + }) + }) +} + +contract('memory', async () => new MemoryBlobStore()) + +let dir: string +contract('fs', async () => { + dir = await mkdtemp(join(tmpdir(), 'ul-blob-')) + return new FsBlobStore({ root: dir, publicUrl: 'http://localhost:4100', secret: 's3cret' }) +}) +afterAll(async () => { + if (dir) await rm(dir, { recursive: true, force: true }) +}) + +let s3: FakeS3 +contract('s3', async () => { + s3 = await startFakeS3() + return new S3BlobStore({ + endpoint: s3.url, + bucket: 'test', + accessKeyId: 'k', + secretAccessKey: 's', + }) +}) +afterAll(async () => s3?.close()) + +describe('fs presigned URLs', () => { + it('serve GET and PUT with a valid signature only', async () => { + const root = await mkdtemp(join(tmpdir(), 'ul-blob-')) + const store = new FsBlobStore({ root, publicUrl: 'http://localhost:4100', secret: 'k' }) + const put = await store.presignPut('uploads/u1', { expiresIn: 60 }) + const res = await serveSignedBlob(store, new Request(put, { method: 'PUT', body: 'bytes!' })) + expect(res.status).toBe(200) + const get = await store.presignGet('uploads/u1', { expiresIn: 60, disposition: 'attachment' }) + const got = await serveSignedBlob(store, new Request(get)) + expect(await got.text()).toBe('bytes!') + expect(got.headers.get('content-disposition')).toBe('attachment') + const tampered = get.replace('uploads%2Fu1', 'uploads%2Fu2').replace('uploads/u1', 'uploads/u2') + expect((await serveSignedBlob(store, new Request(tampered))).status).toBe(403) + const wrongMethod = await serveSignedBlob(store, new Request(get, { method: 'PUT', body: 'x' })) + expect(wrongMethod.status).toBe(403) + await rm(root, { recursive: true, force: true }) + }) +}) + +describe('s3 multipart', () => { + it('assembles parts uploaded to presigned part URLs', async () => { + const fake = await startFakeS3() + const store = new S3BlobStore({ + endpoint: fake.url, + bucket: 'test', + accessKeyId: 'k', + secretAccessKey: 's', + }) + const id = await store.createMultipart('files/big', 'application/octet-stream') + const parts = [] + for (const [n, text] of [ + [1, 'part one,'], + [2, 'part two'], + ] as const) { + const url = await store.presignPart('files/big', id, n, 60) + expect(url).toContain('X-Amz-Signature') + const r = await fetch(url, { method: 'PUT', body: text }) + parts.push({ partNumber: n, etag: r.headers.get('etag')! }) + } + await store.completeMultipart('files/big', id, parts) + expect(await (await store.get('files/big'))!.text()).toBe('part one,part two') + await fake.close() + }) +}) diff --git a/packages/server/test/db.test.ts b/packages/server/test/db.test.ts new file mode 100644 index 0000000..0323d73 --- /dev/null +++ b/packages/server/test/db.test.ts @@ -0,0 +1,98 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { eq } from 'drizzle-orm' +import { afterAll, describe, expect, it } from 'vitest' + +import { MemoryBlobStore } from '../src/blob/memory.js' +import { MemoryCache } from '../src/cache.js' +import { openNodeDb } from '../src/db/node.js' +import * as schema from '../src/db/schema.js' +import { drainSqliteJobs, registerJob, SqliteJobs } from '../src/jobs.js' +import type { Ports } from '../src/ports.js' + +const dirs: string[] = [] +afterAll(async () => { + for (const d of dirs) await rm(d, { recursive: true, force: true }) +}) + +async function tempDb() { + const dir = await mkdtemp(join(tmpdir(), 'ul-db-')) + dirs.push(dir) + return openNodeDb(`file:${join(dir, 'test.sqlite')}`) +} + +describe('SQLite on Node', () => { + it('migrates and round-trips rows, including JSON and timestamps', async () => { + const db = await tempDb() + await db.insert(schema.organization).values({ id: 'org1', name: 'Org', slug: 'org' }) + const [c] = await db + .insert(schema.collections) + .values({ + organizationId: 'org1', + slug: 'c', + name: 'C', + privateSalt: 'ab'.repeat(32), + summary: { tags: ['x'] }, + }) + .returning() + expect(c!.summary).toEqual({ tags: ['x'] }) + expect(c!.createdAt).toBeInstanceOf(Date) + expect(Math.abs(c!.createdAt.getTime() - Date.now())).toBeLessThan(5000) + }) + + it('runs atomic batches, and rolls back the whole batch on failure', async () => { + const db = await tempDb() + await db.insert(schema.organization).values({ id: 'org1', name: 'Org', slug: 'org' }) + await expect( + db.batch([ + db.insert(schema.organization).values({ id: 'org2', name: 'Two', slug: 'two' }), + db.insert(schema.organization).values({ id: 'org3', name: 'Dup', slug: 'org' }), // unique violation + ]), + ).rejects.toThrow() + expect( + await db.select().from(schema.organization).where(eq(schema.organization.id, 'org2')), + ).toEqual([]) + }) +}) + +describe('SQLite jobs', () => { + it('runs jobs, retries failures with backoff, and claims each job once', async () => { + const db = await tempDb() + const ports: Ports = { + db, + blobs: new MemoryBlobStore(), + cache: new MemoryCache(), + jobs: new SqliteJobs(db), + waitUntil: () => {}, + } + const seen: string[] = [] + let failOnce = true + registerJob('test.echo', async (job) => { + seen.push(String(job.value)) + }) + registerJob('test.flaky', async () => { + if (failOnce) { + failOnce = false + throw new Error('boom') + } + seen.push('flaky ok') + }) + await ports.jobs.enqueueBatch([ + { type: 'test.echo', value: 'a' }, + { type: 'test.echo', value: 'b' }, + ]) + await ports.jobs.enqueue({ type: 'test.flaky' }) + expect(await drainSqliteJobs(ports)).toBe(3) + expect(seen.sort()).toEqual(['a', 'b']) + // The flaky job is back in the queue with a future runAt. + const [retry] = await db.select().from(schema.jobs) + expect(retry!.status).toBe('queued') + expect(retry!.attempts).toBe(1) + await db.update(schema.jobs).set({ runAt: new Date(0) }) + expect(await drainSqliteJobs(ports)).toBe(1) + expect(seen).toContain('flaky ok') + expect(await db.select().from(schema.jobs)).toEqual([]) + }) +}) diff --git a/packages/server/test/fake-s3.ts b/packages/server/test/fake-s3.ts new file mode 100644 index 0000000..9e593a6 --- /dev/null +++ b/packages/server/test/fake-s3.ts @@ -0,0 +1,136 @@ +/** + * A tiny S3-compatible server for adapter tests: path-style GET (with Range), PUT + * (with If-None-Match), HEAD, DELETE, ListObjectsV2 and multipart uploads. It + * checks that requests are signed (header or query) but doesn't verify + * signatures; signing itself is aws4fetch's. + */ +import { createServer, type Server } from 'node:http' + +export interface FakeS3 { + url: string + objects: Map + requests: { method: string; url: string }[] + close(): Promise +} + +export async function startFakeS3(bucket = 'test'): Promise { + const objects = new Map() + const uploads = new Map }>() + const requests: { method: string; url: string }[] = [] + + const server: Server = createServer(async (req, res) => { + const chunks: Buffer[] = [] + for await (const c of req) chunks.push(c as Buffer) + const body = Buffer.concat(chunks) + const url = new URL(req.url!, 'http://x') + requests.push({ method: req.method!, url: req.url! }) + const signed = + !!req.headers.authorization?.startsWith('AWS4-HMAC-SHA256') || + url.searchParams.has('X-Amz-Signature') + if (!signed) { + res.writeHead(403).end('AccessDenied') + return + } + const [, b, ...rest] = url.pathname.split('/') + if (b !== bucket) { + res.writeHead(404).end('NoSuchBucket') + return + } + const key = rest.map(decodeURIComponent).join('/') + const q = url.searchParams + + if (req.method === 'GET' && !key && q.get('list-type') === '2') { + const prefix = q.get('prefix') ?? '' + const keys = [...objects.keys()].filter((k) => k.startsWith(prefix)).sort() + res.writeHead(200, { 'content-type': 'application/xml' }) + res.end( + `${keys.map((k) => `${k}`).join('')}`, + ) + return + } + if (req.method === 'POST' && q.has('uploads')) { + const id = crypto.randomUUID() + uploads.set(id, { key, parts: new Map() }) + res + .writeHead(200) + .end( + `${id}`, + ) + return + } + if (req.method === 'PUT' && q.has('uploadId')) { + const up = uploads.get(q.get('uploadId')!) + if (!up) return void res.writeHead(404).end('NoSuchUpload') + up.parts.set(Number(q.get('partNumber')), body) + res.writeHead(200, { etag: `"p${q.get('partNumber')}"` }).end() + return + } + if (req.method === 'POST' && q.has('uploadId')) { + const up = uploads.get(q.get('uploadId')!) + if (!up) return void res.writeHead(404).end('NoSuchUpload') + const nums = [...body.toString().matchAll(/(\d+)<\/PartNumber>/g)].map((m) => + Number(m[1]), + ) + objects.set(up.key, { + bytes: Buffer.concat(nums.map((n) => up.parts.get(n)!)), + contentType: undefined, + }) + uploads.delete(q.get('uploadId')!) + res.writeHead(200).end('') + return + } + if (req.method === 'DELETE' && q.has('uploadId')) { + uploads.delete(q.get('uploadId')!) + res.writeHead(204).end() + return + } + const obj = objects.get(key) + switch (req.method) { + case 'PUT': + if (req.headers['if-none-match'] === '*' && obj) { + res.writeHead(412).end('PreconditionFailed') + return + } + objects.set(key, { bytes: body, contentType: req.headers['content-type'] }) + res.writeHead(200, { etag: '"x"' }).end() + return + case 'HEAD': + case 'GET': { + if (!obj) + return void res + .writeHead(404) + .end(req.method === 'GET' ? 'NoSuchKey' : undefined) + let bytes = obj.bytes + let status = 200 + const range = /^bytes=(\d+)-(\d*)$/.exec(req.headers.range ?? '') + if (range) { + const start = Number(range[1]) + const end = range[2] ? Number(range[2]) : bytes.length - 1 + bytes = bytes.subarray(start, end + 1) + status = 206 + } + const headers: Record = { + 'content-length': String(bytes.length), + etag: '"x"', + } + if (obj.contentType) headers['content-type'] = obj.contentType + res.writeHead(status, headers) + res.end(req.method === 'GET' ? bytes : undefined) + return + } + case 'DELETE': + objects.delete(key) + res.writeHead(204).end() + return + } + res.writeHead(400).end() + }) + await new Promise((r) => server.listen(0, '127.0.0.1', r)) + const port = (server.address() as { port: number }).port + return { + url: `http://127.0.0.1:${port}`, + objects, + requests, + close: () => new Promise((r) => server.close(() => r())), + } +} diff --git a/packages/server/test/storage.test.ts b/packages/server/test/storage.test.ts new file mode 100644 index 0000000..2fe3a7c --- /dev/null +++ b/packages/server/test/storage.test.ts @@ -0,0 +1,142 @@ +import { + compareUtf8, + diffTrees, + fileTree, + hashRecord, + iterate, + mergeTree, + type RecordEntry, + recordTree, + TreeBuilder, + utf8ByteLength, + verifyTree, +} from '@underlay/core' +import { describe, expect, it } from 'vitest' + +import { MemoryBlobStore } from '../src/blob/memory.js' +import { noCache } from '../src/cache.js' +import { Lru } from '../src/lib/lru.js' +import { + BlobSink, + BlobSource, + bodyOfRecord, + dropRecordBody, + keys, + Objects, + OUT_OF_LINE_BYTES, + recordPayloadBytes, +} from '../src/storage/objects.js' + +const freshObjects = () => { + const blobs = new MemoryBlobStore() + // A private LRU per test, so reads really go to the store. + return { blobs, objects: new Objects(blobs, noCache, new Lru(8 * 1024 * 1024, () => 1024)) } +} + +const record = (id: string, v = 0): RecordEntry => { + const { hash, canonical } = hashRecord(id, 'T', { id, v, pad: 'x'.repeat(200) }) + return { key: id, hash, size: utf8ByteLength(canonical), body: canonical } +} + +const ids = (n: number) => + Array.from({ length: n }, (_, i) => `rec-${String(i).padStart(6, '0')}`).sort(compareUtf8) + +async function collect(it: AsyncIterable): Promise { + const out: T[] = [] + for await (const x of it) out.push(x) + return out +} + +describe('record trees in the blob store', () => { + it('round-trips nodes and bodies, and verifies', async () => { + const { blobs, objects } = freshObjects() + const sink = new BlobSink(objects, { bodyOf: bodyOfRecord }) + const b = new TreeBuilder(recordTree, sink) + for (const id of ids(5000)) b.addEntry(record(id)) + const { root } = b.finish() + await sink.flush() + expect(root!.count).toBe(5000) + expect([...blobs.objects.keys()].filter((k) => k.startsWith('bodies/')).length).toBeGreaterThan( + 1, + ) + + const source = new BlobSource(recordTree, objects) + expect((await verifyTree(source, root!.hash)).ok).toBe(true) + const rows = await collect(iterate(source, root!.hash, { payloads: true, offset: 4990 })) + expect(rows.map((r) => r.key)).toEqual(ids(5000).slice(4990)) + for (const r of rows) expect(JSON.parse(r.body!).id).toBe(r.key) + }) + + it('merges against stored trees, rewriting only changed leaves', async () => { + const { blobs, objects } = freshObjects() + const sink = new BlobSink(objects, { bodyOf: bodyOfRecord }) + const b = new TreeBuilder(recordTree, sink) + const all = ids(20_000) + for (const id of all) b.addEntry(record(id)) + const base = b.finish().root! + await sink.flush() + const putsBefore = blobs.puts + + const source = new BlobSource(recordTree, objects) + const changes = [all[10]!, all[15_000]!].map((id) => ({ key: id, entry: record(id, 1) })) + const sink2 = new BlobSink(objects, { bodyOf: bodyOfRecord }) + const merged = await mergeTree(source, sink2, base.hash, changes) + await sink2.flush() + expect(merged.stats.updated).toBe(2) + // Two leaves (node + body each) and their paths; nothing else. + expect(blobs.puts - putsBefore).toBeLessThanOrEqual(2 * 2 + 2 * (base.level + 1)) + const diff = await collect(diffTrees(source, base.hash, merged.root!.hash)) + expect(diff.map((d) => d.key)).toEqual([all[10], all[15_000]]) + const changed = await collect( + iterate(source, merged.root!.hash, { payloads: true, offset: 10 }), + ) + expect(JSON.parse(changed[0]!.body!).data.v).toBe(1) + }) + + it('spills large leaf bodies into parts and reads them back', async () => { + const { blobs, objects } = freshObjects() + const sink = new BlobSink(objects, { bodyOf: bodyOfRecord }) + const b = new TreeBuilder(recordTree, sink, { + payloadBytes: recordPayloadBytes, + dropPayload: dropRecordBody, + spillBytes: 20_000, // tiny, to force parts + }) + for (const id of ids(3000)) b.addEntry(record(id)) + const root = b.finish().root! + await sink.flush() + expect([...blobs.objects.keys()].some((k) => k.startsWith('bodyparts/'))).toBe(true) + const source = new BlobSource(recordTree, objects) + const rows = await collect(iterate(source, root.hash, { payloads: true })) + expect(rows.length).toBe(3000) + expect(rows.every((r) => JSON.parse(r.body!).id === r.key)).toBe(true) + }) + + it('resolves out-of-line records', async () => { + const { objects } = freshObjects() + const big = hashRecord('big', 'T', { blob: 'y'.repeat(OUT_OF_LINE_BYTES + 10) }) + const pointer = await objects.putOutOfLine(big.hash, big.canonical) + const sink = new BlobSink(objects, { bodyOf: bodyOfRecord }) + const b = new TreeBuilder(recordTree, sink) + b.addEntry(record('a')) + b.addEntry({ key: 'big', hash: big.hash, size: utf8ByteLength(big.canonical), body: pointer }) + const root = b.finish().root! + await sink.flush() + const rows = await collect( + iterate(new BlobSource(recordTree, objects), root.hash, { payloads: true }), + ) + expect(rows[1]!.body).toBe(big.canonical) + }) + + it('stores file trees without bodies', async () => { + const { blobs, objects } = freshObjects() + const sink = new BlobSink(objects) + const b = new TreeBuilder(fileTree, sink) + b.addEntry({ key: 'a'.repeat(64), size: 10 }) + b.addEntry({ key: 'b'.repeat(64), size: 20 }) + const root = b.finish().root! + await sink.flush() + expect([...blobs.objects.keys()]).toEqual([keys.node(root.hash)]) + const entries = await collect(iterate(new BlobSource(fileTree, objects), root.hash)) + expect(entries.map((e) => e.size)).toEqual([10, 20]) + }) +}) diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json new file mode 100644 index 0000000..96f05dd --- /dev/null +++ b/packages/server/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "target": "ES2024", + "lib": ["ES2024", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "skipLibCheck": true, + "isolatedModules": true, + "resolveJsonModule": true, + "types": ["node"], + "noEmit": true + }, + "include": ["src", "test", "drizzle.config.ts"] +} diff --git a/packages/server/vitest.config.ts b/packages/server/vitest.config.ts new file mode 100644 index 0000000..3e42797 --- /dev/null +++ b/packages/server/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + include: ['test/**/*.test.ts'], + }, +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b33f653..a40ce42 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,7 +16,7 @@ importers: version: 3.1104.0 '@better-auth/api-key': specifier: ^1.6.11 - version: 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)))) + version: 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)))) '@codemirror/autocomplete': specifier: ^6.20.1 version: 6.20.2 @@ -49,7 +49,7 @@ importers: version: 3.0.1(ajv@8.20.0) better-auth: specifier: ^1.6.11 - version: 1.6.11(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) + version: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) better-sqlite3: specifier: ^12.9.0 version: 12.9.0 @@ -58,7 +58,7 @@ importers: version: 14.0.3 drizzle-orm: specifier: ^0.45.0 - version: 0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) + version: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) hono: specifier: ^4 version: 4.12.18 @@ -170,10 +170,23 @@ importers: version: 0.25.12 packages/core: + dependencies: + '@cfworker/json-schema': + specifier: ^4.1.1 + version: 4.1.1 devDependencies: + '@hyperjump/json-schema': + specifier: ^1.17.9 + version: 1.17.9(@hyperjump/browser@1.5.1) '@types/node': specifier: ^25.0.0 version: 25.6.2 + ajv: + specifier: ^8.20.0 + version: 8.20.0 + ajv-formats: + specifier: ^3.0.1 + version: 3.0.1(ajv@8.20.0) fast-check: specifier: ^4.3.0 version: 4.10.2 @@ -187,6 +200,46 @@ importers: specifier: ^4.1.6 version: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + packages/server: + dependencies: + '@hono/node-server': + specifier: ^1.19.14 + version: 1.19.14(hono@4.12.18) + '@libsql/client': + specifier: ^0.18.0 + version: 0.18.0 + '@underlay/core': + specifier: workspace:* + version: link:../core + aws4fetch: + specifier: ^1.0.20 + version: 1.0.20 + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) + hono: + specifier: ^4.12.18 + version: 4.12.18 + devDependencies: + '@cloudflare/workers-types': + specifier: ^5.20261003.1 + version: 5.20261003.1 + '@types/node': + specifier: ^25.0.0 + version: 25.6.2 + drizzle-kit: + specifier: ^0.31.10 + version: 0.31.10 + tsx: + specifier: ^4.19.0 + version: 4.21.0 + typescript: + specifier: ^6.0.0 + version: 6.0.3 + vitest: + specifier: ^4.1.6 + version: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + packages: '@asamuzakjp/css-color@5.1.11': @@ -560,6 +613,12 @@ packages: resolution: {integrity: sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==} hasBin: true + '@cfworker/json-schema@4.1.1': + resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} + + '@cloudflare/workers-types@5.20261003.1': + resolution: {integrity: sha512-Uii0J5qUd/R6lIDK/u0Sw8dEi7oI3ljXw7EDybuPntoXg2Yy5Y9H9UeNQ5R87oQ/Q4jq6uEJmYOOFQ3Z7C94lA==} + '@codemirror/autocomplete@6.20.2': resolution: {integrity: sha512-G5FPkgIiLjOgZMjqVjvuKQ1rGPtHogLldJr33eFJdVLtmwY+giGrlv/ewljLz6b9BSQLkjxuwBc6g6omDM+YxQ==} @@ -1090,6 +1149,27 @@ packages: hono: '>=3.9.0' zod: ^3.25.0 || ^4.0.0 + '@hyperjump/browser@1.5.1': + resolution: {integrity: sha512-dMuhTjbQLQbF5IcsWP1kEfI7B0QuH/j9RaaV8y9orfLg1ZRcLMRRGUWMKUQ7Ox7I5Fi4x/+XdxdjgaMY8WRhYA==} + engines: {node: '>=18.0.0'} + + '@hyperjump/json-pointer@1.1.3': + resolution: {integrity: sha512-GVSs+wzefSO5VOEVMuOz118MDJrJtMdm8h+2vlghzMG/5QpboTSQ12gaMKvGQo7S4oPMw9rCyFWNAJ9DUEdA5Q==} + + '@hyperjump/json-schema-formats@1.0.8': + resolution: {integrity: sha512-iE+U2Jbe4m3k7t6YXrQe9VhhTgCg6rTL7I1Pdc5soGCvXGFfuaNvzlWaeQI2CnGFscnEAvM2jkas5g1QfTdrTw==} + + '@hyperjump/json-schema@1.17.9': + resolution: {integrity: sha512-09H1fd51Zu5xFEUEutLTGQK5/bFfJVovWhT/RK1R35e/Zf3Sw3nxBDZj11LDcDWDjE5MVfvgCg4fy81F4RbGZA==} + peerDependencies: + '@hyperjump/browser': ^1.1.0 + + '@hyperjump/pact@1.4.0': + resolution: {integrity: sha512-01Q7VY6BcAkp9W31Fv+ciiZycxZHGlR2N6ba9BifgyclHYHdbaZgITo0U6QMhYRlem4k8pf8J31/tApxvqAz8A==} + + '@hyperjump/uri@1.3.8': + resolution: {integrity: sha512-pQ1IFUIdUrMzj6TjG0wyaAGZ0Go2bi4eFS8EydFLuBKGYpU0sTjd/R1td+lqhn3hCq7G17qZp975WL87MU84dw==} + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -1115,9 +1195,69 @@ packages: '@lezer/lr@1.4.10': resolution: {integrity: sha512-rnCpTIBafOx4mRp43xOxDJbFipJm/c0cia/V5TiGlhmMa+wsSdoGmUN3w5Bqrks/09Q/D4tNAmWaT8p6NRi77A==} + '@libsql/client@0.18.0': + resolution: {integrity: sha512-zMCCo58vv7w27j7+cnt/GW+X6/Rih6cHZ86PKi0AxCMOqMfrVgG9OZ5lW2bDMJhSeOjsv2szH2nJF3A9l36UmA==} + + '@libsql/core@0.18.0': + resolution: {integrity: sha512-N8RR2CIpcgtbKZnXs2KVpw2lJQfinUj3I56e7qRDRpK18Hty0Rr8nr00VmFb4R/DsqOanPB/TaP3+QAQLZeXMg==} + + '@libsql/darwin-arm64@0.5.29': + resolution: {integrity: sha512-K+2RIB1OGFPYQbfay48GakLhqf3ArcbHqPFu7EZiaUcRgFcdw8RoltsMyvbj5ix2fY0HV3Q3Ioa/ByvQdaSM0A==} + cpu: [arm64] + os: [darwin] + + '@libsql/darwin-x64@0.5.29': + resolution: {integrity: sha512-OtT+KFHsKFy1R5FVadr8FJ2Bb1mghtXTyJkxv0trocq7NuHntSki1eUbxpO5ezJesDvBlqFjnWaYYY516QNLhQ==} + cpu: [x64] + os: [darwin] + + '@libsql/hrana-client@0.10.0': + resolution: {integrity: sha512-OoA4EMqRAC7kn7V2P6EQqRcpZf2W+AjsNIyCizBg339Tq/aMC7sRnzs3SklderhmQWAqEzvv8A2vhxVmWpkVvw==} + + '@libsql/isomorphic-ws@0.1.5': + resolution: {integrity: sha512-DtLWIH29onUYR00i0GlQ3UdcTRC6EP4u9w/h9LxpUZJWRMARk6dQwZ6Jkd+QdwVpuAOrdxt18v0K2uIYR3fwFg==} + + '@libsql/linux-arm-gnueabihf@0.5.29': + resolution: {integrity: sha512-CD4n4zj7SJTHso4nf5cuMoWoMSS7asn5hHygsDuhRl8jjjCTT3yE+xdUvI4J7zsyb53VO5ISh4cwwOtf6k2UhQ==} + cpu: [arm] + os: [linux] + + '@libsql/linux-arm-musleabihf@0.5.29': + resolution: {integrity: sha512-2Z9qBVpEJV7OeflzIR3+l5yAd4uTOLxklScYTwpZnkm2vDSGlC1PRlueLaufc4EFITkLKXK2MWBpexuNJfMVcg==} + cpu: [arm] + os: [linux] + + '@libsql/linux-arm64-gnu@0.5.29': + resolution: {integrity: sha512-gURBqaiXIGGwFNEaUj8Ldk7Hps4STtG+31aEidCk5evMMdtsdfL3HPCpvys+ZF/tkOs2MWlRWoSq7SOuCE9k3w==} + cpu: [arm64] + os: [linux] + + '@libsql/linux-arm64-musl@0.5.29': + resolution: {integrity: sha512-fwgYZ0H8mUkyVqXZHF3mT/92iIh1N94Owi/f66cPVNsk9BdGKq5gVpoKO+7UxaNzuEH1roJp2QEwsCZMvBLpqg==} + cpu: [arm64] + os: [linux] + + '@libsql/linux-x64-gnu@0.5.29': + resolution: {integrity: sha512-y14V0vY0nmMC6G0pHeJcEarcnGU2H6cm21ZceRkacWHvQAEhAG0latQkCtoS2njFOXiYIg+JYPfAoWKbi82rkg==} + cpu: [x64] + os: [linux] + + '@libsql/linux-x64-musl@0.5.29': + resolution: {integrity: sha512-gquqwA/39tH4pFl+J9n3SOMSymjX+6kZ3kWgY3b94nXFTwac9bnFNMffIomgvlFaC4ArVqMnOZD3nuJ3H3VO1w==} + cpu: [x64] + os: [linux] + + '@libsql/win32-x64-msvc@0.5.29': + resolution: {integrity: sha512-4/0CvEdhi6+KjMxMaVbFM2n2Z44escBRoEYpR+gZg64DdetzGnYm8mcNLcoySaDJZNaBd6wz5DNdgRmcI4hXcg==} + cpu: [x64] + os: [win32] + '@marijn/find-cluster-break@1.0.2': resolution: {integrity: sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==} + '@neon-rs/load@0.0.4': + resolution: {integrity: sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==} + '@noble/ciphers@2.2.0': resolution: {integrity: sha512-Z6pjIZ/8IJcCGzb2S/0Px5J81yij85xASuk1teLNeg75bfT07MV3a/O2Mtn1I2se43k3lkVEcFaR10N4cgQcZA==} engines: {node: '>= 20.19.0'} @@ -1982,6 +2122,9 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + aws4fetch@1.0.20: + resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} + b4a@1.8.1: resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==} peerDependencies: @@ -2161,6 +2304,10 @@ packages: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} @@ -2205,6 +2352,10 @@ packages: defu@6.1.7: resolution: {integrity: sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==} + detect-libc@2.0.2: + resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==} + engines: {node: '>=8'} + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -2448,6 +2599,10 @@ packages: resolution: {integrity: sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} + idn-hostname@15.1.19: + resolution: {integrity: sha512-pRGRbFpQWCptOBWw9Gn5DXtPdern5/XtdkcXdoEavjd+sh72jNm2eVI1eeysp64UisXPfaqoQrnLUKpSN4hJWw==} + engines: {node: '>=20.12.0 <21 || >=22.0.0'} + ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} @@ -2475,6 +2630,9 @@ packages: jose@6.2.3: resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} + js-base64@3.9.4: + resolution: {integrity: sha512-PtOMXpEGuP0RRiRXsjzHzl44dMHxSu2CPvAhinupR1tBa88me+1DPqsobl7eupn5UukjLkln1ZnAOAOXOrYG0Q==} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -2500,10 +2658,18 @@ packages: engines: {node: '>=6'} hasBin: true + just-curry-it@5.3.0: + resolution: {integrity: sha512-silMIRiFjUWlfaDhkgSzpuAyQ6EX/o09Eu8ZBfmFwQMbax7+LQzeIU2CBrICT6Ne4l86ITCGvUCBpCubWYy0Yw==} + kysely@0.28.17: resolution: {integrity: sha512-nbD8lB9EB3wNdMhOCdx5Li8DxnLbvKByylRLcJ1h+4SkrowVeECAyZlyiKMThF7xFdRz0jSQ2MoJr+wXux2y0Q==} engines: {node: '>=20.0.0'} + libsql@0.5.29: + resolution: {integrity: sha512-8lMP8iMgiBzzoNbAPQ59qdVcj6UaE/Vnm+fiwX4doX4Narook0a4GPKWBEv+CR8a1OwbfkgL18uBfBjWdF0Fzg==} + cpu: [x64, arm64, wasm32, arm] + os: [darwin, linux, win32] + lightningcss-android-arm64@1.32.0: resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} engines: {node: '>= 12.0.0'} @@ -2726,6 +2892,9 @@ packages: deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. hasBin: true + promise-limit@2.7.0: + resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==} + pump@3.0.4: resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} @@ -3775,14 +3944,14 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.28.5 - '@better-auth/api-key@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))))': + '@better-auth/api-key@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))))': dependencies: - '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) + '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/utils': 0.4.0 - better-auth: 1.6.11(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) + better-auth: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) zod: 4.4.3 - '@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)': + '@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)': dependencies: '@better-auth/utils': 0.4.0 '@better-fetch/fetch': 1.1.21 @@ -3793,39 +3962,41 @@ snapshots: kysely: 0.28.17 nanostores: 1.3.0 zod: 4.4.3 + optionalDependencies: + '@cloudflare/workers-types': 5.20261003.1 - '@better-auth/drizzle-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))': + '@better-auth/drizzle-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))': dependencies: - '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) + '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/utils': 0.4.0 optionalDependencies: - drizzle-orm: 0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) + drizzle-orm: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) - '@better-auth/kysely-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(kysely@0.28.17)': + '@better-auth/kysely-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(kysely@0.28.17)': dependencies: - '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) + '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/utils': 0.4.0 optionalDependencies: kysely: 0.28.17 - '@better-auth/memory-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)': + '@better-auth/memory-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)': dependencies: - '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) + '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/utils': 0.4.0 - '@better-auth/mongo-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)': + '@better-auth/mongo-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)': dependencies: - '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) + '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/utils': 0.4.0 - '@better-auth/prisma-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)': + '@better-auth/prisma-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)': dependencies: - '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) + '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/utils': 0.4.0 - '@better-auth/telemetry@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)': + '@better-auth/telemetry@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)': dependencies: - '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) + '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/utils': 0.4.0 '@better-fetch/fetch': 1.1.21 @@ -3839,6 +4010,10 @@ snapshots: dependencies: css-tree: 3.2.1 + '@cfworker/json-schema@4.1.1': {} + + '@cloudflare/workers-types@5.20261003.1': {} + '@codemirror/autocomplete@6.20.2': dependencies: '@codemirror/language': 6.12.3 @@ -4153,6 +4328,34 @@ snapshots: hono: 4.12.18 zod: 4.4.3 + '@hyperjump/browser@1.5.1': + dependencies: + '@hyperjump/json-pointer': 1.1.3 + '@hyperjump/uri': 1.3.8 + content-type: 1.0.5 + + '@hyperjump/json-pointer@1.1.3': {} + + '@hyperjump/json-schema-formats@1.0.8': + dependencies: + '@hyperjump/uri': 1.3.8 + idn-hostname: 15.1.19 + + '@hyperjump/json-schema@1.17.9(@hyperjump/browser@1.5.1)': + dependencies: + '@hyperjump/browser': 1.5.1 + '@hyperjump/json-pointer': 1.1.3 + '@hyperjump/json-schema-formats': 1.0.8 + '@hyperjump/pact': 1.4.0 + '@hyperjump/uri': 1.3.8 + content-type: 1.0.5 + just-curry-it: 5.3.0 + uuid: 14.0.0 + + '@hyperjump/pact@1.4.0': {} + + '@hyperjump/uri@1.3.8': {} + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -4182,8 +4385,68 @@ snapshots: dependencies: '@lezer/common': 1.5.2 + '@libsql/client@0.18.0': + dependencies: + '@libsql/core': 0.18.0 + '@libsql/hrana-client': 0.10.0 + js-base64: 3.9.4 + libsql: 0.5.29 + promise-limit: 2.7.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@libsql/core@0.18.0': + dependencies: + js-base64: 3.9.4 + + '@libsql/darwin-arm64@0.5.29': + optional: true + + '@libsql/darwin-x64@0.5.29': + optional: true + + '@libsql/hrana-client@0.10.0': + dependencies: + '@libsql/isomorphic-ws': 0.1.5 + js-base64: 3.9.4 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@libsql/isomorphic-ws@0.1.5': + dependencies: + '@types/ws': 8.18.1 + ws: 8.20.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@libsql/linux-arm-gnueabihf@0.5.29': + optional: true + + '@libsql/linux-arm-musleabihf@0.5.29': + optional: true + + '@libsql/linux-arm64-gnu@0.5.29': + optional: true + + '@libsql/linux-arm64-musl@0.5.29': + optional: true + + '@libsql/linux-x64-gnu@0.5.29': + optional: true + + '@libsql/linux-x64-musl@0.5.29': + optional: true + + '@libsql/win32-x64-msvc@0.5.29': + optional: true + '@marijn/find-cluster-break@1.0.2': {} + '@neon-rs/load@0.0.4': {} + '@noble/ciphers@2.2.0': {} '@noble/hashes@2.2.0': {} @@ -4968,6 +5231,8 @@ snapshots: assertion-error@2.0.1: {} + aws4fetch@1.0.20: {} + b4a@1.8.1: {} babel-plugin-react-compiler@1.0.0: @@ -5010,15 +5275,15 @@ snapshots: baseline-browser-mapping@2.10.29: {} - better-auth@1.6.11(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))): + better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))): dependencies: - '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) - '@better-auth/drizzle-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1)) - '@better-auth/kysely-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(kysely@0.28.17) - '@better-auth/memory-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0) - '@better-auth/mongo-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0) - '@better-auth/prisma-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0) - '@better-auth/telemetry': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21) + '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) + '@better-auth/drizzle-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1)) + '@better-auth/kysely-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(kysely@0.28.17) + '@better-auth/memory-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0) + '@better-auth/mongo-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0) + '@better-auth/prisma-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0) + '@better-auth/telemetry': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21) '@better-auth/utils': 0.4.0 '@better-fetch/fetch': 1.1.21 '@noble/ciphers': 2.2.0 @@ -5032,7 +5297,7 @@ snapshots: optionalDependencies: better-sqlite3: 12.9.0 drizzle-kit: 0.31.10 - drizzle-orm: 0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) + drizzle-orm: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) vitest: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) @@ -5102,6 +5367,8 @@ snapshots: commander@14.0.3: {} + content-type@1.0.5: {} + convert-source-map@2.0.0: {} cookie@1.1.1: {} @@ -5136,6 +5403,8 @@ snapshots: defu@6.1.7: {} + detect-libc@2.0.2: {} + detect-libc@2.1.2: {} dompurify@3.4.8: @@ -5149,8 +5418,10 @@ snapshots: esbuild: 0.25.12 tsx: 4.21.0 - drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1): + drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1): optionalDependencies: + '@cloudflare/workers-types': 5.20261003.1 + '@libsql/client': 0.18.0 '@types/better-sqlite3': 7.6.13 better-sqlite3: 12.9.0 kysely: 0.28.17 @@ -5351,6 +5622,10 @@ snapshots: transitivePeerDependencies: - '@noble/hashes' + idn-hostname@15.1.19: + dependencies: + punycode: 2.3.1 + ieee754@1.2.1: {} inherits@2.0.4: {} @@ -5375,6 +5650,8 @@ snapshots: jose@6.2.3: {} + js-base64@3.9.4: {} + js-tokens@4.0.0: {} jsdom@29.1.1(@noble/hashes@2.2.0): @@ -5409,8 +5686,25 @@ snapshots: json5@2.2.3: {} + just-curry-it@5.3.0: {} + kysely@0.28.17: {} + libsql@0.5.29: + dependencies: + '@neon-rs/load': 0.0.4 + detect-libc: 2.0.2 + optionalDependencies: + '@libsql/darwin-arm64': 0.5.29 + '@libsql/darwin-x64': 0.5.29 + '@libsql/linux-arm-gnueabihf': 0.5.29 + '@libsql/linux-arm-musleabihf': 0.5.29 + '@libsql/linux-arm64-gnu': 0.5.29 + '@libsql/linux-arm64-musl': 0.5.29 + '@libsql/linux-x64-gnu': 0.5.29 + '@libsql/linux-x64-musl': 0.5.29 + '@libsql/win32-x64-msvc': 0.5.29 + lightningcss-android-arm64@1.32.0: optional: true @@ -5620,6 +5914,8 @@ snapshots: tar-fs: 2.1.4 tunnel-agent: 0.6.0 + promise-limit@2.7.0: {} + pump@3.0.4: dependencies: end-of-stream: 1.4.5 From 46f9b7a9a5a867b81e9ee4fb424f4c755b66026c Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 16:36:57 -0400 Subject: [PATCH 004/178] =?UTF-8?q?v2:=20placements=20=E2=80=94=20@underla?= =?UTF-8?q?y/repo,=20per-collection=20stores,=20version=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows the plan's placements update (primary + mirrors, documented repository layout, signed version log): - New @underlay/repo: repository reads and writes through a BlobStore in the documented layout (nodes/, bodies/.ndjson.gz, records/.json.gz, roots/, schemas/, private/, files/, collections//{collection,head}.json and log/.json), with hash verification for untrusted locations. Blob adapters, gzip and the LRU move here from the server. - Large leaf bodies are one object of concatenated gzip members (replacing separate part objects); RepoSink records every key it writes, which is a commit's mirror sync list. - Signed, hash-chained version log (Ed25519 via WebCrypto) with verifyLog for restore and audits. - Server: storage_locations and placements tables (platform location seeded; one primary per collection enforced), and Stores replaces the global blob store: forCollection(id) resolves the primary, internal holds sessions/uploads/reference log. - Spec: repository layout and version log (section 11). --- docs/protocol-v2.md | 75 +- packages/repo/package.json | 26 + packages/{server => repo}/src/blob/fs.ts | 2 +- packages/{server => repo}/src/blob/memory.ts | 2 +- packages/{server => repo}/src/blob/s3.ts | 2 +- packages/{server/src/lib => repo/src}/gzip.ts | 0 packages/repo/src/index.ts | 5 + packages/repo/src/log.ts | 216 ++ packages/{server/src/lib => repo/src}/lru.ts | 0 packages/repo/src/repo.ts | 398 +++ packages/repo/src/types.ts | 115 + packages/{server => repo}/test/blob.test.ts | 2 +- packages/{server => repo}/test/fake-s3.ts | 0 packages/repo/test/log.test.ts | 68 + packages/repo/test/objects.test.ts | 191 ++ packages/repo/tsconfig.json | 17 + packages/repo/vitest.config.ts | 7 + packages/server/drizzle/0000_init.sql | 37 + .../server/drizzle/0001_platform_location.sql | 4 + .../server/drizzle/meta/0000_snapshot.json | 250 +- .../server/drizzle/meta/0001_snapshot.json | 2736 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 9 +- packages/server/package.json | 2 +- packages/server/src/cache.ts | 9 +- packages/server/src/db/schema.ts | 70 + packages/server/src/node/main.ts | 15 +- packages/server/src/ports.ts | 84 +- packages/server/src/storage/objects.ts | 375 --- packages/server/src/stores.ts | 79 + packages/server/src/worker.ts | 28 +- packages/server/test/db.test.ts | 51 +- packages/server/test/storage.test.ts | 142 - pnpm-lock.yaml | 25 +- 33 files changed, 4426 insertions(+), 616 deletions(-) create mode 100644 packages/repo/package.json rename packages/{server => repo}/src/blob/fs.ts (99%) rename packages/{server => repo}/src/blob/memory.ts (99%) rename packages/{server => repo}/src/blob/s3.ts (99%) rename packages/{server/src/lib => repo/src}/gzip.ts (100%) create mode 100644 packages/repo/src/index.ts create mode 100644 packages/repo/src/log.ts rename packages/{server/src/lib => repo/src}/lru.ts (100%) create mode 100644 packages/repo/src/repo.ts create mode 100644 packages/repo/src/types.ts rename packages/{server => repo}/test/blob.test.ts (99%) rename packages/{server => repo}/test/fake-s3.ts (100%) create mode 100644 packages/repo/test/log.test.ts create mode 100644 packages/repo/test/objects.test.ts create mode 100644 packages/repo/tsconfig.json create mode 100644 packages/repo/vitest.config.ts create mode 100644 packages/server/drizzle/0001_platform_location.sql create mode 100644 packages/server/drizzle/meta/0001_snapshot.json delete mode 100644 packages/server/src/storage/objects.ts create mode 100644 packages/server/src/stores.ts delete mode 100644 packages/server/test/storage.test.ts diff --git a/docs/protocol-v2.md b/docs/protocol-v2.md index bfc65bc..287be2f 100644 --- a/docs/protocol-v2.md +++ b/docs/protocol-v2.md @@ -220,11 +220,78 @@ What each reader can check: From the root they learn only whether a private set exists. - Owners also get the private set object, salt included, and can check it against the commitment. -## 11. Limits and constants +## 11. Repository layout + +A repository is how a storage location holds collections. It is the same on the platform's own +bucket, on a customer's mirror and in a restore source. Keys are relative to the location's prefix. +The reference implementation is `packages/repo` (`@underlay/repo`). + +``` +nodes/ node JSON (section 8.2), gzip-compressed +bodies/.ndjson.gz a record leaf's records +records/.json.gz an out-of-line record, gzip +schemas/.json JCS(schema) +roots/.json JCS(root); is the version hash without "ulv2:" +private/.json JCS(private set object); only in locations that hold private sets +files/ file bytes +collections//collection.json +collections//log/.json +collections//head.json +``` + +- **Bodies.** The body of leaf L has one line per entry of L, in entry order, each followed by + `\n`. The body is one or more gzip members concatenated (RFC 1952 §2.2); readers must accept any + number of members. + - A line is either the canonical record (section 4), whose hash is the entry's record hash, or + an out-of-line pointer `{"$ref":""}`, whose record is stored in `records/`. + - Which records go out of line, and where members split, are the writer's choice. + - For a body with no pointers, the concatenation of a type's bodies in tree order is that type's + records as gzip NDJSON. +- **Content-addressed objects** (everything except `collections/`) never change once written. + Readers that don't trust a location verify each object before use: + - nodes against their hash; + - body lines against the leaf's record hashes; + - roots against the version hash; + - schemas and private set objects against theirs. +- **Write order.** All objects a version reaches (leaves and bodies, then interior nodes, then the + root and private set object), then the log entry, then `head.json`. A reader that finds + `head.json` can read everything below it. +- **Self-contained.** Nothing in a location refers to another location. +- Platform-internal data (push sessions, staging uploads, the reference log) is not part of a + repository and is never copied to one. + +### 11.1 Version log + +Each collection has one log entry per version: + +``` +entry = {"actorId","appId","baseSemver","createdAt","keyId","message","prev","semver","seq","sig","versionHash"} +``` + +- `seq` counts from 1. `createdAt` is ISO 8601 UTC. `appId`, `actorId`, `baseSemver` and + `message` may be `null`. Pusher identity is not recorded (open question). +- `sig` is base64url (no padding) of the Ed25519 signature over the UTF-8 bytes of JCS(entry + without `sig`). +- `keyId` names the signing key. It is the first 16 hex characters of hash(raw public key). +- **Entry hash** = hash(JCS(entry)), signature included. +- `prev` is the entry hash of entry `seq − 1`, or `null` for `seq` 1. Entries form a hash chain, + so a dropped, reordered or altered entry is detectable. +- `head.json` = JCS(`{"entryHash","seq","versionHash"}`) of the latest entry. It is overwritten + after the entry is written. +- `collection.json` holds the collection's id, owner and slug, its name and description, and + `keys`: the public keys (`{"id","alg":"Ed25519","publicKey": base64url raw}`) that sign its log. + The platform also publishes its keys at a well-known URL (to be fixed with the Cloudflare + deployment). + +A log is valid when every entry is present from 1 to `head.seq`, each `prev` chains, each signature +verifies against a trusted key, and `head.entryHash` is the last entry's hash (`verifyLog` in +`packages/repo/src/log.ts`). + +## 12. Limits and constants All protocol constants are in `packages/core/src/constants.ts`, and the vectors file repeats them. -## 12. Format 1 hashes +## 13. Format 1 hashes Format 1 canonicalized `data` and schemas by sorting keys into a new object and then calling `JSON.stringify`. That puts array-index keys (canonical decimal integers below 2³² − 1) first, in @@ -270,3 +337,7 @@ These were made during implementation and recorded with their reasons in `edge-r 6. `LEAF_MAX_ENTRIES` is 8,192 (the plan had 16,384). The chunking rule stays fixed-probability rather than size-aware, because size-aware boundaries depend on position and that rules out parallel commit units. +7. Roots are stored as `roots/.json`, without the `ulv2:` prefix, which would put a colon in + the key. +8. A large leaf body is one object of several concatenated gzip members (section 11). There are no + separate part objects, so mirrors and third-party readers need only one rule. diff --git a/packages/repo/package.json b/packages/repo/package.json new file mode 100644 index 0000000..82e38d2 --- /dev/null +++ b/packages/repo/package.json @@ -0,0 +1,26 @@ +{ + "name": "@underlay/repo", + "version": "0.0.0", + "private": true, + "description": "Read and write an Underlay repository (the documented object layout and signed version log) through a BlobStore. No SQLite, no auth: used by the server, mirror sync, restore and mirror frontends.", + "type": "module", + "exports": { + ".": "./src/index.ts", + "./blob/memory": "./src/blob/memory.ts", + "./blob/s3": "./src/blob/s3.ts", + "./blob/fs": "./src/blob/fs.ts" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@underlay/core": "workspace:*", + "aws4fetch": "^1.0.20" + }, + "devDependencies": { + "@types/node": "^25.0.0", + "typescript": "^6.0.0", + "vitest": "^4.1.6" + } +} diff --git a/packages/server/src/blob/fs.ts b/packages/repo/src/blob/fs.ts similarity index 99% rename from packages/server/src/blob/fs.ts rename to packages/repo/src/blob/fs.ts index 7a2c927..7446b31 100644 --- a/packages/server/src/blob/fs.ts +++ b/packages/repo/src/blob/fs.ts @@ -19,7 +19,7 @@ import type { PresignGetOptions, PresignPutOptions, PutOptions, -} from '../ports.js' +} from '../types.js' export interface FsBlobConfig { root: string diff --git a/packages/server/src/blob/memory.ts b/packages/repo/src/blob/memory.ts similarity index 99% rename from packages/server/src/blob/memory.ts rename to packages/repo/src/blob/memory.ts index 195dbac..e23f14a 100644 --- a/packages/server/src/blob/memory.ts +++ b/packages/repo/src/blob/memory.ts @@ -1,4 +1,4 @@ -import type { BlobHead, BlobObject, BlobStore, PutOptions } from '../ports.js' +import type { BlobHead, BlobObject, BlobStore, PutOptions } from '../types.js' const enc = new TextEncoder() diff --git a/packages/server/src/blob/s3.ts b/packages/repo/src/blob/s3.ts similarity index 99% rename from packages/server/src/blob/s3.ts rename to packages/repo/src/blob/s3.ts index 4eba042..88740ff 100644 --- a/packages/server/src/blob/s3.ts +++ b/packages/repo/src/blob/s3.ts @@ -12,7 +12,7 @@ import type { PresignGetOptions, PresignPutOptions, PutOptions, -} from '../ports.js' +} from '../types.js' export interface S3Config { endpoint: string diff --git a/packages/server/src/lib/gzip.ts b/packages/repo/src/gzip.ts similarity index 100% rename from packages/server/src/lib/gzip.ts rename to packages/repo/src/gzip.ts diff --git a/packages/repo/src/index.ts b/packages/repo/src/index.ts new file mode 100644 index 0000000..71b1c74 --- /dev/null +++ b/packages/repo/src/index.ts @@ -0,0 +1,5 @@ +export * from './types.js' +export * from './repo.js' +export * from './log.js' +export { gunzip, gunzipText, gzip, isGzip, splitLines } from './gzip.js' +export { Lru } from './lru.js' diff --git a/packages/repo/src/log.ts b/packages/repo/src/log.ts new file mode 100644 index 0000000..932b1c2 --- /dev/null +++ b/packages/repo/src/log.ts @@ -0,0 +1,216 @@ +/** + * The per-collection version log: one signed, hash-chained entry per version. + * + * It lets anyone holding a copy (a mirror, a restore, a frontend) check the + * version history without the platform database, and it is the per-collection + * head log from the plan's Security notes: entries chain by `prev`, so a server + * can't silently drop or reorder versions for one reader and not another. + * + * entry = {seq, semver, versionHash, baseSemver, message, appId, actorId, + * createdAt, prev, keyId, sig} + * signed = JCS(entry without "sig"), Ed25519, "sig" = base64url + * entryHash = sha256(JCS(entry)) (with "sig") + * prev = entryHash of seq − 1, or null for seq 1 + * head.json = {"seq", "entryHash", "versionHash"}, overwritten after the entry is written + * + * Pusher identity is omitted from the log (an open question in the plan). + */ +import { jcs, sha256Hex } from '@underlay/core' + +import { IntegrityError, keys, type Repo } from './repo.js' + +export interface LogEntry { + seq: number + semver: string + versionHash: string + baseSemver: string | null + message: string | null + appId: string | null + actorId: string | null + /** ISO 8601, UTC. */ + createdAt: string + prev: string | null + keyId: string + sig: string +} + +export type UnsignedEntry = Omit + +export interface Head { + seq: number + entryHash: string + versionHash: string +} + +export interface CollectionInfo { + id: string + owner: string + slug: string + name: string + /** Public keys that sign this collection's log. */ + keys: PublicKeyInfo[] + [k: string]: unknown +} + +export interface PublicKeyInfo { + id: string + alg: 'Ed25519' + /** Raw 32-byte public key, base64url. */ + publicKey: string +} + +const b64url = (bytes: Uint8Array) => + btoa(String.fromCharCode(...bytes)) + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/, '') +const fromB64url = (s: string) => + Uint8Array.from(atob(s.replace(/-/g, '+').replace(/_/g, '/')), (c) => c.charCodeAt(0)) +const enc = new TextEncoder() + +export const entryHash = (e: LogEntry) => sha256Hex(jcs(e)) + +/** The bytes that are signed: the entry without its signature. */ +const signedBytes = (e: Omit) => enc.encode(jcs(e)) + +export interface Signer { + keyId: string + publicKey: PublicKeyInfo + sign(bytes: Uint8Array): Promise +} + +/** A signer from a raw 32-byte Ed25519 private key (seed), base64url. */ +export async function ed25519Signer(privateKeyB64: string): Promise { + // WebCrypto imports Ed25519 private keys as PKCS#8; wrap the seed. + const seed = fromB64url(privateKeyB64) + if (seed.length !== 32) throw new Error('Ed25519 private key must be 32 bytes') + const pkcs8 = new Uint8Array([ + 0x30, + 0x2e, + 0x02, + 0x01, + 0x00, + 0x30, + 0x05, + 0x06, + 0x03, + 0x2b, + 0x65, + 0x70, + 0x04, + 0x22, + 0x04, + 0x20, + ...seed, + ]) + const key = await crypto.subtle.importKey('pkcs8', pkcs8, { name: 'Ed25519' }, true, ['sign']) + const jwk = await crypto.subtle.exportKey('jwk', key) + const publicKey = fromB64url(jwk.x!) + const id = sha256Hex(publicKey).slice(0, 16) + return { + keyId: id, + publicKey: { id, alg: 'Ed25519', publicKey: b64url(publicKey) }, + sign: async (bytes) => + new Uint8Array(await crypto.subtle.sign('Ed25519', key, bytes as Uint8Array)), + } +} + +/** Generate a new signing key; returns the private key seed (base64url) to store as a secret. */ +export async function generateSigningKey(): Promise { + const pair = (await crypto.subtle.generateKey({ name: 'Ed25519' }, true, [ + 'sign', + 'verify', + ])) as CryptoKeyPair + const jwk = await crypto.subtle.exportKey('jwk', pair.privateKey) + return jwk.d! +} + +export async function signEntry(signer: Signer, e: UnsignedEntry): Promise { + const unsigned = { ...e, keyId: signer.keyId } + const sig = b64url(await signer.sign(signedBytes(unsigned))) + return { ...unsigned, sig } +} + +export async function verifyEntry(e: LogEntry, keys: PublicKeyInfo[]): Promise { + const k = keys.find((x) => x.id === e.keyId) + if (!k || k.alg !== 'Ed25519') return false + const pub = await crypto.subtle.importKey( + 'raw', + fromB64url(k.publicKey) as Uint8Array, + { name: 'Ed25519' }, + false, + ['verify'], + ) + const { sig, ...unsigned } = e + return crypto.subtle.verify( + 'Ed25519', + pub, + fromB64url(sig) as Uint8Array, + signedBytes(unsigned) as Uint8Array, + ) +} + +// --- Reading and writing the log in a repository --- + +const dec = new TextDecoder() + +async function readJson(repo: Repo, key: string): Promise { + const obj = await repo.blobs.get(key) + return obj ? (JSON.parse(dec.decode(await obj.bytes())) as T) : null +} + +export const readHead = (repo: Repo, collectionId: string) => + readJson(repo, keys.head(collectionId)) +export const readCollectionInfo = (repo: Repo, collectionId: string) => + readJson(repo, keys.collection(collectionId)) +export const readLogEntry = (repo: Repo, collectionId: string, seq: number) => + readJson(repo, keys.logEntry(collectionId, seq)) + +export async function writeCollectionInfo(repo: Repo, info: CollectionInfo): Promise { + await repo.blobs.put(keys.collection(info.id), JSON.stringify(info, null, 1), { + contentType: 'application/json', + }) +} + +/** + * Append a log entry and then move the head. Call only after every object the + * version reaches (nodes, bodies, root, private set) is written: a reader that + * finds head.json can always read everything below it. + */ +export async function appendLog(repo: Repo, collectionId: string, entry: LogEntry): Promise { + await repo.blobs.put(keys.logEntry(collectionId, entry.seq), jcs(entry), { + contentType: 'application/json', + ifAbsent: true, + }) + const head: Head = { seq: entry.seq, entryHash: entryHash(entry), versionHash: entry.versionHash } + await repo.blobs.put(keys.head(collectionId), jcs(head), { contentType: 'application/json' }) + return head +} + +/** + * Check a collection's whole log: signatures, the hash chain, and that the head + * matches the last entry. O(versions); for restore and audits. + */ +export async function verifyLog( + repo: Repo, + collectionId: string, + trustedKeys: PublicKeyInfo[], +): Promise<{ head: Head; entries: LogEntry[] }> { + const head = await readHead(repo, collectionId) + if (!head) throw new IntegrityError(`Collection ${collectionId} has no head`) + const entries: LogEntry[] = [] + let prev: string | null = null + for (let seq = 1; seq <= head.seq; seq++) { + const e = await readLogEntry(repo, collectionId, seq) + if (!e) throw new IntegrityError(`Log entry ${seq} is missing`) + if (e.seq !== seq) throw new IntegrityError(`Log entry ${seq} has seq ${e.seq}`) + if (e.prev !== prev) throw new IntegrityError(`Log entry ${seq} does not chain to ${seq - 1}`) + if (!(await verifyEntry(e, trustedKeys))) + throw new IntegrityError(`Log entry ${seq} has a bad signature`) + prev = entryHash(e) + entries.push(e) + } + if (prev !== head.entryHash) + throw new IntegrityError('head.json does not match the last log entry') + return { head, entries } +} diff --git a/packages/server/src/lib/lru.ts b/packages/repo/src/lru.ts similarity index 100% rename from packages/server/src/lib/lru.ts rename to packages/repo/src/lru.ts diff --git a/packages/repo/src/repo.ts b/packages/repo/src/repo.ts new file mode 100644 index 0000000..b76ce31 --- /dev/null +++ b/packages/repo/src/repo.ts @@ -0,0 +1,398 @@ +/** + * A repository: one storage location's objects, in the documented layout + * (docs/protocol-v2.md, "Repository layout"), read and written through a + * BlobStore. The platform primary, customer mirrors and restore all use this. + * + * nodes/ tree node JSON, gzip (hash of the uncompressed bytes) + * bodies/.ndjson.gz the leaf's records, one canonical record per line, in + * entry order; one or more concatenated gzip members + * records/.json.gz an out-of-line record; its body line is {"$ref":""} + * schemas/.json canonical JSON + * roots/.json version roots (the hex after "ulv2:") + * private/.json private set objects (public+private locations only) + * files/ file bytes + * collections//collection.json collection description and signing keys + * collections//log/.json signed version log entries + * collections//head.json the latest entry + * + * Content-addressed objects are immutable and cached forever: the isolate LRU, + * then the shared cache, then the bucket. Objects read from a location we don't + * operate (`trusted: false`) are hash-verified before they are used or cached. + * How a body is split into gzip members, and which records go out of line, are + * writer choices; readers handle any of them. + */ +import { + type DecodedNode, + decodeNode, + hashSchema, + jcs, + type NodeDesc, + type NodeSource, + type PrivateSetObject, + privateCommitment, + type RecordEntry, + recordTree, + sha256Hex, + type TreeSink, + type TreeSpec, + type VersionRoot, + versionDigest, + versionHash, +} from '@underlay/core' + +import { gunzipText, gzip, splitLines } from './gzip.js' +import { Lru } from './lru.js' +import { type BlobStore, type Cache, noCache } from './types.js' + +/** Records over this size are stored once under records/ (writer policy, not protocol). */ +export const OUT_OF_LINE_BYTES = 64 * 1024 +/** Raw bytes per gzip member when a leaf body is written in pieces (writer policy). */ +export const BODY_MEMBER_BYTES = 4 * 1024 * 1024 + +export const keys = { + node: (h: string) => `nodes/${h}`, + body: (leafHash: string) => `bodies/${leafHash}.ndjson.gz`, + record: (h: string) => `records/${h}.json.gz`, + schema: (h: string) => `schemas/${h}.json`, + root: (versionHashOrDigest: string) => + `roots/${versionHashOrDigest.includes(':') ? versionDigest(versionHashOrDigest) : versionHashOrDigest}.json`, + privateSet: (commitment: string) => `private/${commitment}.json`, + file: (h: string) => `files/${h}`, + collection: (id: string) => `collections/${id}/collection.json`, + logEntry: (id: string, seq: number) => `collections/${id}/log/${seq}.json`, + head: (id: string) => `collections/${id}/head.json`, +} + +const REF_PREFIX = '{"$ref":"' +export const outOfLinePointer = (recordHash: string) => `${REF_PREFIX}${recordHash}"}` + +const dec = new TextDecoder() + +/** Per-isolate memory shared by every request and every repository. */ +const isolateLru = new Lru(32 * 1024 * 1024, (v) => { + if (typeof v === 'string') return v.length * 2 + if (v && typeof v === 'object' && 'approxBytes' in v) + return (v as { approxBytes: number }).approxBytes + return 1024 +}) + +export class IntegrityError extends Error { + constructor(message: string) { + super(message) + this.name = 'IntegrityError' + } +} + +export interface RepoOptions { + cache?: Cache + /** Namespaces cache and LRU keys (the location id). */ + scope: string + /** False for locations we don't operate: verify every object before use. */ + trusted: boolean + lru?: Lru +} + +export class Repo { + readonly cache: Cache + readonly lru: Lru + readonly scope: string + readonly trusted: boolean + + constructor( + readonly blobs: BlobStore, + opts: RepoOptions, + ) { + this.cache = opts.cache ?? noCache + this.lru = opts.lru ?? isolateLru + this.scope = opts.scope + this.trusted = opts.trusted + } + + #ck(key: string) { + return `${this.scope}/${key}` + } + + /** An immutable object's bytes: shared cache, then the bucket. `verify` runs before caching. */ + async #immutable( + key: string, + verify?: (bytes: Uint8Array) => Promise, + ): Promise { + const cached = await this.cache.get(this.#ck(key)) + if (cached) return cached + const obj = await this.blobs.get(key) + if (!obj) return null + const bytes = await obj.bytes() + if (verify && !this.trusted) await verify(bytes) + await this.cache.put(this.#ck(key), bytes) + return bytes + } + + // --- Nodes --- + + async nodeJson(hash: string): Promise { + const key = keys.node(hash) + const hit = this.lru.get(this.#ck(key)) + if (typeof hit === 'string') return hit + const bytes = await this.#immutable(key, async (b) => { + if (sha256Hex(await gunzipText(b)) !== hash) + throw new IntegrityError(`Node ${hash} fails its hash`) + }) + if (!bytes) throw new Error(`Missing node ${hash}`) + const json = await gunzipText(bytes) + this.lru.set(this.#ck(key), json) + return json + } + + async putNode(hash: string, json: string): Promise { + await this.blobs.put(keys.node(hash), await gzip(json), { + contentType: 'application/gzip', + ifAbsent: true, + }) + } + + async decoded(spec: TreeSpec, hash: string): Promise> { + const key = this.#ck(`decoded:${spec.name}:${hash}`) + const hit = this.lru.get(key) + if (hit) return (hit as { node: DecodedNode }).node + const json = await this.nodeJson(hash) + // decodeNode checks the hash and the canonical encoding. + const node = decodeNode(spec, json, hash) + this.lru.set(key, { node, approxBytes: json.length * 3 }) + return node + } + + // --- Bodies --- + + /** + * A leaf's body lines, in entry order, with out-of-line records resolved. On an + * untrusted location, each line is checked against the leaf's record hashes. + */ + async bodyLines(leaf: { hash: string; entries: readonly RecordEntry[] }): Promise { + const key = keys.body(leaf.hash) + const hit = this.lru.get(this.#ck(key)) + if (hit) return (hit as { lines: string[] }).lines + const check = async (b: Uint8Array) => { + const lines = await this.#resolve(splitLines(await gunzipText(b))) + if (lines.length !== leaf.entries.length) + throw new IntegrityError(`Body of ${leaf.hash}: wrong line count`) + lines.forEach((l, i) => { + if (sha256Hex(l) !== leaf.entries[i]!.hash) + throw new IntegrityError(`Body of ${leaf.hash}: line ${i} fails its hash`) + }) + } + const bytes = await this.#immutable(key, check) + if (!bytes) throw new Error(`Missing body for leaf ${leaf.hash}`) + const lines = await this.#resolve(splitLines(await gunzipText(bytes))) + if (lines.length !== leaf.entries.length) { + throw new IntegrityError( + `Body of ${leaf.hash} has ${lines.length} lines for ${leaf.entries.length} entries`, + ) + } + this.lru.set(this.#ck(key), { lines, approxBytes: lines.reduce((n, l) => n + l.length * 2, 0) }) + return lines + } + + async #resolve(lines: string[]): Promise { + if (!lines.some((l) => l.startsWith(REF_PREFIX))) return lines + return Promise.all(lines.map((l) => (l.startsWith(REF_PREFIX) ? this.#outOfLine(l) : l))) + } + + async #outOfLine(pointer: string): Promise { + const hash = (JSON.parse(pointer) as { $ref: string }).$ref + const b = await this.#immutable(keys.record(hash), async (bytes) => { + if (sha256Hex(await gunzipText(bytes)) !== hash) + throw new IntegrityError(`Record ${hash} fails its hash`) + }) + if (!b) throw new Error(`Missing out-of-line record ${hash}`) + return gunzipText(b) + } + + /** Write a body from its gzip members (already compressed, in order). */ + async putBody(leafHash: string, members: readonly Uint8Array[]): Promise { + const total = members.reduce((n, m) => n + m.byteLength, 0) + const bytes = new Uint8Array(total) + let off = 0 + for (const m of members) { + bytes.set(m, off) + off += m.byteLength + } + await this.blobs.put(keys.body(leafHash), bytes, { + contentType: 'application/gzip', + ifAbsent: true, + }) + } + + /** Store a large record out of line; returns the pointer line that goes in the body. */ + async putOutOfLine(recordHash: string, canonical: string): Promise { + await this.blobs.put(keys.record(recordHash), await gzip(canonical), { + contentType: 'application/gzip', + ifAbsent: true, + }) + return outOfLinePointer(recordHash) + } + + // --- Roots, private sets, schemas --- + + async #json(key: string, verify: (text: string, value: T) => void): Promise { + const hit = this.lru.get(this.#ck(key)) + if (hit) return (hit as { value: T }).value + const bytes = await this.#immutable(key, async (b) => { + const text = dec.decode(b) + verify(text, JSON.parse(text) as T) + }) + if (!bytes) throw new Error(`Missing ${key}`) + const text = dec.decode(bytes) + const value = JSON.parse(text) as T + this.lru.set(this.#ck(key), { value, approxBytes: text.length * 3 }) + return value + } + + root(hash: string): Promise { + return this.#json(keys.root(hash), (_t, root) => { + if (versionHash(root) !== hash) throw new IntegrityError(`Root ${hash} fails its hash`) + }) + } + + async putRoot(root: VersionRoot): Promise { + const hash = versionHash(root) + await this.blobs.put(keys.root(hash), jcs(root), { + contentType: 'application/json', + ifAbsent: true, + }) + return hash + } + + privateSet(commitment: string): Promise { + return this.#json(keys.privateSet(commitment), (_t, set) => { + if (privateCommitment(set) !== commitment) + throw new IntegrityError(`Private set ${commitment} fails its hash`) + }) + } + + async putPrivateSet(set: PrivateSetObject): Promise { + const commitment = privateCommitment(set) + await this.blobs.put(keys.privateSet(commitment), jcs(set), { + contentType: 'application/json', + ifAbsent: true, + }) + return commitment + } + + schema(hash: string): Promise> { + return this.#json>(keys.schema(hash), (_t, s) => { + if (hashSchema(s) !== hash) throw new IntegrityError(`Schema ${hash} fails its hash`) + }) + } + + async putSchema(schema: unknown): Promise { + const hash = hashSchema(schema) + await this.blobs.put(keys.schema(hash), jcs(schema), { + contentType: 'application/json', + ifAbsent: true, + }) + return hash + } +} + +/** NodeSource over a repository. Record trees get bodies through `leafEntries`. */ +export class RepoSource implements NodeSource { + constructor( + readonly spec: TreeSpec, + readonly repo: Repo, + ) {} + + node(hash: string): Promise> { + return this.repo.decoded(this.spec, hash) + } + + async leafEntries(hash: string): Promise { + const node = await this.node(hash) + if (node.kind !== 'leaf') throw new Error(`Node ${hash} is not a leaf`) + if (this.spec !== (recordTree as TreeSpec)) return node.entries.slice() + const entries = node.entries as RecordEntry[] + const lines = await this.repo.bodyLines({ hash, entries }) + return entries.map((e, i) => ({ ...e, body: lines[i]! })) as E[] + } +} + +/** + * TreeSink that writes nodes (and, for record trees, bodies) to a repository. + * The builder is synchronous; writes run in the background with bounded + * concurrency. Call `drain()` between units of work and `flush()` before using + * the root. `written` lists every key written, in order, which is the commit's + * sync work for mirrors. + * + * A body is written once its leaf ends (its key is the leaf hash). Until then, + * spilled entries are held as compressed gzip members, so a large leaf costs its + * compressed size in memory rather than its raw size. + */ +export class RepoSink implements TreeSink { + readonly #pending = new Set>() + #error: unknown = null + #members: Promise[] = [] + readonly written: string[] = [] + + constructor( + readonly repo: Repo, + readonly opts: { bodyOf?: (e: E) => string; concurrency?: number } = {}, + ) {} + + #run(key: string, work: () => Promise) { + const p = work() + .catch((err) => { + this.#error ??= err + }) + .finally(() => this.#pending.delete(p)) + this.#pending.add(p) + this.written.push(key) + } + + #member(entries: readonly E[]): Promise { + const p = gzip(entries.map(this.opts.bodyOf!).join('\n') + '\n') + p.catch(() => {}) // surfaced when the body is written + return p + } + + spill(entries: readonly E[]): void { + if (this.opts.bodyOf) this.#members.push(this.#member(entries)) + } + + leaf(desc: NodeDesc, json: string, entries: readonly E[], spilled: number): void { + this.#run(keys.node(desc.hash), () => this.repo.putNode(desc.hash, json)) + if (!this.opts.bodyOf) return + const tail = entries.slice(spilled) + const members = [...this.#members, ...(tail.length > 0 ? [this.#member(tail)] : [])] + this.#members = [] + this.#run(keys.body(desc.hash), async () => + this.repo.putBody(desc.hash, await Promise.all(members)), + ) + } + + interior(desc: NodeDesc, json: string): void { + this.#run(keys.node(desc.hash), () => this.repo.putNode(desc.hash, json)) + } + + async drain(): Promise { + const limit = this.opts.concurrency ?? 16 + while (this.#pending.size >= limit) await Promise.race(this.#pending) + if (this.#error) throw this.#error + } + + async flush(): Promise { + while (this.#pending.size > 0) await Promise.race(this.#pending) + if (this.#error) throw this.#error + } +} + +export const bodyOfRecord = (e: RecordEntry): string => { + if (e.body === undefined) throw new Error(`Record ${e.key} has no body`) + return e.body +} + +/** Payload size for spilling: the body line's length. */ +export const recordPayloadBytes = (e: RecordEntry) => e.body?.length ?? 0 + +/** Clear a record entry's body once it is in a compressed member. */ +export const dropRecordBody = (e: RecordEntry) => { + delete e.body +} diff --git a/packages/repo/src/types.ts b/packages/repo/src/types.ts new file mode 100644 index 0000000..86dce9b --- /dev/null +++ b/packages/repo/src/types.ts @@ -0,0 +1,115 @@ +/** The storage interfaces a repository is read and written through. */ + +export interface BlobHead { + size: number + etag: string + contentType: string | null +} + +export interface BlobObject extends BlobHead { + body: ReadableStream + bytes(): Promise + text(): Promise +} + +export interface PutOptions { + contentType?: string + /** Only write if the key doesn't exist. Immutable keys make this an optimization. */ + ifAbsent?: boolean +} + +export interface PresignGetOptions { + expiresIn: number + /** Content-Disposition for the response. */ + disposition?: string + contentType?: string +} + +export interface PresignPutOptions { + expiresIn: number + contentType?: string +} + +/** An S3-like bucket. Adapters: S3 API (R2, S3, MinIO), filesystem, memory. */ +export interface BlobStore { + get(key: string, range?: { offset: number; length?: number }): Promise + head(key: string): Promise + put(key: string, body: Uint8Array | string, opts?: PutOptions): Promise + delete(key: string): Promise + list(prefix: string, cursor?: string): Promise<{ keys: string[]; cursor?: string }> + presignGet(key: string, opts: PresignGetOptions): Promise + presignPut(key: string, opts: PresignPutOptions): Promise + createMultipart(key: string, contentType?: string): Promise + presignPart(key: string, uploadId: string, partNumber: number, expiresIn: number): Promise + completeMultipart( + key: string, + uploadId: string, + parts: { partNumber: number; etag: string }[], + ): Promise + abortMultipart(key: string, uploadId: string): Promise +} + +/** A shared cache for immutable, hash-keyed bytes (nodes, roots, schemas, bodies). */ +export interface Cache { + get(key: string): Promise + put(key: string, value: Uint8Array, opts?: { ttlSeconds?: number }): Promise +} + +/** A cache that holds nothing (tests that count blob reads; untrusted locations). */ +export const noCache: Cache = { + get: async () => null, + put: async () => {}, +} + +/** + * A BlobStore whose keys live under a prefix: one location's repository inside a + * bucket shared with other things. `prefix` has no trailing slash ('' for none). + */ +export class PrefixedBlobStore implements BlobStore { + readonly #p: string + constructor( + readonly inner: BlobStore, + prefix: string, + ) { + this.#p = prefix ? `${prefix.replace(/\/+$/, '')}/` : '' + } + #k = (key: string) => this.#p + key + get(key: string, range?: { offset: number; length?: number }) { + return this.inner.get(this.#k(key), range) + } + head(key: string) { + return this.inner.head(this.#k(key)) + } + put(key: string, body: Uint8Array | string, opts?: PutOptions) { + return this.inner.put(this.#k(key), body, opts) + } + delete(key: string) { + return this.inner.delete(this.#k(key)) + } + async list(prefix: string, cursor?: string) { + const r = await this.inner.list( + this.#k(prefix), + cursor === undefined ? undefined : this.#k(cursor), + ) + const keys = r.keys.map((k) => k.slice(this.#p.length)) + return r.cursor === undefined ? { keys } : { keys, cursor: r.cursor.slice(this.#p.length) } + } + presignGet(key: string, opts: PresignGetOptions) { + return this.inner.presignGet(this.#k(key), opts) + } + presignPut(key: string, opts: PresignPutOptions) { + return this.inner.presignPut(this.#k(key), opts) + } + createMultipart(key: string, contentType?: string) { + return this.inner.createMultipart(this.#k(key), contentType) + } + presignPart(key: string, uploadId: string, partNumber: number, expiresIn: number) { + return this.inner.presignPart(this.#k(key), uploadId, partNumber, expiresIn) + } + completeMultipart(key: string, uploadId: string, parts: { partNumber: number; etag: string }[]) { + return this.inner.completeMultipart(this.#k(key), uploadId, parts) + } + abortMultipart(key: string, uploadId: string) { + return this.inner.abortMultipart(this.#k(key), uploadId) + } +} diff --git a/packages/server/test/blob.test.ts b/packages/repo/test/blob.test.ts similarity index 99% rename from packages/server/test/blob.test.ts rename to packages/repo/test/blob.test.ts index b262c72..32da7c5 100644 --- a/packages/server/test/blob.test.ts +++ b/packages/repo/test/blob.test.ts @@ -7,7 +7,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { FsBlobStore, serveSignedBlob } from '../src/blob/fs.js' import { MemoryBlobStore } from '../src/blob/memory.js' import { S3BlobStore } from '../src/blob/s3.js' -import type { BlobStore } from '../src/ports.js' +import type { BlobStore } from '../src/types.js' import { type FakeS3, startFakeS3 } from './fake-s3.js' const enc = new TextEncoder() diff --git a/packages/server/test/fake-s3.ts b/packages/repo/test/fake-s3.ts similarity index 100% rename from packages/server/test/fake-s3.ts rename to packages/repo/test/fake-s3.ts diff --git a/packages/repo/test/log.test.ts b/packages/repo/test/log.test.ts new file mode 100644 index 0000000..c07690e --- /dev/null +++ b/packages/repo/test/log.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest' + +import { MemoryBlobStore } from '../src/blob/memory.js' +import { + appendLog, + ed25519Signer, + entryHash, + generateSigningKey, + readHead, + signEntry, + verifyEntry, + verifyLog, +} from '../src/log.js' +import { IntegrityError, keys, Repo } from '../src/repo.js' + +const repoOver = (blobs = new MemoryBlobStore()) => ({ + blobs, + repo: new Repo(blobs, { scope: 't', trusted: true }), +}) + +const entry = (seq: number, prev: string | null) => ({ + seq, + semver: `v1.${seq - 1}.0`, + versionHash: `ulv2:${String(seq).padStart(64, '0')}`, + baseSemver: seq === 1 ? null : `v1.${seq - 2}.0`, + message: `push ${seq}`, + appId: null, + actorId: null, + createdAt: new Date(Date.UTC(2026, 9, 3, 12, seq)).toISOString(), + prev, +}) + +describe('version log', () => { + it('signs and verifies entries with Ed25519', async () => { + const signer = await ed25519Signer(await generateSigningKey()) + const e = await signEntry(signer, entry(1, null)) + expect(await verifyEntry(e, [signer.publicKey])).toBe(true) + expect(await verifyEntry({ ...e, message: 'tampered' }, [signer.publicKey])).toBe(false) + const other = await ed25519Signer(await generateSigningKey()) + expect(await verifyEntry(e, [other.publicKey])).toBe(false) + }) + + it('derives the same key from the same seed', async () => { + const seed = await generateSigningKey() + expect((await ed25519Signer(seed)).publicKey).toEqual((await ed25519Signer(seed)).publicKey) + }) + + it('appends a hash-chained log and verifies it end to end', async () => { + const { blobs, repo } = repoOver() + const signer = await ed25519Signer(await generateSigningKey()) + let prev: string | null = null + for (let seq = 1; seq <= 3; seq++) { + const e = await signEntry(signer, entry(seq, prev)) + await appendLog(repo, 'c1', e) + prev = entryHash(e) + } + expect((await readHead(repo, 'c1'))!.seq).toBe(3) + const { entries } = await verifyLog(repo, 'c1', [signer.publicKey]) + expect(entries.map((e) => e.seq)).toEqual([1, 2, 3]) + + // A dropped entry breaks the chain. + const two = blobs.objects.get(keys.logEntry('c1', 2))! + blobs.objects.delete(keys.logEntry('c1', 2)) + await expect(verifyLog(repo, 'c1', [signer.publicKey])).rejects.toThrow(IntegrityError) + blobs.objects.set(keys.logEntry('c1', 2), two) + await expect(verifyLog(repo, 'c1', [signer.publicKey])).resolves.toBeTruthy() + }) +}) diff --git a/packages/repo/test/objects.test.ts b/packages/repo/test/objects.test.ts new file mode 100644 index 0000000..373a7b9 --- /dev/null +++ b/packages/repo/test/objects.test.ts @@ -0,0 +1,191 @@ +import { + compareUtf8, + diffTrees, + fileTree, + hashRecord, + iterate, + mergeTree, + type RecordEntry, + recordTree, + TreeBuilder, + utf8ByteLength, + verifyTree, +} from '@underlay/core' +import { describe, expect, it } from 'vitest' + +import { MemoryBlobStore } from '../src/blob/memory.js' +import { gzip } from '../src/gzip.js' +import { Lru } from '../src/lru.js' +import { + bodyOfRecord, + dropRecordBody, + IntegrityError, + keys, + OUT_OF_LINE_BYTES, + recordPayloadBytes, + Repo, + RepoSink, + RepoSource, +} from '../src/repo.js' +import { PrefixedBlobStore } from '../src/types.js' + +const freshRepo = (opts: { trusted?: boolean } = {}) => { + const blobs = new MemoryBlobStore() + // A private LRU per test, so reads really go to the store. + const repo = new Repo(blobs, { + scope: 'test', + trusted: opts.trusted ?? true, + lru: new Lru(8 << 20, () => 1024), + }) + return { blobs, repo } +} + +const record = (id: string, v = 0): RecordEntry => { + const { hash, canonical } = hashRecord(id, 'T', { id, v, pad: 'x'.repeat(200) }) + return { key: id, hash, size: utf8ByteLength(canonical), body: canonical } +} + +const ids = (n: number) => + Array.from({ length: n }, (_, i) => `rec-${String(i).padStart(6, '0')}`).sort(compareUtf8) + +async function collect(it: AsyncIterable): Promise { + const out: T[] = [] + for await (const x of it) out.push(x) + return out +} + +async function buildRecords(repo: Repo, n: number) { + const sink = new RepoSink(repo, { bodyOf: bodyOfRecord }) + const b = new TreeBuilder(recordTree, sink) + for (const id of ids(n)) b.addEntry(record(id)) + const root = b.finish().root! + await sink.flush() + return { root, sink } +} + +describe('record trees in a repository', () => { + it('round-trips nodes and bodies in the documented layout, and verifies', async () => { + const { blobs, repo } = freshRepo() + const { root, sink } = await buildRecords(repo, 5000) + expect(root.count).toBe(5000) + const stored = [...blobs.objects.keys()] + expect( + stored.every((k) => /^nodes\/[0-9a-f]{64}$|^bodies\/[0-9a-f]{64}\.ndjson\.gz$/.test(k)), + ).toBe(true) + // The sink lists exactly what it wrote: a commit's sync work for mirrors. + expect(new Set(sink.written)).toEqual(new Set(stored)) + + const source = new RepoSource(recordTree, repo) + expect((await verifyTree(source, root.hash)).ok).toBe(true) + const rows = await collect(iterate(source, root.hash, { payloads: true, offset: 4990 })) + expect(rows.map((r) => r.key)).toEqual(ids(5000).slice(4990)) + for (const r of rows) expect(JSON.parse(r.body!).id).toBe(r.key) + }) + + it('merges against stored trees, rewriting only changed leaves', async () => { + const { blobs, repo } = freshRepo() + const { root: base } = await buildRecords(repo, 20_000) + const putsBefore = blobs.puts + const all = ids(20_000) + const source = new RepoSource(recordTree, repo) + const changes = [all[10]!, all[15_000]!].map((id) => ({ key: id, entry: record(id, 1) })) + const sink = new RepoSink(repo, { bodyOf: bodyOfRecord }) + const merged = await mergeTree(source, sink, base.hash, changes) + await sink.flush() + expect(merged.stats.updated).toBe(2) + expect(blobs.puts - putsBefore).toBeLessThanOrEqual(2 * 2 + 2 * (base.level + 1)) + const diff = await collect(diffTrees(source, base.hash, merged.root!.hash)) + expect(diff.map((d) => d.key)).toEqual([all[10], all[15_000]]) + const changed = await collect( + iterate(source, merged.root!.hash, { payloads: true, offset: 10 }), + ) + expect(JSON.parse(changed[0]!.body!).data.v).toBe(1) + }) + + it('writes a large leaf body as several gzip members in one object', async () => { + const { blobs, repo } = freshRepo() + const sink = new RepoSink(repo, { bodyOf: bodyOfRecord }) + const b = new TreeBuilder(recordTree, sink, { + payloadBytes: recordPayloadBytes, + dropPayload: dropRecordBody, + spillBytes: 20_000, // tiny, to force several members + }) + for (const id of ids(3000)) b.addEntry(record(id)) + const root = b.finish().root! + await sink.flush() + const body = [...blobs.objects.entries()].find(([k]) => k.startsWith('bodies/'))![1].bytes + let members = 0 + for (let i = 0; i + 2 < body.length; i++) + if (body[i] === 0x1f && body[i + 1] === 0x8b && body[i + 2] === 8) members++ + expect(members).toBeGreaterThan(1) + const rows = await collect( + iterate(new RepoSource(recordTree, repo), root.hash, { payloads: true }), + ) + expect(rows.length).toBe(3000) + expect(rows.every((r) => JSON.parse(r.body!).id === r.key)).toBe(true) + }) + + it('resolves out-of-line records', async () => { + const { repo } = freshRepo() + const big = hashRecord('big', 'T', { blob: 'y'.repeat(OUT_OF_LINE_BYTES + 10) }) + const pointer = await repo.putOutOfLine(big.hash, big.canonical) + const sink = new RepoSink(repo, { bodyOf: bodyOfRecord }) + const b = new TreeBuilder(recordTree, sink) + b.addEntry(record('a')) + b.addEntry({ key: 'big', hash: big.hash, size: utf8ByteLength(big.canonical), body: pointer }) + const root = b.finish().root! + await sink.flush() + const rows = await collect( + iterate(new RepoSource(recordTree, repo), root.hash, { payloads: true }), + ) + expect(rows[1]!.body).toBe(big.canonical) + }) + + it('stores file trees without bodies', async () => { + const { blobs, repo } = freshRepo() + const sink = new RepoSink(repo) + const b = new TreeBuilder(fileTree, sink) + b.addEntry({ key: 'a'.repeat(64), size: 10 }) + b.addEntry({ key: 'b'.repeat(64), size: 20 }) + const root = b.finish().root! + await sink.flush() + expect([...blobs.objects.keys()]).toEqual([keys.node(root.hash)]) + const entries = await collect(iterate(new RepoSource(fileTree, repo), root.hash)) + expect(entries.map((e) => e.size)).toEqual([10, 20]) + }) + + it('verifies bodies from untrusted locations, and never caches a bad one', async () => { + const { blobs, repo } = freshRepo() + const { root } = await buildRecords(repo, 300) + // Tamper with one record in the stored body. + const bodyKey = [...blobs.objects.keys()].find((k) => k.startsWith('bodies/'))! + const lines = ( + await new Response( + new Blob([blobs.objects.get(bodyKey)!.bytes as Uint8Array]) + .stream() + .pipeThrough(new DecompressionStream('gzip')), + ).text() + ).split('\n') + lines[5] = lines[5]!.replace('"v":0', '"v":9') + blobs.objects.set(bodyKey, { bytes: await gzip(lines.join('\n')), contentType: null }) + + const untrusted = new Repo(blobs, { + scope: 'mirror', + trusted: false, + lru: new Lru(8 << 20, () => 1024), + }) + await expect( + collect(iterate(new RepoSource(recordTree, untrusted), root.hash, { payloads: true })), + ).rejects.toThrow(IntegrityError) + }) + + it('keeps each location under its prefix', async () => { + const shared = new MemoryBlobStore() + const repo = new Repo(new PrefixedBlobStore(shared, 'underlay/v2'), { + scope: 'p', + trusted: true, + }) + await repo.putSchema({ type: 'object' }) + expect([...shared.objects.keys()][0]).toMatch(/^underlay\/v2\/schemas\/[0-9a-f]{64}\.json$/) + }) +}) diff --git a/packages/repo/tsconfig.json b/packages/repo/tsconfig.json new file mode 100644 index 0000000..8c8d80a --- /dev/null +++ b/packages/repo/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "target": "ES2024", + "lib": ["ES2024", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "skipLibCheck": true, + "isolatedModules": true, + "resolveJsonModule": true, + "types": ["node"], + "noEmit": true + }, + "include": ["src", "test"] +} diff --git a/packages/repo/vitest.config.ts b/packages/repo/vitest.config.ts new file mode 100644 index 0000000..3e42797 --- /dev/null +++ b/packages/repo/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + include: ['test/**/*.test.ts'], + }, +}) diff --git a/packages/server/drizzle/0000_init.sql b/packages/server/drizzle/0000_init.sql index 0fdda7b..4f1213f 100644 --- a/packages/server/drizzle/0000_init.sql +++ b/packages/server/drizzle/0000_init.sql @@ -230,6 +230,25 @@ CREATE TABLE `page_comments` ( ); --> statement-breakpoint CREATE INDEX `page_comments_page_idx` ON `page_comments` (`page`);--> statement-breakpoint +CREATE TABLE `placements` ( + `id` text PRIMARY KEY NOT NULL, + `collection_id` text, + `organization_id` text, + `location_id` text NOT NULL, + `role` text NOT NULL, + `sets` text NOT NULL, + `state` text DEFAULT 'active' NOT NULL, + `synced_seq` integer DEFAULT 0 NOT NULL, + `last_error` text, + `updated_at` integer NOT NULL, + FOREIGN KEY (`collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`organization_id`) REFERENCES `organization`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`location_id`) REFERENCES `storage_locations`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX `placements_one_primary_uq` ON `placements` (`collection_id`) WHERE "placements"."role" = 'primary' AND "placements"."collection_id" IS NOT NULL;--> statement-breakpoint +CREATE UNIQUE INDEX `placements_target_location_uq` ON `placements` (`collection_id`,`organization_id`,`location_id`);--> statement-breakpoint +CREATE INDEX `placements_location_idx` ON `placements` (`location_id`);--> statement-breakpoint CREATE TABLE `push_runs` ( `session_id` text NOT NULL, `seq` integer NOT NULL, @@ -310,6 +329,24 @@ CREATE TABLE `session` ( --> statement-breakpoint CREATE UNIQUE INDEX `session_token_unique` ON `session` (`token`);--> statement-breakpoint CREATE INDEX `session_user_id_idx` ON `session` (`user_id`);--> statement-breakpoint +CREATE TABLE `storage_locations` ( + `id` text PRIMARY KEY NOT NULL, + `organization_id` text, + `kind` text NOT NULL, + `name` text NOT NULL, + `endpoint` text, + `region` text, + `bucket` text, + `prefix` text DEFAULT '' NOT NULL, + `credentials` text, + `permissions` text NOT NULL, + `status` text DEFAULT 'unverified' NOT NULL, + `last_error` text, + `verified_at` integer, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + FOREIGN KEY (`organization_id`) REFERENCES `organization`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint CREATE TABLE `user` ( `id` text PRIMARY KEY NOT NULL, `name` text NOT NULL, diff --git a/packages/server/drizzle/0001_platform_location.sql b/packages/server/drizzle/0001_platform_location.sql new file mode 100644 index 0000000..180e883 --- /dev/null +++ b/packages/server/drizzle/0001_platform_location.sql @@ -0,0 +1,4 @@ +-- The deployment's own storage location. Its bucket and credentials come from +-- the deployment's bindings and secrets, not from this row. +INSERT INTO `storage_locations` (`id`, `organization_id`, `kind`, `name`, `prefix`, `permissions`, `status`) +VALUES ('platform', NULL, 'platform', 'Underlay', '', 'read_write', 'active'); diff --git a/packages/server/drizzle/meta/0000_snapshot.json b/packages/server/drizzle/meta/0000_snapshot.json index 3dcb588..deb18d3 100644 --- a/packages/server/drizzle/meta/0000_snapshot.json +++ b/packages/server/drizzle/meta/0000_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "8af0a44f-14a2-4c79-8b3a-0976bbe54d57", + "id": "80efd3e4-fb23-47e6-ae97-3e5cd9522e1c", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { "account": { @@ -1475,6 +1475,133 @@ "uniqueConstraints": {}, "checkConstraints": {} }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_target_location_uq": { + "name": "placements_target_location_uq", + "columns": ["collection_id", "organization_id", "location_id"], + "isUnique": true + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": ["location_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, "push_runs": { "name": "push_runs", "columns": { @@ -1978,6 +2105,127 @@ "uniqueConstraints": {}, "checkConstraints": {} }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, "user": { "name": "user", "columns": { diff --git a/packages/server/drizzle/meta/0001_snapshot.json b/packages/server/drizzle/meta/0001_snapshot.json new file mode 100644 index 0000000..5f8ca77 --- /dev/null +++ b/packages/server/drizzle/meta/0001_snapshot.json @@ -0,0 +1,2736 @@ +{ + "id": "f05761d0-4c78-46d5-9fbf-f21a30ad9e2c", + "prevId": "80efd3e4-fb23-47e6-ae97-3e5cd9522e1c", + "version": "6", + "dialect": "sqlite", + "tables": { + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "columnsFrom": ["user_id"], + "tableTo": "user", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "columnsFrom": ["collection_id"], + "tableTo": "collections", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "columnsFrom": ["collection_id"], + "tableTo": "collections", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "columnsFrom": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'all'" + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "columnsFrom": ["collection_id"], + "tableTo": "collections", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "columnsFrom": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "columnsFrom": ["child_collection_id"], + "tableTo": "collections", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "columnsFrom": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "columnsFrom": ["inviter_id"], + "tableTo": "user", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "legacy_hashes": { + "name": "legacy_hashes", + "columns": { + "legacy_hash": { + "name": "legacy_hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "columnsFrom": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "columnsFrom": ["user_id"], + "tableTo": "user", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "page_comments": { + "name": "page_comments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "anchor": { + "name": "anchor", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "quote": { + "name": "quote", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "quote_context": { + "name": "quote_context", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolution_note": { + "name": "resolution_note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "edited_at": { + "name": "edited_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "page_comments_page_idx": { + "name": "page_comments_page_idx", + "columns": ["page"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL", + "isUnique": true + }, + "placements_target_location_uq": { + "name": "placements_target_location_uq", + "columns": ["collection_id", "organization_id", "location_id"], + "isUnique": true + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "columnsFrom": ["collection_id"], + "tableTo": "collections", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "columnsFrom": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "columnsFrom": ["location_id"], + "tableTo": "storage_locations", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "columnsFrom": ["session_id"], + "tableTo": "push_sessions", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "manifest_expected": { + "name": "manifest_expected", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest_received": { + "name": "manifest_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "manifest_needed": { + "name": "manifest_needed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "columnsFrom": ["collection_id"], + "tableTo": "collections", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "columnsFrom": ["collection_id"], + "tableTo": "collections", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "columnsFrom": ["user_id"], + "tableTo": "user", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "columnsFrom": ["organization_id"], + "tableTo": "organization", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "legacy_hash": { + "name": "legacy_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "legacy_public_hash": { + "name": "legacy_public_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_legacy_hash_idx": { + "name": "versions_legacy_hash_idx", + "columns": ["legacy_hash"], + "isUnique": false + }, + "versions_legacy_public_hash_idx": { + "name": "versions_legacy_public_hash_idx", + "columns": ["legacy_public_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "columnsFrom": ["collection_id"], + "tableTo": "collections", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "columnsFrom": ["webhook_id"], + "tableTo": "collection_webhooks", + "columnsTo": ["id"], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index b008607..f7f5dc7 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -5,9 +5,16 @@ { "idx": 0, "version": "6", - "when": 1791059262989, + "when": 1791059696177, "tag": "0000_init", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1791059701968, + "tag": "0001_platform_location", + "breakpoints": true } ] } diff --git a/packages/server/package.json b/packages/server/package.json index e7206a3..e9699d4 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -17,7 +17,7 @@ "@hono/node-server": "^1.19.14", "@libsql/client": "^0.18.0", "@underlay/core": "workspace:*", - "aws4fetch": "^1.0.20", + "@underlay/repo": "workspace:*", "drizzle-orm": "^0.45.2", "hono": "^4.12.18" }, diff --git a/packages/server/src/cache.ts b/packages/server/src/cache.ts index c370693..bb172e9 100644 --- a/packages/server/src/cache.ts +++ b/packages/server/src/cache.ts @@ -1,5 +1,4 @@ -import { Lru } from './lib/lru.js' -import type { Cache } from './ports.js' +import { type Cache, Lru } from '@underlay/repo' /** Node: an in-process LRU (the isolate LRU in Objects sits in front of it). */ export class MemoryCache implements Cache { @@ -15,12 +14,6 @@ export class MemoryCache implements Cache { } } -/** A cache that holds nothing (tests that count blob reads). */ -export const noCache: Cache = { - get: async () => null, - put: async () => {}, -} - interface CfCacheStorage { default: { match(req: Request): Promise diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 2ec49db..b13b431 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -331,6 +331,76 @@ export const legacyHashes = sqliteTable('legacy_hashes', { hash: text('hash').notNull(), }) +// --- Storage locations and placements (edge-redesign.md, "Placements") ----------------- + +/** + * Where repositories can live. The platform location's bucket and credentials + * come from the deployment's bindings and secrets; customer locations carry + * their own (encrypted with the platform key, never returned to clients). + */ +export const storageLocations = sqliteTable('storage_locations', { + id: id(), + /** Owning org; null for platform locations. */ + organizationId: text('organization_id').references(() => organization.id, { + onDelete: 'cascade', + }), + kind: text('kind', { enum: ['platform', 's3'] }).notNull(), + name: text('name').notNull(), + endpoint: text('endpoint'), + region: text('region'), + bucket: text('bucket'), + prefix: text('prefix').notNull().default(''), + /** Encrypted JSON {accessKeyId, secretAccessKey}; null for platform locations. */ + credentials: text('credentials'), + permissions: text('permissions', { enum: ['write', 'read_write'] }).notNull(), + status: text('status', { enum: ['active', 'unverified', 'broken', 'disabled'] }) + .notNull() + .default('unverified'), + lastError: text('last_error'), + verifiedAt: ts('verified_at'), + createdAt: createdAt(), +}) + +/** The id of the deployment's own location, seeded by the first migration. */ +export const PLATFORM_LOCATION_ID = 'platform' + +/** + * Which locations hold a collection: exactly one primary (today always a + * platform location) and any number of mirrors. A row with `organizationId` and + * no `collectionId` is an org-wide default inherited by the org's collections. + */ +export const placements = sqliteTable( + 'placements', + { + id: id(), + collectionId: text('collection_id').references(() => collections.id, { onDelete: 'cascade' }), + organizationId: text('organization_id').references(() => organization.id, { + onDelete: 'cascade', + }), + locationId: text('location_id') + .notNull() + .references(() => storageLocations.id, { onDelete: 'cascade' }), + role: text('role', { enum: ['primary', 'mirror'] }).notNull(), + sets: text('sets', { enum: ['public', 'public+private'] }).notNull(), + state: text('state', { enum: ['active', 'backfilling', 'lagging', 'error', 'paused'] }) + .notNull() + .default('active'), + /** The last version (seq) fully copied to this location. */ + syncedSeq: integer('synced_seq').notNull().default(0), + lastError: text('last_error'), + updatedAt: ts('updated_at') + .notNull() + .$defaultFn(() => new Date()), + }, + (t) => [ + uniqueIndex('placements_one_primary_uq') + .on(t.collectionId) + .where(sql`${t.role} = 'primary' AND ${t.collectionId} IS NOT NULL`), + uniqueIndex('placements_target_location_uq').on(t.collectionId, t.organizationId, t.locationId), + index('placements_location_idx').on(t.locationId), + ], +) + // --- Files ------------------------------------------------------------------------- export const files = sqliteTable('files', { diff --git a/packages/server/src/node/main.ts b/packages/server/src/node/main.ts index f576397..9fea8c0 100644 --- a/packages/server/src/node/main.ts +++ b/packages/server/src/node/main.ts @@ -8,16 +8,18 @@ * BLOB_DIR use the filesystem blob store under this directory, or * S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY, S3_REGION * BLOB_URL_SECRET HMAC key for filesystem presigned URLs + * REPO_PREFIX (repo), INTERNAL_PREFIX (internal) key prefixes in the platform bucket */ import { serve } from '@hono/node-server' +import { FsBlobStore, serveSignedBlob } from '@underlay/repo/blob/fs' +import { S3BlobStore } from '@underlay/repo/blob/s3' import { createApp } from '../app.js' -import { FsBlobStore, serveSignedBlob } from '../blob/fs.js' -import { S3BlobStore } from '../blob/s3.js' import { MemoryCache } from '../cache.js' import { openNodeDb } from '../db/node.js' import { drainSqliteJobs, SqliteJobs } from '../jobs.js' import type { BlobStore, Ports } from '../ports.js' +import { createStores } from '../stores.js' const env = process.env const port = Number(env.PORT ?? 4200) @@ -45,10 +47,15 @@ if (env.S3_ENDPOINT) { } let kick: () => void = () => {} +const cache = new MemoryCache() const ports: Ports = { db, - blobs, - cache: new MemoryCache(), + stores: createStores(db, cache, { + bucket: blobs, + repoPrefix: env.REPO_PREFIX ?? 'repo', + internalPrefix: env.INTERNAL_PREFIX ?? 'internal', + }), + cache, jobs: { async enqueue(job, opts) { await jobsTable.enqueue(job, opts) diff --git a/packages/server/src/ports.ts b/packages/server/src/ports.ts index f3d2af8..ea8f5fa 100644 --- a/packages/server/src/ports.ts +++ b/packages/server/src/ports.ts @@ -1,67 +1,19 @@ /** - * The four ports the server is written against. Cloudflare and Node each supply + * The ports the server is written against. Cloudflare and Node each supply * adapters; nothing outside the adapters knows which runtime it's on. * - * BlobStore S3 API via aws4fetch (R2, S3, MinIO), filesystem and memory for dev/tests + * Stores repositories resolved per collection from its placement (never a global + * bucket), plus the platform's internal area (sessions, uploads, reference log) * Db Drizzle sqlite-core over D1 or libsql — async, batches only (no interactive transactions) * Jobs Cloudflare Queues, or a SQLite jobs table polled by the Node process * Cache Cache API on Workers, in-memory LRU on Node */ +import type { BlobStore, Cache, Repo } from '@underlay/repo' import type { LibSQLDatabase } from 'drizzle-orm/libsql' import type * as schema from './db/schema.js' -// --- Blob store ------------------------------------------------------------------ - -export interface BlobHead { - size: number - etag: string - contentType: string | null -} - -export interface BlobObject extends BlobHead { - body: ReadableStream - bytes(): Promise - text(): Promise -} - -export interface PutOptions { - contentType?: string - /** Only write if the key doesn't exist. Immutable keys make this an optimization. */ - ifAbsent?: boolean -} - -export interface PresignGetOptions { - expiresIn: number - /** Content-Disposition for the response. */ - disposition?: string - contentType?: string -} - -export interface PresignPutOptions { - expiresIn: number - contentType?: string -} - -export interface BlobStore { - get(key: string, range?: { offset: number; length?: number }): Promise - head(key: string): Promise - put(key: string, body: Uint8Array | string, opts?: PutOptions): Promise - delete(key: string): Promise - list(prefix: string, cursor?: string): Promise<{ keys: string[]; cursor?: string }> - presignGet(key: string, opts: PresignGetOptions): Promise - presignPut(key: string, opts: PresignPutOptions): Promise - createMultipart(key: string, contentType?: string): Promise - presignPart(key: string, uploadId: string, partNumber: number, expiresIn: number): Promise - completeMultipart( - key: string, - uploadId: string, - parts: { partNumber: number; etag: string }[], - ): Promise - abortMultipart(key: string, uploadId: string): Promise -} - -// --- Database ---------------------------------------------------------------------- +export type { BlobStore, Cache } from '@underlay/repo' /** * Drizzle over SQLite. Both adapters are async and support `db.batch([...])`, @@ -70,8 +22,6 @@ export interface BlobStore { */ export type Db = LibSQLDatabase -// --- Jobs -------------------------------------------------------------------------------- - /** Messages carry ids only (Queues caps messages at 128 KB); data is in SQLite and blobs. */ export interface JobMessage { type: string @@ -83,18 +33,24 @@ export interface Jobs { enqueueBatch(jobs: JobMessage[]): Promise } -// --- Cache -------------------------------------------------------------------------------- - -/** A shared cache for immutable, hash-keyed bytes (nodes, roots, schemas). */ -export interface Cache { - get(key: string): Promise - put(key: string, value: Uint8Array, opts?: { ttlSeconds?: number }): Promise +/** + * Where repositories live. A collection's objects are read and written through + * the repository of its primary placement; mirrors are written by sync jobs. + */ +export interface Stores { + /** The repository of a collection's primary placement. */ + forCollection(collectionId: string): Promise + /** The repository at a storage location. */ + forLocation(locationId: string): Promise + /** + * Platform-internal objects that never leave the platform and are never + * mirrored: push sessions, staging uploads, the reference log. + */ + internal: BlobStore } -// --- Everything the app needs ------------------------------------------------------- - export interface Ports { - blobs: BlobStore + stores: Stores db: Db jobs: Jobs cache: Cache diff --git a/packages/server/src/storage/objects.ts b/packages/server/src/storage/objects.ts deleted file mode 100644 index 5c5e298..0000000 --- a/packages/server/src/storage/objects.ts +++ /dev/null @@ -1,375 +0,0 @@ -/** - * The object layout in the blob store, and typed access to it. - * - * nodes/ tree node JSON, gzip (the hash is of the uncompressed bytes) - * bodies/ the leaf's records, one canonical record per line, one gzip member; - * or, for a leaf over BODY_PART_BYTES, a JSON part list {"parts":[…]} - * bodyparts/ one part of a large leaf body, gzip (hash of the uncompressed part) - * records/ an out-of-line record over OUT_OF_LINE_BYTES, gzip; - * its body line is {"$ref":""} - * roots/.json version roots - * private/.json private set objects - * schemas/.json schemas, as canonical JSON - * sessions//… push session inputs and runs (expire by lifecycle rule) - * files/…, uploads/ file bytes (unchanged from v1) - * - * Everything except sessions/ and uploads/ is immutable and keyed by content, so - * it is cached by key forever: the isolate LRU first, then the shared cache, then - * the bucket. Storage layout (part sizes, out-of-line records) is not protocol. - */ -import { - type DecodedNode, - decodeNode, - hashSchema, - jcs, - type NodeDesc, - type NodeSource, - type PrivateSetObject, - privateCommitment, - type RecordEntry, - recordTree, - sha256Hex, - type TreeSink, - type TreeSpec, - type VersionRoot, - versionDigest, - versionHash, -} from '@underlay/core' - -import { gunzip, gunzipText, gzip, isGzip, splitLines } from '../lib/gzip.js' -import { Lru } from '../lib/lru.js' -import type { BlobStore, Cache } from '../ports.js' - -export const OUT_OF_LINE_BYTES = 64 * 1024 -export const BODY_PART_BYTES = 8 * 1024 * 1024 - -export const keys = { - node: (h: string) => `nodes/${h}`, - body: (h: string) => `bodies/${h}`, - part: (h: string) => `bodyparts/${h}`, - record: (h: string) => `records/${h}`, - root: (versionHashOrDigest: string) => - `roots/${versionHashOrDigest.includes(':') ? versionDigest(versionHashOrDigest) : versionHashOrDigest}.json`, - privateSet: (commitment: string) => `private/${commitment}.json`, - schema: (h: string) => `schemas/${h}.json`, - session: (id: string, name: string) => `sessions/${id}/${name}`, -} - -const REF_PREFIX = '{"$ref":"' -const enc = new TextEncoder() -const dec = new TextDecoder() - -/** Per-isolate memory: decoded nodes, roots, schemas, bodies. Shared by every request. */ -const isolateLru = new Lru(32 * 1024 * 1024, (v) => { - if (typeof v === 'string') return v.length * 2 - if (v && typeof v === 'object' && 'approxBytes' in v) - return (v as { approxBytes: number }).approxBytes - return 1024 -}) - -export class Objects { - constructor( - readonly blobs: BlobStore, - readonly cache: Cache, - readonly lru: Lru = isolateLru, - ) {} - - /** An immutable object's bytes: shared cache, then the bucket. */ - async #immutable(key: string): Promise { - const cached = await this.cache.get(key) - if (cached) return cached - const obj = await this.blobs.get(key) - if (!obj) return null - const bytes = await obj.bytes() - await this.cache.put(key, bytes) - return bytes - } - - // --- Nodes --- - - async nodeJson(hash: string): Promise { - const key = keys.node(hash) - const hit = this.lru.get(key) - if (typeof hit === 'string') return hit - const bytes = await this.#immutable(key) - if (!bytes) throw new Error(`Missing node ${hash}`) - const json = await gunzipText(bytes) - this.lru.set(key, json) - return json - } - - async putNode(hash: string, json: string): Promise { - await this.blobs.put(keys.node(hash), await gzip(json), { - contentType: 'application/gzip', - ifAbsent: true, - }) - } - - // --- Bodies --- - - /** A leaf's body lines, in entry order, with out-of-line records resolved. */ - async bodyLines(leafHash: string): Promise { - const key = keys.body(leafHash) - const hit = this.lru.get(key) - if (hit) return (hit as { lines: string[] }).lines - const bytes = await this.#immutable(key) - if (!bytes) throw new Error(`Missing body for leaf ${leafHash}`) - let lines: string[] - if (isGzip(bytes)) { - lines = splitLines(await gunzipText(bytes)) - } else { - const { parts } = JSON.parse(dec.decode(bytes)) as { parts: string[] } - const texts = await Promise.all( - parts.map(async (h) => { - const b = await this.#immutable(keys.part(h)) - if (!b) throw new Error(`Missing body part ${h}`) - return splitLines(await gunzipText(b)) - }), - ) - lines = texts.flat() - } - if (lines.some((l) => l.startsWith(REF_PREFIX))) { - lines = await Promise.all( - lines.map((l) => (l.startsWith(REF_PREFIX) ? this.#outOfLine(l) : l)), - ) - } - const approxBytes = lines.reduce((n, l) => n + l.length * 2, 0) - this.lru.set(key, { lines, approxBytes }) - return lines - } - - async #outOfLine(pointer: string): Promise { - const hash = (JSON.parse(pointer) as { $ref: string }).$ref - const b = await this.#immutable(keys.record(hash)) - if (!b) throw new Error(`Missing out-of-line record ${hash}`) - return gunzipText(b) - } - - async putBody(leafHash: string, lines: readonly string[]): Promise { - await this.blobs.put(keys.body(leafHash), await gzip(lines.join('\n') + '\n'), { - contentType: 'application/gzip', - ifAbsent: true, - }) - } - - async putBodyParts(leafHash: string, parts: readonly string[]): Promise { - await this.blobs.put(keys.body(leafHash), JSON.stringify({ parts }), { - contentType: 'application/json', - ifAbsent: true, - }) - } - - /** Write one body part; returns its hash (of the uncompressed text). */ - partOf(lines: readonly string[]): { hash: string; write: () => Promise } { - const text = lines.join('\n') + '\n' - const hash = sha256Hex(text) - return { - hash, - write: async () => - this.blobs.put(keys.part(hash), await gzip(text), { - contentType: 'application/gzip', - ifAbsent: true, - }), - } - } - - /** Store a large record out of line; returns the pointer line that goes in the body. */ - async putOutOfLine(recordHash: string, canonical: string): Promise { - await this.blobs.put(keys.record(recordHash), await gzip(canonical), { - contentType: 'application/gzip', - ifAbsent: true, - }) - return outOfLinePointer(recordHash) - } - - // --- Roots, private sets, schemas --- - - async root(hash: string): Promise { - const key = keys.root(hash) - const hit = this.lru.get(key) - if (hit) return (hit as { root: VersionRoot }).root - const bytes = await this.#immutable(key) - if (!bytes) throw new Error(`Missing root ${hash}`) - const text = dec.decode(bytes) - const root = JSON.parse(text) as VersionRoot - this.lru.set(key, { root, approxBytes: text.length * 3 }) - return root - } - - async putRoot(root: VersionRoot): Promise { - const hash = versionHash(root) - await this.blobs.put(keys.root(hash), jcs(root), { - contentType: 'application/json', - ifAbsent: true, - }) - return hash - } - - async privateSet(commitment: string): Promise { - const key = keys.privateSet(commitment) - const hit = this.lru.get(key) - if (hit) return (hit as { set: PrivateSetObject }).set - const bytes = await this.#immutable(key) - if (!bytes) throw new Error(`Missing private set ${commitment}`) - const text = dec.decode(bytes) - const set = JSON.parse(text) as PrivateSetObject - this.lru.set(key, { set, approxBytes: text.length * 3 }) - return set - } - - async putPrivateSet(set: PrivateSetObject): Promise { - const commitment = privateCommitment(set) - await this.blobs.put(keys.privateSet(commitment), jcs(set), { - contentType: 'application/json', - ifAbsent: true, - }) - return commitment - } - - async schema(hash: string): Promise> { - const key = keys.schema(hash) - const hit = this.lru.get(key) - if (hit) return (hit as { schema: Record }).schema - const bytes = await this.#immutable(key) - if (!bytes) throw new Error(`Missing schema ${hash}`) - const text = dec.decode(bytes) - const schema = JSON.parse(text) as Record - this.lru.set(key, { schema, approxBytes: text.length * 3 }) - return schema - } - - async putSchema(schema: unknown): Promise { - const hash = hashSchema(schema) - await this.blobs.put(keys.schema(hash), jcs(schema), { - contentType: 'application/json', - ifAbsent: true, - }) - return hash - } - - // --- Decoded nodes (for tree reads) --- - - async decoded(spec: TreeSpec, hash: string): Promise> { - const key = `decoded:${spec.name}:${hash}` - const hit = this.lru.get(key) - if (hit) return (hit as { node: DecodedNode }).node - const json = await this.nodeJson(hash) - const node = decodeNode(spec, json, hash) - this.lru.set(key, { node, approxBytes: json.length * 3 }) - return node - } -} - -export const outOfLinePointer = (recordHash: string) => `${REF_PREFIX}${recordHash}"}` - -/** NodeSource over the blob store. Record trees get bodies through `leafEntries`. */ -export class BlobSource implements NodeSource { - constructor( - readonly spec: TreeSpec, - readonly objects: Objects, - ) {} - - node(hash: string): Promise> { - return this.objects.decoded(this.spec, hash) - } - - async leafEntries(hash: string): Promise { - const node = await this.node(hash) - if (node.kind !== 'leaf') throw new Error(`Node ${hash} is not a leaf`) - if (this.spec !== (recordTree as TreeSpec)) return node.entries.slice() - const lines = await this.objects.bodyLines(hash) - if (lines.length !== node.entries.length) { - throw new Error( - `Body of leaf ${hash} has ${lines.length} lines for ${node.entries.length} entries`, - ) - } - return (node.entries as RecordEntry[]).map((e, i) => ({ ...e, body: lines[i]! })) as E[] - } -} - -/** - * TreeSink that writes nodes (and, for record trees, bodies) to the blob store. - * The builder is synchronous; writes run in the background with bounded - * concurrency. Call `drain()` between units of work to apply backpressure and - * `flush()` before using the root. - */ -export class BlobSink implements TreeSink { - readonly #pending = new Set>() - #error: unknown = null - #parts: string[] = [] - written = 0 - - constructor( - readonly objects: Objects, - readonly opts: { bodyOf?: (e: E) => string; concurrency?: number } = {}, - ) {} - - #run(work: () => Promise) { - const p = work() - .catch((err) => { - this.#error ??= err - }) - .finally(() => this.#pending.delete(p)) - this.#pending.add(p) - this.written++ - } - - spill(entries: readonly E[]): void { - const { bodyOf } = this.opts - if (!bodyOf) return - const part = this.objects.partOf(entries.map(bodyOf)) - this.#parts.push(part.hash) - this.#run(part.write) - } - - leaf(desc: NodeDesc, json: string, entries: readonly E[], spilled: number): void { - this.#run(() => this.objects.putNode(desc.hash, json)) - const { bodyOf } = this.opts - if (!bodyOf) return - if (this.#parts.length === 0) { - const lines = entries.map(bodyOf) - this.#run(() => this.objects.putBody(desc.hash, lines)) - return - } - const tail = entries.slice(spilled) - if (tail.length > 0) { - const part = this.objects.partOf(tail.map(bodyOf)) - this.#parts.push(part.hash) - this.#run(part.write) - } - const parts = this.#parts - this.#parts = [] - this.#run(() => this.objects.putBodyParts(desc.hash, parts)) - } - - interior(desc: NodeDesc, json: string): void { - this.#run(() => this.objects.putNode(desc.hash, json)) - } - - async drain(): Promise { - const limit = this.opts.concurrency ?? 16 - while (this.#pending.size >= limit) await Promise.race(this.#pending) - if (this.#error) throw this.#error - } - - async flush(): Promise { - while (this.#pending.size > 0) await Promise.race(this.#pending) - if (this.#error) throw this.#error - } -} - -export const bodyOfRecord = (e: RecordEntry): string => { - if (e.body === undefined) throw new Error(`Record ${e.key} has no body`) - return e.body -} - -/** Payload size for spilling: the body line's bytes. */ -export const recordPayloadBytes = (e: RecordEntry) => e.body?.length ?? 0 - -/** Clear a record entry's body after it was written in a part. */ -export const dropRecordBody = (e: RecordEntry) => { - delete e.body -} - -export const textBytes = (s: string) => enc.encode(s) -export { gunzip } diff --git a/packages/server/src/stores.ts b/packages/server/src/stores.ts new file mode 100644 index 0000000..6b511e0 --- /dev/null +++ b/packages/server/src/stores.ts @@ -0,0 +1,79 @@ +/** + * Resolve repositories from placements (edge-redesign.md, "Placements"). + * + * Today every collection's primary is the platform location, but nothing below + * assumes it: the primary is looked up per collection, and a location is turned + * into a Repo from its row. Customer (s3) locations become readable once + * credential encryption lands with bucket mirrors (phase 11). + */ +import { type BlobStore, type Cache, PrefixedBlobStore, Repo } from '@underlay/repo' +import { and, eq } from 'drizzle-orm' + +import * as schema from './db/schema.js' +import type { Db, Stores } from './ports.js' + +export interface PlatformStorage { + /** The deployment's own bucket. */ + bucket: BlobStore + /** Key prefix for repositories in it ('' for the bucket root). */ + repoPrefix: string + /** Key prefix for platform-internal objects. */ + internalPrefix: string +} + +/** Primary placements change rarely; remember them briefly per isolate. */ +const PRIMARY_TTL_MS = 60_000 +const primaryCache = new Map() + +export function createStores(db: Db, cache: Cache, platform: PlatformStorage): Stores { + const repos = new Map() + + const forLocation = async (locationId: string): Promise => { + const known = repos.get(locationId) + if (known) return known + const [loc] = await db + .select() + .from(schema.storageLocations) + .where(eq(schema.storageLocations.id, locationId)) + .limit(1) + if (!loc) throw new Error(`Unknown storage location ${locationId}`) + if (loc.kind !== 'platform') { + throw new Error(`Storage location ${locationId} (${loc.kind}) is not readable yet`) + } + const prefix = [platform.repoPrefix, loc.prefix].filter(Boolean).join('/') + const repo = new Repo(new PrefixedBlobStore(platform.bucket, prefix), { + cache, + scope: `loc:${locationId}`, + trusted: true, + }) + repos.set(locationId, repo) + return repo + } + + return { + forLocation, + async forCollection(collectionId: string): Promise { + const hit = primaryCache.get(collectionId) + if (hit && Date.now() - hit.at < PRIMARY_TTL_MS) return forLocation(hit.locationId) + const [row] = await db + .select({ locationId: schema.placements.locationId }) + .from(schema.placements) + .where( + and( + eq(schema.placements.collectionId, collectionId), + eq(schema.placements.role, 'primary'), + ), + ) + .limit(1) + if (!row) throw new Error(`Collection ${collectionId} has no primary placement`) + primaryCache.set(collectionId, { locationId: row.locationId, at: Date.now() }) + return forLocation(row.locationId) + }, + internal: new PrefixedBlobStore(platform.bucket, platform.internalPrefix), + } +} + +/** Forget a cached primary (after promoting a mirror). */ +export function invalidatePrimary(collectionId: string): void { + primaryCache.delete(collectionId) +} diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index cadbbd4..b581b3a 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -9,13 +9,14 @@ import type { Queue, ScheduledController, } from '@cloudflare/workers-types' +import { S3BlobStore } from '@underlay/repo/blob/s3' import { createApp } from './app.js' -import { S3BlobStore } from './blob/s3.js' import { CfCache } from './cache.js' import { openD1 } from './db/d1.js' import { QueueJobs, runJob } from './jobs.js' import type { JobMessage, Ports } from './ports.js' +import { createStores } from './stores.js' export interface Env { DB: D1Database @@ -26,19 +27,28 @@ export interface Env { R2_BUCKET: string R2_ACCESS_KEY_ID: string R2_SECRET_ACCESS_KEY: string + REPO_PREFIX?: string + INTERNAL_PREFIX?: string } function makePorts(env: Env, ctx: ExecutionContext): Ports { + const db = openD1(env.DB) + const cache = new CfCache(caches as never, env.DEPLOYMENT) + const bucket = new S3BlobStore({ + endpoint: env.R2_ENDPOINT, + bucket: env.R2_BUCKET, + accessKeyId: env.R2_ACCESS_KEY_ID, + secretAccessKey: env.R2_SECRET_ACCESS_KEY, + region: 'auto', + }) return { - db: openD1(env.DB), - blobs: new S3BlobStore({ - endpoint: env.R2_ENDPOINT, - bucket: env.R2_BUCKET, - accessKeyId: env.R2_ACCESS_KEY_ID, - secretAccessKey: env.R2_SECRET_ACCESS_KEY, - region: 'auto', + db, + stores: createStores(db, cache, { + bucket, + repoPrefix: env.REPO_PREFIX ?? 'repo', + internalPrefix: env.INTERNAL_PREFIX ?? 'internal', }), - cache: new CfCache(caches as never, env.DEPLOYMENT), + cache, jobs: new QueueJobs(env.JOBS as never), waitUntil: (p) => ctx.waitUntil(p), } diff --git a/packages/server/test/db.test.ts b/packages/server/test/db.test.ts index 0323d73..203b6fb 100644 --- a/packages/server/test/db.test.ts +++ b/packages/server/test/db.test.ts @@ -2,15 +2,16 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { MemoryBlobStore } from '@underlay/repo/blob/memory' import { eq } from 'drizzle-orm' import { afterAll, describe, expect, it } from 'vitest' -import { MemoryBlobStore } from '../src/blob/memory.js' import { MemoryCache } from '../src/cache.js' import { openNodeDb } from '../src/db/node.js' import * as schema from '../src/db/schema.js' import { drainSqliteJobs, registerJob, SqliteJobs } from '../src/jobs.js' import type { Ports } from '../src/ports.js' +import { createStores } from '../src/stores.js' const dirs: string[] = [] afterAll(async () => { @@ -62,7 +63,11 @@ describe('SQLite jobs', () => { const db = await tempDb() const ports: Ports = { db, - blobs: new MemoryBlobStore(), + stores: createStores(db, new MemoryCache(), { + bucket: new MemoryBlobStore(), + repoPrefix: 'repo', + internalPrefix: 'internal', + }), cache: new MemoryCache(), jobs: new SqliteJobs(db), waitUntil: () => {}, @@ -96,3 +101,45 @@ describe('SQLite jobs', () => { expect(await db.select().from(schema.jobs)).toEqual([]) }) }) + +describe('placements', () => { + it('resolves a collection to its primary location, and allows only one primary', async () => { + const db = await tempDb() + const bucket = new MemoryBlobStore() + const stores = createStores(db, new MemoryCache(), { + bucket, + repoPrefix: 'repo', + internalPrefix: 'internal', + }) + await db.insert(schema.organization).values({ id: 'org1', name: 'Org', slug: 'org' }) + const [c] = await db + .insert(schema.collections) + .values({ organizationId: 'org1', slug: 'c', name: 'C', privateSalt: 'ab'.repeat(32) }) + .returning() + await expect(stores.forCollection(c!.id)).rejects.toThrow(/no primary/) + await db.insert(schema.placements).values({ + collectionId: c!.id, + locationId: schema.PLATFORM_LOCATION_ID, + role: 'primary', + sets: 'public+private', + }) + const repo = await stores.forCollection(c!.id) + await repo.putSchema({ type: 'object' }) + await stores.internal.put('sessions/s1/x', 'y') + expect([...bucket.objects.keys()].sort()).toEqual([ + 'internal/sessions/s1/x', + expect.stringMatching(/^repo\/schemas\/[0-9a-f]{64}\.json$/), + ]) + await db + .insert(schema.storageLocations) + .values({ id: 'other', kind: 'platform', name: 'Other', permissions: 'read_write' }) + await expect( + db + .insert(schema.placements) + .values({ collectionId: c!.id, locationId: 'other', role: 'primary', sets: 'public' }), + ).rejects.toThrow() + await db + .insert(schema.placements) + .values({ collectionId: c!.id, locationId: 'other', role: 'mirror', sets: 'public' }) + }) +}) diff --git a/packages/server/test/storage.test.ts b/packages/server/test/storage.test.ts deleted file mode 100644 index 2fe3a7c..0000000 --- a/packages/server/test/storage.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { - compareUtf8, - diffTrees, - fileTree, - hashRecord, - iterate, - mergeTree, - type RecordEntry, - recordTree, - TreeBuilder, - utf8ByteLength, - verifyTree, -} from '@underlay/core' -import { describe, expect, it } from 'vitest' - -import { MemoryBlobStore } from '../src/blob/memory.js' -import { noCache } from '../src/cache.js' -import { Lru } from '../src/lib/lru.js' -import { - BlobSink, - BlobSource, - bodyOfRecord, - dropRecordBody, - keys, - Objects, - OUT_OF_LINE_BYTES, - recordPayloadBytes, -} from '../src/storage/objects.js' - -const freshObjects = () => { - const blobs = new MemoryBlobStore() - // A private LRU per test, so reads really go to the store. - return { blobs, objects: new Objects(blobs, noCache, new Lru(8 * 1024 * 1024, () => 1024)) } -} - -const record = (id: string, v = 0): RecordEntry => { - const { hash, canonical } = hashRecord(id, 'T', { id, v, pad: 'x'.repeat(200) }) - return { key: id, hash, size: utf8ByteLength(canonical), body: canonical } -} - -const ids = (n: number) => - Array.from({ length: n }, (_, i) => `rec-${String(i).padStart(6, '0')}`).sort(compareUtf8) - -async function collect(it: AsyncIterable): Promise { - const out: T[] = [] - for await (const x of it) out.push(x) - return out -} - -describe('record trees in the blob store', () => { - it('round-trips nodes and bodies, and verifies', async () => { - const { blobs, objects } = freshObjects() - const sink = new BlobSink(objects, { bodyOf: bodyOfRecord }) - const b = new TreeBuilder(recordTree, sink) - for (const id of ids(5000)) b.addEntry(record(id)) - const { root } = b.finish() - await sink.flush() - expect(root!.count).toBe(5000) - expect([...blobs.objects.keys()].filter((k) => k.startsWith('bodies/')).length).toBeGreaterThan( - 1, - ) - - const source = new BlobSource(recordTree, objects) - expect((await verifyTree(source, root!.hash)).ok).toBe(true) - const rows = await collect(iterate(source, root!.hash, { payloads: true, offset: 4990 })) - expect(rows.map((r) => r.key)).toEqual(ids(5000).slice(4990)) - for (const r of rows) expect(JSON.parse(r.body!).id).toBe(r.key) - }) - - it('merges against stored trees, rewriting only changed leaves', async () => { - const { blobs, objects } = freshObjects() - const sink = new BlobSink(objects, { bodyOf: bodyOfRecord }) - const b = new TreeBuilder(recordTree, sink) - const all = ids(20_000) - for (const id of all) b.addEntry(record(id)) - const base = b.finish().root! - await sink.flush() - const putsBefore = blobs.puts - - const source = new BlobSource(recordTree, objects) - const changes = [all[10]!, all[15_000]!].map((id) => ({ key: id, entry: record(id, 1) })) - const sink2 = new BlobSink(objects, { bodyOf: bodyOfRecord }) - const merged = await mergeTree(source, sink2, base.hash, changes) - await sink2.flush() - expect(merged.stats.updated).toBe(2) - // Two leaves (node + body each) and their paths; nothing else. - expect(blobs.puts - putsBefore).toBeLessThanOrEqual(2 * 2 + 2 * (base.level + 1)) - const diff = await collect(diffTrees(source, base.hash, merged.root!.hash)) - expect(diff.map((d) => d.key)).toEqual([all[10], all[15_000]]) - const changed = await collect( - iterate(source, merged.root!.hash, { payloads: true, offset: 10 }), - ) - expect(JSON.parse(changed[0]!.body!).data.v).toBe(1) - }) - - it('spills large leaf bodies into parts and reads them back', async () => { - const { blobs, objects } = freshObjects() - const sink = new BlobSink(objects, { bodyOf: bodyOfRecord }) - const b = new TreeBuilder(recordTree, sink, { - payloadBytes: recordPayloadBytes, - dropPayload: dropRecordBody, - spillBytes: 20_000, // tiny, to force parts - }) - for (const id of ids(3000)) b.addEntry(record(id)) - const root = b.finish().root! - await sink.flush() - expect([...blobs.objects.keys()].some((k) => k.startsWith('bodyparts/'))).toBe(true) - const source = new BlobSource(recordTree, objects) - const rows = await collect(iterate(source, root.hash, { payloads: true })) - expect(rows.length).toBe(3000) - expect(rows.every((r) => JSON.parse(r.body!).id === r.key)).toBe(true) - }) - - it('resolves out-of-line records', async () => { - const { objects } = freshObjects() - const big = hashRecord('big', 'T', { blob: 'y'.repeat(OUT_OF_LINE_BYTES + 10) }) - const pointer = await objects.putOutOfLine(big.hash, big.canonical) - const sink = new BlobSink(objects, { bodyOf: bodyOfRecord }) - const b = new TreeBuilder(recordTree, sink) - b.addEntry(record('a')) - b.addEntry({ key: 'big', hash: big.hash, size: utf8ByteLength(big.canonical), body: pointer }) - const root = b.finish().root! - await sink.flush() - const rows = await collect( - iterate(new BlobSource(recordTree, objects), root.hash, { payloads: true }), - ) - expect(rows[1]!.body).toBe(big.canonical) - }) - - it('stores file trees without bodies', async () => { - const { blobs, objects } = freshObjects() - const sink = new BlobSink(objects) - const b = new TreeBuilder(fileTree, sink) - b.addEntry({ key: 'a'.repeat(64), size: 10 }) - b.addEntry({ key: 'b'.repeat(64), size: 20 }) - const root = b.finish().root! - await sink.flush() - expect([...blobs.objects.keys()]).toEqual([keys.node(root.hash)]) - const entries = await collect(iterate(new BlobSource(fileTree, objects), root.hash)) - expect(entries.map((e) => e.size)).toEqual([10, 20]) - }) -}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a40ce42..4b3e226 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -200,6 +200,25 @@ importers: specifier: ^4.1.6 version: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + packages/repo: + dependencies: + '@underlay/core': + specifier: workspace:* + version: link:../core + aws4fetch: + specifier: ^1.0.20 + version: 1.0.20 + devDependencies: + '@types/node': + specifier: ^25.0.0 + version: 25.6.2 + typescript: + specifier: ^6.0.0 + version: 6.0.3 + vitest: + specifier: ^4.1.6 + version: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + packages/server: dependencies: '@hono/node-server': @@ -211,9 +230,9 @@ importers: '@underlay/core': specifier: workspace:* version: link:../core - aws4fetch: - specifier: ^1.0.20 - version: 1.0.20 + '@underlay/repo': + specifier: workspace:* + version: link:../repo drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) From 3869282ea4e8f513e297f5d738260f717bdba6df Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 16:43:58 -0400 Subject: [PATCH 005/178] v2 write path: commit engine and CAS publish commitVersion merges each (set, type) tree with its sorted changes, moves whole trees when a type changes sets, keeps file sets right in O(changes) with per-set reference-count sidecar trees, assembles the root and private set, and publishes with one conditional SQLite batch (insert version / move head / schema_usage, all guarded on the head still being the base). The signed log entry and head.json follow, with a repair job if that write fails. Tests cover CAS conflicts, set moves, type removal, file sets and schema revalidation. --- packages/server/drizzle/0000_init.sql | 2 + .../server/drizzle/meta/0000_snapshot.json | 16 +- .../server/drizzle/meta/0001_snapshot.json | 18 +- packages/server/drizzle/meta/_journal.json | 4 +- packages/server/src/db/schema.ts | 7 + packages/server/src/handlers.ts | 33 ++ packages/server/src/node/main.ts | 11 + packages/server/src/ports.ts | 4 +- packages/server/src/versions/commit.ts | 488 ++++++++++++++++++ packages/server/src/versions/file-refs.ts | 177 +++++++ packages/server/src/versions/publish.ts | 139 +++++ packages/server/src/versions/semver.ts | 38 ++ packages/server/src/worker.ts | 6 + packages/server/test/commit.test.ts | 329 ++++++++++++ packages/server/test/db.test.ts | 3 + packages/server/test/harness.ts | 73 +++ 16 files changed, 1342 insertions(+), 6 deletions(-) create mode 100644 packages/server/src/handlers.ts create mode 100644 packages/server/src/versions/commit.ts create mode 100644 packages/server/src/versions/file-refs.ts create mode 100644 packages/server/src/versions/publish.ts create mode 100644 packages/server/src/versions/semver.ts create mode 100644 packages/server/test/commit.test.ts create mode 100644 packages/server/test/harness.ts diff --git a/packages/server/drizzle/0000_init.sql b/packages/server/drizzle/0000_init.sql index 4f1213f..b8bb218 100644 --- a/packages/server/drizzle/0000_init.sql +++ b/packages/server/drizzle/0000_init.sql @@ -392,6 +392,8 @@ CREATE TABLE `versions` ( `type_counts` text NOT NULL, `public_type_counts` text NOT NULL, `has_private` integer DEFAULT false NOT NULL, + `public_refs_root` text, + `private_refs_root` text, `changes` text, `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, FOREIGN KEY (`collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade diff --git a/packages/server/drizzle/meta/0000_snapshot.json b/packages/server/drizzle/meta/0000_snapshot.json index deb18d3..5fbdbb5 100644 --- a/packages/server/drizzle/meta/0000_snapshot.json +++ b/packages/server/drizzle/meta/0000_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "80efd3e4-fb23-47e6-ae97-3e5cd9522e1c", + "id": "fb1d3f77-7095-4ca8-8fe8-45e5eb96f631", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { "account": { @@ -2517,6 +2517,20 @@ "autoincrement": false, "default": false }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, "changes": { "name": "changes", "type": "text", diff --git a/packages/server/drizzle/meta/0001_snapshot.json b/packages/server/drizzle/meta/0001_snapshot.json index 5f8ca77..1e88ce5 100644 --- a/packages/server/drizzle/meta/0001_snapshot.json +++ b/packages/server/drizzle/meta/0001_snapshot.json @@ -1,6 +1,6 @@ { - "id": "f05761d0-4c78-46d5-9fbf-f21a30ad9e2c", - "prevId": "80efd3e4-fb23-47e6-ae97-3e5cd9522e1c", + "id": "153626ea-7e2f-4690-a012-0514c340dcdb", + "prevId": "fb1d3f77-7095-4ca8-8fe8-45e5eb96f631", "version": "6", "dialect": "sqlite", "tables": { @@ -2517,6 +2517,20 @@ "autoincrement": false, "default": false }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, "changes": { "name": "changes", "type": "text", diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index f7f5dc7..bc977ad 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -5,14 +5,14 @@ { "idx": 0, "version": "6", - "when": 1791059696177, + "when": 1791059939117, "tag": "0000_init", "breakpoints": true }, { "idx": 1, "version": "6", - "when": 1791059701968, + "when": 1791059939881, "tag": "0001_platform_location", "breakpoints": true } diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index b13b431..e3ab113 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -257,6 +257,13 @@ export const versions = sqliteTable( typeCounts: json>('type_counts').notNull(), publicTypeCounts: json>('public_type_counts').notNull(), hasPrivate: bool('has_private').notNull().default(false), + /** + * Roots of the per-set file reference count trees (not protocol): how many of + * the set's records reference each file, plus declared-file markers. They let + * the next commit keep the file sets right in O(changes). + */ + publicRefsRoot: text('public_refs_root'), + privateRefsRoot: text('private_refs_root'), /** Change counts against the previous version (drive semver and webhooks). */ changes: json<{ added: number; removed: number; updated: number }>('changes'), createdAt: createdAt(), diff --git a/packages/server/src/handlers.ts b/packages/server/src/handlers.ts new file mode 100644 index 0000000..19f55e5 --- /dev/null +++ b/packages/server/src/handlers.ts @@ -0,0 +1,33 @@ +/** + * Job handlers. Imported by every entry so the registry is populated. + * + * version.published after a CAS publish: webhooks, mirror sync and the + * reference log hang off this (phases 4, 7, 11) + * repo.repairLog rewrite a collection's version log entries that a + * commit failed to write after publishing + */ +import { readHead } from '@underlay/repo' +import { and, asc, eq, gt } from 'drizzle-orm' + +import * as schema from './db/schema.js' +import { registerJob } from './jobs.js' +import { appendVersionLog } from './versions/commit.js' + +registerJob('version.published', async () => { + // Webhook deliveries, mirror sync and reference-log segments are added here + // as their phases land. +}) + +registerJob('repo.repairLog', async (job, ports) => { + const collectionId = String(job.collectionId) + const repo = await ports.stores.forCollection(collectionId) + const head = await readHead(repo, collectionId) + const missing = await ports.db + .select() + .from(schema.versions) + .where( + and(eq(schema.versions.collectionId, collectionId), gt(schema.versions.seq, head?.seq ?? 0)), + ) + .orderBy(asc(schema.versions.seq)) + for (const v of missing) await appendVersionLog(ports, repo, collectionId, v) +}) diff --git a/packages/server/src/node/main.ts b/packages/server/src/node/main.ts index 9fea8c0..556e12f 100644 --- a/packages/server/src/node/main.ts +++ b/packages/server/src/node/main.ts @@ -9,11 +9,14 @@ * S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY, S3_REGION * BLOB_URL_SECRET HMAC key for filesystem presigned URLs * REPO_PREFIX (repo), INTERNAL_PREFIX (internal) key prefixes in the platform bucket + * SIGNING_KEY Ed25519 private key seed (base64url) that signs version logs */ import { serve } from '@hono/node-server' +import { ed25519Signer, generateSigningKey, type Signer } from '@underlay/repo' import { FsBlobStore, serveSignedBlob } from '@underlay/repo/blob/fs' import { S3BlobStore } from '@underlay/repo/blob/s3' +import '../handlers.js' import { createApp } from '../app.js' import { MemoryCache } from '../cache.js' import { openNodeDb } from '../db/node.js' @@ -46,6 +49,13 @@ if (env.S3_ENDPOINT) { blobs = fsBlobs } +let signingKey = env.SIGNING_KEY +if (!signingKey) { + signingKey = await generateSigningKey() + console.warn('[underlay] SIGNING_KEY is not set; version logs are signed with a throwaway key') +} +const signer: Promise = ed25519Signer(signingKey) + let kick: () => void = () => {} const cache = new MemoryCache() const ports: Ports = { @@ -56,6 +66,7 @@ const ports: Ports = { internalPrefix: env.INTERNAL_PREFIX ?? 'internal', }), cache, + signer: () => signer, jobs: { async enqueue(job, opts) { await jobsTable.enqueue(job, opts) diff --git a/packages/server/src/ports.ts b/packages/server/src/ports.ts index ea8f5fa..2407955 100644 --- a/packages/server/src/ports.ts +++ b/packages/server/src/ports.ts @@ -8,7 +8,7 @@ * Jobs Cloudflare Queues, or a SQLite jobs table polled by the Node process * Cache Cache API on Workers, in-memory LRU on Node */ -import type { BlobStore, Cache, Repo } from '@underlay/repo' +import type { BlobStore, Cache, Repo, Signer } from '@underlay/repo' import type { LibSQLDatabase } from 'drizzle-orm/libsql' import type * as schema from './db/schema.js' @@ -54,6 +54,8 @@ export interface Ports { db: Db jobs: Jobs cache: Cache + /** Signs version log entries (the deployment's Ed25519 key, imported once per isolate). */ + signer(): Promise /** Run work after the response (Workers: ctx.waitUntil; Node: fire and forget with logging). */ waitUntil(p: Promise): void } diff --git a/packages/server/src/versions/commit.ts b/packages/server/src/versions/commit.ts new file mode 100644 index 0000000..1bca45c --- /dev/null +++ b/packages/server/src/versions/commit.ts @@ -0,0 +1,488 @@ +/** + * The commit engine: turn a base version plus sorted changes into a new + * version, and publish it. + * + * 1. per (set, type): merge the base tree with that set's changes + * (unchanged trees are reused as they are; type flips move whole trees); + * 2. file sets from reference-count deltas (file-refs.ts); + * 3. set objects, the private set object, the root, the version hash; + * 4. CAS publish (publish.ts), then the signed log entry and head.json. + * + * Every object is written before the publish, keyed by content, so a crash or a + * lost race leaves only unreferenced objects. Cost is O(changes) except where + * the work is inherently per-record: revalidating a type whose schema changed, + * removing a type, and moving a type between sets when both sets hold records. + * + * This runs a whole commit in one call, which suits small and medium pushes. + * Large ones split the per-(set, type) merges into key-range units run as jobs + * (edge-redesign.md, Commit step 2); the merge code is the same. + */ +import { + type Change, + compareUtf8, + emptySet, + fileTree, + isEmptySet, + iterate, + makeRoot, + mergeTree, + type PrivateSetObject, + type RecordEntry, + recordTree, + type SetObject, + setRecordTotals, + type TreeSummary, +} from '@underlay/core' +import { + appendLog, + bodyOfRecord, + dropRecordBody, + readHead, + recordPayloadBytes, + type Repo, + RepoSink, + RepoSource, + signEntry, +} from '@underlay/repo' +import { eq } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Ports } from '../ports.js' +import { applyFileSet, FileRefDelta, fileSizes, type SetName } from './file-refs.js' +import { publishVersion, type SchemaUsageChange } from './publish.js' +import { bumpType, deriveSemver } from './semver.js' + +export type ChangeSource = Iterable> | AsyncIterable> + +export interface TypeInput { + slug: string + schema: Record + schemaHash: string + /** Sorted changes per set (null: no changes there). Upserts carry `body`. */ + public: ChangeSource | null + private: ChangeSource | null +} + +export interface BaseVersion { + id: string + seq: number + semver: string + hash: string + publicRefsRoot: string | null + privateRefsRoot: string | null +} + +export interface CommitInput { + collectionId: string + base: BaseVersion | null + /** The full new type set: types in the base but not here are removed. */ + types: TypeInput[] + /** The full new metadata. */ + metadata: Record | null + /** Changes to declared (possibly unreferenced) files. */ + declaredFiles?: { add: string[]; remove: string[] } + message?: string | null + pushedBy?: string | null + appId?: string | null + actorId?: string | null + /** Validate a record's data against its type's schema; errors or null. Used when a schema changes. */ + validate?: (schema: Record, data: unknown) => string[] | null +} + +export type CommitResult = + | { status: 'committed'; version: typeof schema.versions.$inferSelect; written: string[] } + | { status: 'no_changes'; versionHash: string } + | { status: 'conflict'; headVersionId: string | null } + | { + status: 'invalid' + errors: { recordId: string; type: string; errors: string[] }[] + total: number + } + +const MAX_REPORTED_ERRORS = 100 +const SPILL_BYTES = 4 * 1024 * 1024 + +const summaryOf = (t: { + root: { hash: string; count: number; bytes: number } | null +}): TreeSummary => + t.root + ? { root: t.root.hash, count: t.root.count, bytes: t.root.bytes } + : { root: null, count: 0, bytes: 0 } + +const isPrivateSchema = (s: Record) => s.private === true + +async function* asAsync(src: Iterable | AsyncIterable): AsyncGenerator { + yield* src as AsyncIterable +} + +/** Merge two sorted change streams; on equal keys the second wins. */ +async function* overlay( + a: AsyncIterable>, + b: AsyncIterable>, +): AsyncGenerator> { + const ai = a[Symbol.asyncIterator]() + const bi = b[Symbol.asyncIterator]() + let x = await ai.next() + let y = await bi.next() + while (!x.done || !y.done) { + if (y.done || (!x.done && compareUtf8(x.value.key, y.value.key) < 0)) { + yield x.value + x = await ai.next() + } else if (x.done || compareUtf8(y.value.key, x.value.key) < 0) { + yield y.value + y = await bi.next() + } else { + yield y.value + x = await ai.next() + y = await bi.next() + } + } +} + +/** Every entry of a tree (with bodies) as upserts. */ +async function* treeAsUpserts( + repo: Repo, + root: string | null, +): AsyncGenerator> { + for await (const e of iterate(new RepoSource(recordTree, repo), root, { payloads: true })) { + yield { key: e.key, entry: e } + } +} + +export async function commitVersion(ports: Ports, input: CommitInput): Promise { + const { db } = ports + const repo = await ports.stores.forCollection(input.collectionId) + const [collection] = await db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, input.collectionId)) + .limit(1) + if (!collection) throw new Error(`Collection ${input.collectionId} not found`) + + const base = input.base + const baseRoot = base ? await repo.root(base.hash) : null + const basePublic: SetObject = baseRoot?.public ?? emptySet() + const basePrivate: SetObject = baseRoot?.private + ? await repo.privateSet(baseRoot.private) + : emptySet() + + const source = new RepoSource(recordTree, repo) + const sink = new RepoSink(repo, { bodyOf: bodyOfRecord }) + const builderOpts = { + payloadBytes: recordPayloadBytes, + dropPayload: dropRecordBody, + spillBytes: SPILL_BYTES, + } + const refs = new FileRefDelta() + const stats = { added: 0, removed: 0, updated: 0 } + let recordsChanged = false + + const newPublic: SetObject = emptySet() + const newPrivate: SetObject = emptySet() + + // Stats count per set: a record moving between sets is a removal and an addition. + const merge = async ( + set: SetName, + baseTree: string | null, + changes: AsyncIterable>, + ) => { + const result = await mergeTree(source, sink, baseTree, changes, { + ...builderOpts, + onChange: (before, after) => { + refs.record(set, before, after) + recordsChanged = true + if (before && after) stats.updated++ + else if (after) stats.added++ + else stats.removed++ + }, + }) + return summaryOf(result) + } + + const inputSlugs = new Set(input.types.map((t) => t.slug)) + let schemaChanged = false + + for (const t of input.types) { + const pubBase = basePublic.types[t.slug] + const privBase = basePrivate.types[t.slug] + if ((pubBase ?? privBase)?.schema !== t.schemaHash) schemaChanged = true + const nowPrivate = isPrivateSchema(t.schema) + const wasPrivateType = !pubBase && !!privBase + const pubChanges = t.public ? asAsync(t.public) : null + const privChanges = t.private ? asAsync(t.private) : null + + let pub: TreeSummary = { root: null, count: 0, bytes: 0 } + let priv: TreeSummary = { root: null, count: 0, bytes: 0 } + const pubRoot = pubBase?.root ?? null + const privRoot = privBase?.root ?? null + + if (nowPrivate) { + // Every record of a private type is in the private set. If the type was + // public, its public records move over: a plain move when the private + // tree was empty, otherwise a merge. + if (pubChanges) + throw new Error(`Type ${t.slug} is private; its changes belong to the private set`) + if (pubRoot && !privRoot && !privChanges) { + priv = pubBase! + } else if (pubRoot) { + priv = await merge( + 'private', + privRoot, + overlay(treeAsUpserts(repo, pubRoot), privChanges ?? asAsync([])), + ) + for await (const e of treeAsUpserts(repo, pubRoot)) refs.record('public', e.entry, null) + } else { + priv = privChanges ? await merge('private', privRoot, privChanges) : summary(privBase) + } + if (pubRoot) recordsChanged = true + } else if (wasPrivateType) { + // A private type made public: its records become public (per-record flags + // were not kept while the whole type was private), except those this push + // marks private. + pub = await merge( + 'public', + null, + overlay(treeAsUpserts(repo, privRoot), pubChanges ?? asAsync([])), + ) + for await (const e of treeAsUpserts(repo, privRoot)) refs.record('private', e.entry, null) + priv = privChanges + ? await merge('private', null, privChanges) + : { root: null, count: 0, bytes: 0 } + recordsChanged = true + } else { + pub = pubChanges ? await merge('public', pubRoot, pubChanges) : summary(pubBase) + priv = privChanges ? await merge('private', privRoot, privChanges) : summary(privBase) + } + + if (nowPrivate) { + newPrivate.types[t.slug] = { schema: t.schemaHash, ...priv } + } else { + newPublic.types[t.slug] = { schema: t.schemaHash, ...pub } + if (priv.root) newPrivate.types[t.slug] = { schema: t.schemaHash, ...priv } + } + } + + // Removed types: drop their trees, and release their file references. + for (const [set, base_] of [ + ['public', basePublic], + ['private', basePrivate], + ] as const) { + for (const [slug, entry] of Object.entries(base_.types)) { + if (inputSlugs.has(slug)) continue + schemaChanged = true + for await (const e of treeAsUpserts(repo, entry.root)) { + refs.record(set, e.entry, null) + stats.removed++ + recordsChanged = true + } + } + } + await sink.flush() + + // Revalidate types whose schema changed (records pushed earlier were checked + // against the old schema). O(type size), inherent to a schema change. + const errors: { recordId: string; type: string; errors: string[] }[] = [] + let errorCount = 0 + if (input.validate) { + for (const t of input.types) { + const before = basePublic.types[t.slug] ?? basePrivate.types[t.slug] + if (!before || before.schema === t.schemaHash) continue + for (const tree of [newPublic.types[t.slug], newPrivate.types[t.slug]]) { + for await (const e of iterate(source, tree?.root ?? null, { payloads: true })) { + const errs = input.validate(t.schema, (JSON.parse(e.body!) as { data: unknown }).data) + if (errs) { + errorCount++ + if (errors.length < MAX_REPORTED_ERRORS) + errors.push({ recordId: e.key, type: t.slug, errors: errs }) + } + } + } + } + } + if (errorCount > 0) return { status: 'invalid', errors, total: errorCount } + + // File sets. + const pubFiles = await applyFileSet( + db, + repo, + { refsRoot: base?.publicRefsRoot ?? null, files: basePublic.files }, + refs.refs.public, + null, + ) + const privFiles = await applyFileSet( + db, + repo, + { refsRoot: base?.privateRefsRoot ?? null, files: basePrivate.files }, + refs.refs.private, + input.declaredFiles ?? null, + ) + newPublic.files = pubFiles.files + newPrivate.files = privFiles.files + + // Root. + const privSet: PrivateSetObject = { ...newPrivate, salt: collection.privateSalt } + const root = makeRoot(input.metadata, newPublic, isEmptySet(newPrivate) ? null : privSet) + if (root.private) await repo.putPrivateSet(privSet) + const versionHash = await repo.putRoot(root) + if (base && versionHash === base.hash) return { status: 'no_changes', versionHash } + + const sv = deriveSemver(base?.semver ?? null, schemaChanged, recordsChanged) + + const pubTotals = setRecordTotals(newPublic) + const privTotals = setRecordTotals(newPrivate) + const typeCounts: Record = {} + const publicTypeCounts: Record = {} + for (const [slug, t] of Object.entries(newPublic.types)) { + typeCounts[slug] = (typeCounts[slug] ?? 0) + t.count + publicTypeCounts[slug] = t.count + } + for (const [slug, t] of Object.entries(newPrivate.types)) + typeCounts[slug] = (typeCounts[slug] ?? 0) + t.count + + const usage: SchemaUsageChange[] = [] + for (const [set, before, after] of [ + ['public', basePublic, newPublic], + ['private', basePrivate, newPrivate], + ] as const) { + const slugs = new Set([...Object.keys(before.types), ...Object.keys(after.types)]) + for (const slug of slugs) { + const was = before.types[slug]?.schema ?? null + const now = after.types[slug]?.schema ?? null + if (was !== now) usage.push({ set, typeSlug: slug, schemaHash: now, wasOpen: was !== null }) + } + } + + // The cumulative public files tree: add files that entered the public set. + const publicFilesRoot = await mergeCumulativeFiles( + ports, + repo, + collection.publicFilesRoot, + pubFiles.added, + ) + + const versionId = crypto.randomUUID() + const versionRow = { + id: versionId, + collectionId: input.collectionId, + seq: (base?.seq ?? 0) + 1, + semver: sv.semver, + major: sv.major, + minor: sv.minor, + patch: sv.patch, + hash: versionHash, + baseSemver: base?.semver ?? null, + message: input.message ?? null, + pushedBy: input.pushedBy ?? null, + appId: input.appId ?? null, + actorId: input.actorId ?? null, + recordCount: pubTotals.count + privTotals.count, + publicRecordCount: pubTotals.count, + fileCount: newPublic.files.count + newPrivate.files.count, + totalBytes: pubTotals.bytes + privTotals.bytes + newPublic.files.bytes + newPrivate.files.bytes, + typeCounts, + publicTypeCounts, + hasPrivate: root.private !== null, + publicRefsRoot: pubFiles.refsRoot, + privateRefsRoot: privFiles.refsRoot, + changes: stats, + } + const published = await publishVersion(db, { + version: versionRow, + baseVersionId: base?.id ?? null, + collectionUpdate: { publicFilesRoot, summary: summarize(input.metadata) }, + schemaHashes: [...new Set(input.types.map((t) => t.schemaHash))], + usage, + }) + if (!published.ok) return { status: 'conflict', headVersionId: published.headVersionId } + + const [version] = await db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, versionId)) + .limit(1) + + // The version log and head, after every object the version reaches. If this + // fails the version is still published; the repair job rewrites the log. + try { + await appendVersionLog(ports, repo, input.collectionId, version!) + } catch (err) { + console.error(`[commit] version log for ${versionId} failed; queued a repair`, err) + await ports.jobs.enqueue({ type: 'repo.repairLog', collectionId: input.collectionId }) + } + await ports.jobs.enqueue({ + type: 'version.published', + versionId, + bump: bumpType(schemaChanged, recordsChanged), + }) + return { status: 'committed', version: version!, written: sink.written } +} + +const summary = (t: TreeSummary | undefined): TreeSummary => + t ? { root: t.root, count: t.count, bytes: t.bytes } : { root: null, count: 0, bytes: 0 } + +/** Bounded fields for collection lists, from the version metadata. */ +function summarize(metadata: Record | null): schema.CollectionSummary | null { + if (!metadata) return null + const str = (v: unknown, max: number) => (typeof v === 'string' ? v.slice(0, max) : undefined) + const out: schema.CollectionSummary = {} + const title = str(metadata.title ?? metadata.name, 200) + const description = str(metadata.description, 1000) + const license = str(metadata.license, 100) + if (title) out.title = title + if (description) out.description = description + if (license) out.license = license + if (Array.isArray(metadata.tags)) { + out.tags = metadata.tags + .filter((t): t is string => typeof t === 'string') + .slice(0, 20) + .map((t) => t.slice(0, 50)) + } + return out +} + +async function mergeCumulativeFiles( + ports: Ports, + repo: Repo, + root: string | null, + added: string[], +): Promise { + if (added.length === 0) return root + const sizes = await fileSizes(ports.db, added) + const changes = added + .slice() + .sort(compareUtf8) + .map((h) => ({ key: h, entry: { key: h, size: sizes.get(h)! } })) + const sink = new RepoSink(repo) + const merged = await mergeTree(new RepoSource(fileTree, repo), sink, root, changes) + await sink.flush() + return merged.root?.hash ?? null +} + +/** Write the signed log entry for a published version, then head.json. Idempotent. */ +export async function appendVersionLog( + ports: Ports, + repo: Repo, + collectionId: string, + v: Pick< + typeof schema.versions.$inferSelect, + 'seq' | 'semver' | 'hash' | 'baseSemver' | 'message' | 'appId' | 'actorId' | 'createdAt' + >, +): Promise { + const head = await readHead(repo, collectionId) + if (head && head.seq >= v.seq) return + if ((head?.seq ?? 0) !== v.seq - 1) { + throw new Error(`Log for ${collectionId} is at ${head?.seq ?? 0}, cannot append ${v.seq}`) + } + const entry = await signEntry(await ports.signer(), { + seq: v.seq, + semver: v.semver, + versionHash: v.hash, + baseSemver: v.baseSemver, + message: v.message, + appId: v.appId, + actorId: v.actorId, + createdAt: v.createdAt.toISOString(), + prev: head?.entryHash ?? null, + }) + await appendLog(repo, collectionId, entry) +} diff --git a/packages/server/src/versions/file-refs.ts b/packages/server/src/versions/file-refs.ts new file mode 100644 index 0000000..3a3c574 --- /dev/null +++ b/packages/server/src/versions/file-refs.ts @@ -0,0 +1,177 @@ +/** + * Keeping each set's file tree right in O(changes) (edge-redesign-build.md, + * finding 5). + * + * Protocol (docs/protocol-v2.md §9): a file belongs to every set that has a + * record referencing it; a file declared in a push but referenced by no record + * belongs to the private set. To know when a file leaves a set, each set keeps a + * sidecar count tree (not protocol) next to the version: + * + * "r:" → how many of the set's records reference the file + * "d:" → 1 when the file is declared (private set only) + * + * A commit turns record changes into count deltas, applies them to the count + * tree, and adds or removes files from the protocol file tree only when a file's + * presence actually flips. + */ +import { + type Change, + compareUtf8, + type CountEntry, + countTree, + type FileEntry, + fileRefs, + fileTree, + getEntry, + iterate, + mergeTree, + type RecordEntry, + type TreeSummary, +} from '@underlay/core' +import { type Repo, RepoSink, RepoSource } from '@underlay/repo' +import { inArray } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Db } from '../ports.js' + +export type SetName = 'public' | 'private' + +const REF = 'r:' +const DECLARED = 'd:' + +/** Accumulates reference count changes from record changes, per set. */ +export class FileRefDelta { + readonly refs: Record> = { public: new Map(), private: new Map() } + + #bump(set: SetName, hashes: string[], by: number) { + const m = this.refs[set] + for (const h of hashes) m.set(h, (m.get(h) ?? 0) + by) + } + + /** Feed one record change in a set (mergeTree's onChange). */ + record(set: SetName, before: RecordEntry | null, after: RecordEntry | null): void { + if (before) this.#bump(set, refsOfBody(before.body), -1) + if (after) this.#bump(set, refsOfBody(after.body), +1) + } + + get touched(): boolean { + return [...this.refs.public.values(), ...this.refs.private.values()].some((n) => n !== 0) + } +} + +function refsOfBody(body: string | undefined): string[] { + if (body === undefined) throw new Error('File reference accounting needs record bodies') + // Cheap prefilter: most records reference no files. + if (!body.includes('"$file"')) return [] + return fileRefs((JSON.parse(body) as { data: unknown }).data) +} + +export interface FileSetResult { + files: TreeSummary + refsRoot: string | null + /** Files that entered the set in this commit (for the cumulative public files tree). */ + added: string[] +} + +export class MissingFilesError extends Error { + constructor(readonly hashes: string[]) { + super(`${hashes.length} referenced file(s) are not uploaded`) + this.name = 'MissingFilesError' + } +} + +/** File sizes for hashes, from the files table (chunked for D1's bound-parameter limit). */ +export async function fileSizes(db: Db, hashes: string[]): Promise> { + const out = new Map() + for (let i = 0; i < hashes.length; i += 90) { + const rows = await db + .select({ hash: schema.files.hash, size: schema.files.size }) + .from(schema.files) + .where(inArray(schema.files.hash, hashes.slice(i, i + 90))) + for (const r of rows) out.set(r.hash, r.size) + } + return out +} + +/** The declared-file markers currently in a count tree. O(declared files). */ +export async function declaredFiles(repo: Repo, refsRoot: string | null): Promise> { + const out = new Set() + for await (const e of iterate(new RepoSource(countTree, repo), refsRoot, { after: DECLARED })) { + if (!e.key.startsWith(DECLARED)) break + out.add(e.key.slice(DECLARED.length)) + } + return out +} + +/** + * Apply a set's reference deltas (and, for the private set, declared-file + * changes) to its count tree and file tree. + */ +export async function applyFileSet( + db: Db, + repo: Repo, + base: { refsRoot: string | null; files: TreeSummary }, + refDeltas: Map, + declared: { add: string[]; remove: string[] } | null, +): Promise { + const counts = new RepoSource(countTree, repo) + const keyDelta = new Map() + for (const [h, d] of refDeltas) if (d !== 0) keyDelta.set(REF + h, d) + for (const h of declared?.add ?? []) + keyDelta.set(DECLARED + h, (keyDelta.get(DECLARED + h) ?? 0) + 1) + for (const h of declared?.remove ?? []) + keyDelta.set(DECLARED + h, (keyDelta.get(DECLARED + h) ?? 0) - 1) + if ([...keyDelta.values()].every((d) => d === 0)) { + return { files: base.files, refsRoot: base.refsRoot, added: [] } + } + + // New counts, and whether each touched file is present before and after. + const countChanges: Change[] = [] + const presence = new Map() + const hashesTouched = new Set([...keyDelta.keys()].map((k) => k.slice(2))) + for (const h of hashesTouched) { + const p = { before: false, after: false } + for (const prefix of [REF, DECLARED]) { + const key = prefix + h + const old = (await getEntry(counts, base.refsRoot, key))?.n ?? 0 + const now = old + (keyDelta.get(key) ?? 0) + if (prefix === REF && now < 0) throw new Error(`File reference count for ${h} went negative`) + // Declaring is idempotent: a marker is 0 or 1. + const n = prefix === DECLARED ? Math.min(1, Math.max(0, now)) : now + if (n !== old) countChanges.push({ key, entry: n > 0 ? { key, n } : null }) + p.before ||= old > 0 + p.after ||= n > 0 + } + presence.set(h, p) + } + countChanges.sort((a, b) => compareUtf8(a.key, b.key)) + + const entering = [...presence].filter(([, p]) => !p.before && p.after).map(([h]) => h) + const leaving = [...presence].filter(([, p]) => p.before && !p.after).map(([h]) => h) + const sizes = await fileSizes(db, entering) + const missing = entering.filter((h) => !sizes.has(h)) + if (missing.length > 0) throw new MissingFilesError(missing) + + const fileChanges: Change[] = [ + ...entering.map((h) => ({ key: h, entry: { key: h, size: sizes.get(h)! } })), + ...leaving.map((h) => ({ key: h, entry: null })), + ].sort((a, b) => compareUtf8(a.key, b.key)) + + const sink = new RepoSink(repo) + const refs = await mergeTree(counts, sink, base.refsRoot, countChanges) + const fileSink = new RepoSink(repo) + const files = await mergeTree( + new RepoSource(fileTree, repo), + fileSink, + base.files.root, + fileChanges, + ) + await Promise.all([sink.flush(), fileSink.flush()]) + return { + refsRoot: refs.root?.hash ?? null, + files: files.root + ? { root: files.root.hash, count: files.root.count, bytes: files.root.bytes } + : { root: null, count: 0, bytes: 0 }, + added: entering, + } +} diff --git a/packages/server/src/versions/publish.ts b/packages/server/src/versions/publish.ts new file mode 100644 index 0000000..21ea37c --- /dev/null +++ b/packages/server/src/versions/publish.ts @@ -0,0 +1,139 @@ +/** + * Publish: make a built version visible by moving the collection's head, with + * compare-and-swap (edge-redesign.md, Commit step 5). + * + * D1 has no interactive transactions, so the CAS is one atomic batch whose + * statements all carry the condition in SQL: + * 1. insert the version row only if the head is still the base; + * 2. move the head only if it is still the base and the row from 1 exists; + * 3. close and open schema_usage rows only if the row from 1 exists. + * If another commit won, 1 inserts nothing and the rest match nothing; the + * caller sees that by reading the head back. Everything the version points to is + * already in the repository, so there is never a half-built version. + */ +import { and, eq, isNull, sql } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Db } from '../ports.js' +import type { SetName } from './file-refs.js' + +export interface NewVersionRow { + id: string + collectionId: string + seq: number + semver: string + major: number + minor: number + patch: number + hash: string + baseSemver: string | null + message: string | null + pushedBy: string | null + appId: string | null + actorId: string | null + recordCount: number + publicRecordCount: number + fileCount: number + totalBytes: number + typeCounts: Record + publicTypeCounts: Record + hasPrivate: boolean + publicRefsRoot: string | null + privateRefsRoot: string | null + changes: { added: number; removed: number; updated: number } +} + +export interface SchemaUsageChange { + set: SetName + typeSlug: string + /** The schema the type now uses in the set, or null when it left the set. */ + schemaHash: string | null + /** Whether a usage row is currently open (the type was in the set at the base). */ + wasOpen: boolean +} + +export interface PublishInput { + version: NewVersionRow + baseVersionId: string | null + collectionUpdate: { + publicFilesRoot: string | null + summary: schema.CollectionSummary | null + } + schemaHashes: string[] + usage: SchemaUsageChange[] +} + +export async function publishVersion( + db: Db, + p: PublishInput, +): Promise<{ ok: boolean; headVersionId: string | null }> { + const v = p.version + const now = Date.now() + const versionExists = sql`EXISTS (SELECT 1 FROM ${schema.versions} WHERE ${schema.versions.id} = ${v.id})` + const headIsBase = sql`(SELECT ${schema.collections.headVersionId} FROM ${schema.collections} WHERE ${schema.collections.id} = ${v.collectionId}) IS ${p.baseVersionId}` + + const statements = [ + db.run(sql` + INSERT INTO ${schema.versions} ( + id, collection_id, seq, semver, major, minor, patch, hash, base_semver, message, pushed_by, + app_id, actor_id, record_count, public_record_count, file_count, total_bytes, type_counts, + public_type_counts, has_private, public_refs_root, private_refs_root, changes, created_at + ) + SELECT ${v.id}, ${v.collectionId}, ${v.seq}, ${v.semver}, ${v.major}, ${v.minor}, ${v.patch}, + ${v.hash}, ${v.baseSemver}, ${v.message}, ${v.pushedBy}, ${v.appId}, ${v.actorId}, + ${v.recordCount}, ${v.publicRecordCount}, ${v.fileCount}, ${v.totalBytes}, + ${JSON.stringify(v.typeCounts)}, ${JSON.stringify(v.publicTypeCounts)}, ${v.hasPrivate ? 1 : 0}, + ${v.publicRefsRoot}, ${v.privateRefsRoot}, ${JSON.stringify(v.changes)}, ${now} + WHERE ${headIsBase} + `), + db.run(sql` + UPDATE ${schema.collections} + SET head_version_id = ${v.id}, updated_at = ${now}, + public_files_root = ${p.collectionUpdate.publicFilesRoot}, + summary = ${p.collectionUpdate.summary ? JSON.stringify(p.collectionUpdate.summary) : null} + WHERE id = ${v.collectionId} AND head_version_id IS ${p.baseVersionId} AND ${versionExists} + `), + ...p.schemaHashes.map((h) => + db.run(sql`INSERT OR IGNORE INTO ${schema.schemas} (hash, created_at) VALUES (${h}, ${now})`), + ), + ...p.usage.flatMap((u) => { + const out = [] + if (u.wasOpen) { + out.push( + db.run(sql` + UPDATE ${schema.schemaUsage} SET to_seq = ${v.seq} + WHERE collection_id = ${v.collectionId} AND type_slug = ${u.typeSlug} AND "set" = ${u.set} + AND to_seq IS NULL AND ${versionExists} + `), + ) + } + if (u.schemaHash !== null) { + out.push( + db.run(sql` + INSERT INTO ${schema.schemaUsage} (schema_hash, collection_id, type_slug, "set", from_seq, to_seq) + SELECT ${u.schemaHash}, ${v.collectionId}, ${u.typeSlug}, ${u.set}, ${v.seq}, NULL + WHERE ${versionExists} + `), + ) + } + return out + }), + ] + await db.batch(statements as unknown as Parameters[0]) + + const [row] = await db + .select({ head: schema.collections.headVersionId }) + .from(schema.collections) + .where(eq(schema.collections.id, v.collectionId)) + .limit(1) + return { ok: row?.head === v.id, headVersionId: row?.head ?? null } +} + +/** The open schema usage rows of a collection, as `${set}\u0000${slug}` → schema hash. */ +export async function openSchemaUsage(db: Db, collectionId: string): Promise> { + const rows = await db + .select() + .from(schema.schemaUsage) + .where(and(eq(schema.schemaUsage.collectionId, collectionId), isNull(schema.schemaUsage.toSeq))) + return new Map(rows.map((r) => [`${r.set}\u0000${r.typeSlug}`, r.schemaHash])) +} diff --git a/packages/server/src/versions/semver.ts b/packages/server/src/versions/semver.ts new file mode 100644 index 0000000..e549210 --- /dev/null +++ b/packages/server/src/versions/semver.ts @@ -0,0 +1,38 @@ +/** Semver derivation, unchanged from v1 (src/lib/core/semver.ts). */ +export interface SemverComponents { + semver: string + major: number + minor: number + patch: number +} + +export function parseSemver(semver: string): SemverComponents { + const parts = semver.replace(/^v/, '').split('.').map(Number) + const [major, minor, patch] = [parts[0] ?? 1, parts[1] ?? 0, parts[2] ?? 0] + return { semver: `v${major}.${minor}.${patch}`, major, minor, patch } +} + +export function compareSemver(a: string, b: string): number { + const pa = parseSemver(a) + const pb = parseSemver(b) + return pa.major - pb.major || pa.minor - pb.minor || pa.patch - pb.patch +} + +/** Major on a schema change, minor on a record change, patch otherwise (metadata). */ +export function deriveSemver( + prevSemver: string | null, + schemaChanged: boolean, + recordsChanged: boolean, +): SemverComponents { + if (!prevSemver) return { semver: 'v1.0.0', major: 1, minor: 0, patch: 0 } + const { major, minor, patch } = parseSemver(prevSemver) + if (schemaChanged) return { semver: `v${major + 1}.0.0`, major: major + 1, minor: 0, patch: 0 } + if (recordsChanged) + return { semver: `v${major}.${minor + 1}.0`, major, minor: minor + 1, patch: 0 } + return { semver: `v${major}.${minor}.${patch + 1}`, major, minor, patch: patch + 1 } +} + +export type BumpType = 'major' | 'minor' | 'patch' + +export const bumpType = (schemaChanged: boolean, recordsChanged: boolean): BumpType => + schemaChanged ? 'major' : recordsChanged ? 'minor' : 'patch' diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index b581b3a..1670946 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -9,8 +9,10 @@ import type { Queue, ScheduledController, } from '@cloudflare/workers-types' +import { ed25519Signer, type Signer } from '@underlay/repo' import { S3BlobStore } from '@underlay/repo/blob/s3' +import './handlers.js' import { createApp } from './app.js' import { CfCache } from './cache.js' import { openD1 } from './db/d1.js' @@ -27,10 +29,13 @@ export interface Env { R2_BUCKET: string R2_ACCESS_KEY_ID: string R2_SECRET_ACCESS_KEY: string + SIGNING_KEY: string REPO_PREFIX?: string INTERNAL_PREFIX?: string } +let signer: Promise | null = null + function makePorts(env: Env, ctx: ExecutionContext): Ports { const db = openD1(env.DB) const cache = new CfCache(caches as never, env.DEPLOYMENT) @@ -49,6 +54,7 @@ function makePorts(env: Env, ctx: ExecutionContext): Ports { internalPrefix: env.INTERNAL_PREFIX ?? 'internal', }), cache, + signer: () => (signer ??= ed25519Signer(env.SIGNING_KEY)), jobs: new QueueJobs(env.JOBS as never), waitUntil: (p) => ctx.waitUntil(p), } diff --git a/packages/server/test/commit.test.ts b/packages/server/test/commit.test.ts new file mode 100644 index 0000000..0719fd7 --- /dev/null +++ b/packages/server/test/commit.test.ts @@ -0,0 +1,329 @@ +import { + type Change, + compareUtf8, + fileTree, + getEntry, + hashRecord, + hashSchema, + iterate, + type RecordEntry, + recordTree, + utf8ByteLength, +} from '@underlay/core' +import { readHead, RepoSource, verifyLog } from '@underlay/repo' +import { eq } from 'drizzle-orm' +import { afterAll, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { type BaseVersion, commitVersion, type TypeInput } from '../src/versions/commit.js' +import { MissingFilesError } from '../src/versions/file-refs.js' +import { cleanup, harness } from './harness.js' + +afterAll(cleanup) + +const authorSchema = { type: 'object', properties: { name: { type: 'string' }, photo: {} } } +const secretSchema = { type: 'object', private: true, properties: { note: { type: 'string' } } } + +const rec = (type: string, id: string, data: unknown): RecordEntry => { + const { hash, canonical } = hashRecord(id, type, data) + return { key: id, hash, size: utf8ByteLength(canonical), body: canonical } +} +const up = (type: string, id: string, data: unknown): Change => ({ + key: id, + entry: rec(type, id, data), +}) +const del = (id: string): Change => ({ key: id, entry: null }) +const sorted = (cs: Change[]) => cs.sort((a, b) => compareUtf8(a.key, b.key)) + +const type = ( + slug: string, + s: Record, + pub: Change[] | null, + priv: Change[] | null = null, +): TypeInput => ({ + slug, + schema: s, + schemaHash: hashSchema(s), + public: pub && sorted(pub), + private: priv && sorted(priv), +}) + +const baseOf = (v: typeof schema.versions.$inferSelect): BaseVersion => ({ + id: v.id, + seq: v.seq, + semver: v.semver, + hash: v.hash, + publicRefsRoot: v.publicRefsRoot, + privateRefsRoot: v.privateRefsRoot, +}) + +async function collect(it: AsyncIterable): Promise { + const out: T[] = [] + for await (const x of it) out.push(x) + return out +} + +const FILE = 'f'.repeat(64) + +describe('commitVersion', () => { + it('commits, publishes by CAS, and writes the signed log', async () => { + const h = await harness() + const c = await h.collection() + const authors = Array.from({ length: 3000 }, (_, i) => up('Author', `a${i}`, { name: `A${i}` })) + const r1 = await commitVersion(h.ports, { + collectionId: c.id, + base: null, + types: [type('Author', authorSchema, authors, [up('Author', 'hidden', { name: 'H' })])], + metadata: { title: 'Test', tags: ['x'] }, + message: 'first', + }) + expect(r1.status).toBe('committed') + if (r1.status !== 'committed') return + const v1 = r1.version + expect(v1).toMatchObject({ + seq: 1, + semver: 'v1.0.0', + recordCount: 3001, + publicRecordCount: 3000, + hasPrivate: true, + }) + expect(v1.hash).toMatch(/^ulv2:[0-9a-f]{64}$/) + const [col] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, c.id)) + expect(col!.headVersionId).toBe(v1.id) + expect(col!.summary).toEqual({ title: 'Test', tags: ['x'] }) + + const repo = await h.ports.stores.forCollection(c.id) + const root = await repo.root(v1.hash) + expect(root.public.types.Author!.count).toBe(3000) + expect(root.private).toMatch(/^[0-9a-f]{64}$/) + const priv = await repo.privateSet(root.private!) + expect(priv.types.Author!.count).toBe(1) + expect(priv.salt).toBe(c.privateSalt) + // Public readers can't see the hidden record: it's not in the public tree. + const source = new RepoSource(recordTree, repo) + expect(await getEntry(source, root.public.types.Author!.root, 'hidden')).toBe(null) + + const { entries } = await verifyLog(repo, c.id, [h.signer.publicKey]) + expect(entries.map((e) => [e.seq, e.versionHash, e.message])).toEqual([[1, v1.hash, 'first']]) + + // A small change: minor bump, O(changes) writes. + const putsBefore = h.bucket.puts + const r2 = await commitVersion(h.ports, { + collectionId: c.id, + base: baseOf(v1), + types: [ + type('Author', authorSchema, [ + up('Author', 'a5', { name: 'changed' }), + del('a6'), + up('Author', 'zz', { name: 'Z' }), + ]), + ], + metadata: { title: 'Test', tags: ['x'] }, + }) + expect(r2.status).toBe('committed') + if (r2.status !== 'committed') return + expect(r2.version).toMatchObject({ + seq: 2, + semver: 'v1.1.0', + recordCount: 3001, + changes: { added: 1, removed: 1, updated: 1 }, + }) + expect(h.bucket.puts - putsBefore).toBeLessThan(30) + expect((await readHead(repo, c.id))!.seq).toBe(2) + + // The same content again: no change. A metadata edit: patch. + const r3 = await commitVersion(h.ports, { + collectionId: c.id, + base: baseOf(r2.version), + types: [type('Author', authorSchema, null)], + metadata: { title: 'Test', tags: ['x'] }, + }) + expect(r3.status).toBe('no_changes') + const r4 = await commitVersion(h.ports, { + collectionId: c.id, + base: baseOf(r2.version), + types: [type('Author', authorSchema, null)], + metadata: { title: 'Renamed' }, + }) + expect(r4.status === 'committed' && r4.version.semver).toBe('v1.1.1') + if (r4.status !== 'committed') return + const root4 = await repo.root(r4.version.hash) + expect(root4.public).toEqual((await repo.root(r2.version.hash)).public) + await expect(verifyLog(repo, c.id, [h.signer.publicKey])).resolves.toBeTruthy() + expect(await h.drain()).toBeGreaterThan(0) + }) + + it('refuses a commit whose base is no longer the head', async () => { + const h = await harness() + const c = await h.collection() + const r1 = await commitVersion(h.ports, { + collectionId: c.id, + base: null, + types: [type('Author', authorSchema, [up('Author', 'a', { name: 'A' })])], + metadata: null, + }) + if (r1.status !== 'committed') throw new Error(r1.status) + const fromV1 = (name: string) => + commitVersion(h.ports, { + collectionId: c.id, + base: baseOf(r1.version), + types: [type('Author', authorSchema, [up('Author', 'a', { name })])], + metadata: null, + }) + const [x, y] = [await fromV1('X'), await fromV1('Y')] + expect(x.status).toBe('committed') + expect(y.status).toBe('conflict') + const versions = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.collectionId, c.id)) + expect(versions.map((v) => v.seq).sort()).toEqual([1, 2]) + // A commit racing from an empty collection also loses cleanly. + const fromNull = await commitVersion(h.ports, { + collectionId: c.id, + base: null, + types: [type('Author', authorSchema, [up('Author', 'b', { name: 'B' })])], + metadata: null, + }) + expect(fromNull.status).toBe('conflict') + }) + + it('moves types between sets and removes types', async () => { + const h = await harness() + const c = await h.collection() + const repo = await h.ports.stores.forCollection(c.id) + const v1 = await commitVersion(h.ports, { + collectionId: c.id, + base: null, + types: [ + type('Author', authorSchema, [ + up('Author', 'a', { name: 'A' }), + up('Author', 'b', { name: 'B' }), + ]), + type('Secret', secretSchema, null, [up('Secret', 's', { note: 'shh' })]), + ], + metadata: null, + }) + if (v1.status !== 'committed') throw new Error(v1.status) + const root1 = await repo.root(v1.version.hash) + expect(Object.keys(root1.public.types)).toEqual(['Author']) + expect(Object.keys((await repo.privateSet(root1.private!)).types)).toEqual(['Secret']) + + // Author becomes private: its tree moves to the private set unchanged. + const privAuthor = { ...authorSchema, private: true } + const v2 = await commitVersion(h.ports, { + collectionId: c.id, + base: baseOf(v1.version), + types: [type('Author', privAuthor, null), type('Secret', secretSchema, null)], + metadata: null, + }) + if (v2.status !== 'committed') throw new Error(v2.status) + expect(v2.version.semver).toBe('v2.0.0') + const root2 = await repo.root(v2.version.hash) + const priv2 = await repo.privateSet(root2.private!) + expect(root2.public.types).toEqual({}) + expect(priv2.types.Author!.root).toBe(root1.public.types.Author!.root) + + // Secret is removed; Author becomes public again. + const v3 = await commitVersion(h.ports, { + collectionId: c.id, + base: baseOf(v2.version), + types: [type('Author', authorSchema, null)], + metadata: null, + }) + if (v3.status !== 'committed') throw new Error(v3.status) + const root3 = await repo.root(v3.version.hash) + expect(root3.private).toBe(null) + expect(root3.public.types.Author!.root).toBe(root1.public.types.Author!.root) + expect(v3.version.changes).toMatchObject({ removed: 1 }) + }) + + it('keeps file sets by reference', async () => { + const h = await harness() + const c = await h.collection() + const repo = await h.ports.stores.forCollection(c.id) + const withPhoto = (id: string) => + up('Author', id, { name: id, photo: { $file: `sha256:${FILE}` } }) + const attempt = await commitVersion(h.ports, { + collectionId: c.id, + base: null, + types: [type('Author', authorSchema, [withPhoto('a')])], + metadata: null, + }).catch((e) => e) + expect(attempt).toBeInstanceOf(MissingFilesError) + + await h.ports.db + .insert(schema.files) + .values({ hash: FILE, size: 1234, mimeType: 'image/png', storageKey: `files/${FILE}` }) + const v1 = await commitVersion(h.ports, { + collectionId: c.id, + base: null, + types: [type('Author', authorSchema, [withPhoto('a'), withPhoto('b')], [withPhoto('p')])], + metadata: null, + }) + if (v1.status !== 'committed') throw new Error(v1.status) + const root1 = await repo.root(v1.version.hash) + expect(root1.public.files).toMatchObject({ count: 1, bytes: 1234 }) + expect((await repo.privateSet(root1.private!)).files.count).toBe(1) + + // Dropping one public reference keeps the file; dropping both removes it + // from the public set, but the cumulative public files tree keeps it. + const v2 = await commitVersion(h.ports, { + collectionId: c.id, + base: baseOf(v1.version), + types: [type('Author', authorSchema, [del('a')], null)], + metadata: null, + }) + if (v2.status !== 'committed') throw new Error(v2.status) + expect((await repo.root(v2.version.hash)).public.files.count).toBe(1) + const v3 = await commitVersion(h.ports, { + collectionId: c.id, + base: baseOf(v2.version), + types: [type('Author', authorSchema, [del('b')], null)], + metadata: null, + }) + if (v3.status !== 'committed') throw new Error(v3.status) + expect((await repo.root(v3.version.hash)).public.files.count).toBe(0) + const [col] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, c.id)) + const cumulative = await collect(iterate(new RepoSource(fileTree, repo), col!.publicFilesRoot)) + expect(cumulative.map((f) => f.key)).toEqual([FILE]) + }) + + it('revalidates records when a schema changes', async () => { + const h = await harness() + const c = await h.collection() + const v1 = await commitVersion(h.ports, { + collectionId: c.id, + base: null, + types: [ + type('Author', authorSchema, [ + up('Author', 'a', { name: 'A' }), + up('Author', 'b', { name: 7 }), + ]), + ], + metadata: null, + }) + if (v1.status !== 'committed') throw new Error(v1.status) + const strict = { ...authorSchema, required: ['name'], additionalProperties: false } + const validate = (_s: Record, data: unknown) => + typeof (data as { name?: unknown }).name === 'string' ? null : ['/name must be string'] + const r = await commitVersion(h.ports, { + collectionId: c.id, + base: baseOf(v1.version), + types: [type('Author', strict, null)], + metadata: null, + validate, + }) + expect(r).toMatchObject({ + status: 'invalid', + total: 1, + errors: [{ recordId: 'b', type: 'Author' }], + }) + }) +}) diff --git a/packages/server/test/db.test.ts b/packages/server/test/db.test.ts index 203b6fb..24c8675 100644 --- a/packages/server/test/db.test.ts +++ b/packages/server/test/db.test.ts @@ -69,6 +69,9 @@ describe('SQLite jobs', () => { internalPrefix: 'internal', }), cache: new MemoryCache(), + signer: async () => { + throw new Error('not used') + }, jobs: new SqliteJobs(db), waitUntil: () => {}, } diff --git a/packages/server/test/harness.ts b/packages/server/test/harness.ts new file mode 100644 index 0000000..8d78d1f --- /dev/null +++ b/packages/server/test/harness.ts @@ -0,0 +1,73 @@ +/** Ports over a temp SQLite file and an in-memory bucket, for integration tests. */ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { newSalt } from '@underlay/core' +import { ed25519Signer, generateSigningKey, type Signer } from '@underlay/repo' +import { MemoryBlobStore } from '@underlay/repo/blob/memory' + +import '../src/handlers.js' +import { MemoryCache } from '../src/cache.js' +import { openNodeDb } from '../src/db/node.js' +import * as schema from '../src/db/schema.js' +import { drainSqliteJobs, SqliteJobs } from '../src/jobs.js' +import type { Ports } from '../src/ports.js' +import { createStores } from '../src/stores.js' + +const dirs: string[] = [] + +export async function cleanup(): Promise { + for (const d of dirs.splice(0)) await rm(d, { recursive: true, force: true }) +} + +export interface Harness { + ports: Ports + bucket: MemoryBlobStore + signer: Signer + /** Run queued jobs until none are ready. */ + drain(): Promise + /** An org and a collection with a primary placement on the platform location. */ + collection(slug?: string): Promise +} + +export async function harness(): Promise { + const dir = await mkdtemp(join(tmpdir(), 'ul-it-')) + dirs.push(dir) + const db = await openNodeDb(`file:${join(dir, 'db.sqlite')}`) + const bucket = new MemoryBlobStore() + const cache = new MemoryCache() + const signer = await ed25519Signer(await generateSigningKey()) + const ports: Ports = { + db, + stores: createStores(db, cache, { bucket, repoPrefix: 'repo', internalPrefix: 'internal' }), + cache, + signer: async () => signer, + jobs: new SqliteJobs(db), + waitUntil: (p) => void p.catch((err) => console.error(err)), + } + let orgMade = false + return { + ports, + bucket, + signer, + drain: () => drainSqliteJobs(ports), + async collection(slug = 'c') { + if (!orgMade) { + await db.insert(schema.organization).values({ id: 'org1', name: 'Org', slug: 'org' }) + orgMade = true + } + const [c] = await db + .insert(schema.collections) + .values({ organizationId: 'org1', slug, name: slug, privateSalt: newSalt() }) + .returning() + await db.insert(schema.placements).values({ + collectionId: c!.id, + locationId: schema.PLATFORM_LOCATION_ID, + role: 'primary', + sets: 'public+private', + }) + return c! + }, + } +} From 8bdf52f1df2c4e7314260a947fb17e051affbfcf Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 16:53:07 -0400 Subject: [PATCH 006/178] v2 write path: push sessions, delta push, negotiate compatibility - Sorted runs in the platform's internal area: blocks of ~1 MB, an index per run for per-type reads, k-way merge with later uploads winning, compaction above a fan-in of 16 (property-tested). - Push sessions: inputs object + session row, run bookkeeping, status transitions by compare-and-swap, finalize shared by inline and job-run commits. - Delta push API (/push): upserts and deletes against a base, private flags routed to sets, records validated and hashed at upload, large records stored out of line immediately. - Negotiate API (v1 paths and shapes): presence only from the base version (no global lookup), snapshot diff against the base trees at commit, set moves without re-upload, v1 metadata merge, declared files as a full list, format 1 hashes matched for records with integer-like keys. - Commit now stores schema bodies in the repository. - Collection access resolved once per request; pluggable authenticator (anonymous until better-auth lands). --- packages/server/package.json | 1 + packages/server/src/api/access.ts | 105 +++++ packages/server/src/api/push.ts | 617 +++++++++++++++++++++++++ packages/server/src/app.ts | 24 +- packages/server/src/node/main.ts | 3 +- packages/server/src/push/delta.ts | 330 +++++++++++++ packages/server/src/push/finalize.ts | 110 +++++ packages/server/src/push/negotiate.ts | 363 +++++++++++++++ packages/server/src/push/runs.ts | 256 ++++++++++ packages/server/src/push/session.ts | 139 ++++++ packages/server/src/versions/commit.ts | 10 + packages/server/src/worker.ts | 2 + packages/server/test/harness.ts | 59 ++- packages/server/test/push.test.ts | 324 +++++++++++++ packages/server/test/runs.test.ts | 77 +++ pnpm-lock.yaml | 3 + 16 files changed, 2412 insertions(+), 11 deletions(-) create mode 100644 packages/server/src/api/access.ts create mode 100644 packages/server/src/api/push.ts create mode 100644 packages/server/src/push/delta.ts create mode 100644 packages/server/src/push/finalize.ts create mode 100644 packages/server/src/push/negotiate.ts create mode 100644 packages/server/src/push/runs.ts create mode 100644 packages/server/src/push/session.ts create mode 100644 packages/server/test/push.test.ts create mode 100644 packages/server/test/runs.test.ts diff --git a/packages/server/package.json b/packages/server/package.json index e9699d4..885f7c0 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -25,6 +25,7 @@ "@cloudflare/workers-types": "^5.20261003.1", "@types/node": "^25.0.0", "drizzle-kit": "^0.31.10", + "fast-check": "^4.10.2", "tsx": "^4.19.0", "typescript": "^6.0.0", "vitest": "^4.1.6" diff --git a/packages/server/src/api/access.ts b/packages/server/src/api/access.ts new file mode 100644 index 0000000..89daf30 --- /dev/null +++ b/packages/server/src/api/access.ts @@ -0,0 +1,105 @@ +/** + * Who is calling, and what they may do with a collection. + * + * Authorization happens once per request: the collection's visibility and the + * caller's membership pick which sets the caller may read (edge-redesign.md, + * Read path). Nothing below filters individual records. + */ +import { and, eq } from 'drizzle-orm' +import type { Context } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import type { Db } from '../ports.js' + +export interface Principal { + userId: string + /** 'session' for a signed-in browser; otherwise the API key's scope. */ + scope: 'session' | 'read' | 'write' | 'admin' + /** Collections an API key is limited to; null when unscoped. */ + collectionIds: string[] | null +} + +export interface CollectionAccess { + collection: typeof schema.collections.$inferSelect + owner: typeof schema.organization.$inferSelect + /** Members of the owning org read both sets. */ + isMember: boolean + canRead: boolean + canWrite: boolean + /** Which sets this caller may read. */ + sets: ('public' | 'private')[] +} + +export async function collectionAccess( + db: Db, + principal: Principal | null, + ownerSlug: string, + slug: string, +): Promise { + const [row] = await db + .select({ collection: schema.collections, owner: schema.organization }) + .from(schema.collections) + .innerJoin(schema.organization, eq(schema.organization.id, schema.collections.organizationId)) + .where(and(eq(schema.organization.slug, ownerSlug), eq(schema.collections.slug, slug))) + .limit(1) + if (!row || row.collection.deletedAt) return null + + let isMember = false + if (principal) { + const keyCovers = + principal.collectionIds === null || principal.collectionIds.includes(row.collection.id) + if (keyCovers) { + const [m] = await db + .select({ id: schema.member.id }) + .from(schema.member) + .where( + and( + eq(schema.member.organizationId, row.owner.id), + eq(schema.member.userId, principal.userId), + ), + ) + .limit(1) + isMember = !!m + } + } + const canRead = row.collection.public || isMember + const canWrite = isMember && principal !== null && principal.scope !== 'read' + return { + ...row, + isMember, + canRead, + canWrite, + sets: isMember ? ['public', 'private'] : ['public'], + } +} + +export const jsonError = ( + c: Context, + status: 400 | 401 | 403 | 404 | 409 | 413 | 422 | 500, + error: string, + extra: Record = {}, +) => c.json({ error, statusCode: status, ...extra }, status) + +/** + * Resolve `:owner/:slug` for a request. A collection the caller can't read is a + * 404, so private collections don't leak their existence. + */ +export async function requireCollection( + c: Context, + need: 'read' | 'write', +): Promise { + const access = await collectionAccess( + c.var.ports.db, + c.var.principal, + c.req.param('owner') ?? '', + c.req.param('slug') ?? '', + ) + if (!access || !access.canRead) return jsonError(c, 404, 'Collection not found') + if (need === 'write' && !access.canWrite) { + return c.var.principal + ? jsonError(c, 403, 'Not authorized') + : jsonError(c, 401, 'Authentication required') + } + return access +} diff --git a/packages/server/src/api/push.ts b/packages/server/src/api/push.ts new file mode 100644 index 0000000..819461f --- /dev/null +++ b/packages/server/src/api/push.ts @@ -0,0 +1,617 @@ +/** + * Push routes, mounted at /api/collections. + * + * Delta push (new): + * POST /:owner/:slug/push open a session against a base + * POST /:owner/:slug/push/:sid/records NDJSON {id,type,data,private?} + * POST /:owner/:slug/push/:sid/deletes NDJSON {type,id} + * POST /:owner/:slug/push/:sid/commit ?async=true for a job + * GET /:owner/:slug/push/:sid status (and result after an async commit) + * DELETE /:owner/:slug/push/:sid abandon + * + * Negotiate (v1 compatibility, same paths and shapes as v1): + * POST /:owner/:slug/versions/negotiate + * POST /:owner/:slug/versions/negotiate/:sid/manifest + * POST /:owner/:slug/versions/negotiate/:sid/records + * POST /:owner/:slug/versions/negotiate/:sid/commit + * GET /:owner/:slug/versions/negotiate/:sid + * DELETE /:owner/:slug/versions/negotiate/:sid + */ +import { checkSchema, getEntry, fileTree, InputRuleError, parseStrict } from '@underlay/core' +import { RepoSource } from '@underlay/repo' +import { type Context, Hono } from 'hono' + +import type { AppEnv } from '../app.js' +import { registerJob } from '../jobs.js' +import { + headBase, + ingestDeletes, + ingestRecords, + MAX_BATCH_BYTES, + prepareRecords, +} from '../push/delta.js' +import { finalizeSession } from '../push/finalize.js' +import { + baseTrees, + type ManifestLine, + neededOf, + parseManifestLine, + withLegacyHash, +} from '../push/negotiate.js' +import { writeRun } from '../push/runs.js' +import { + createSession, + getSession, + loadInputs, + nextRunSeq, + recordRun, + type SessionInputs, + type SessionRow, + transition, +} from '../push/session.js' +import { parseSemver } from '../versions/semver.js' +import { type CollectionAccess, jsonError, requireCollection } from './access.js' + +const MAX_OPEN_BYTES = 8 * 1024 * 1024 +const MAX_MANIFEST_CHUNK = 50_000 +const MAX_INLINE_MANIFEST = 50_000 +/** Commits above this many uploaded records run as a job even without ?async. */ +const ASYNC_ABOVE = 100_000 + +registerJob('push.commit', async (job, ports) => { + await finalizeSession(ports, String(job.sessionId)) +}) + +class BodyTooLarge extends Error {} + +/** Read a request body as text, refusing more than `max` bytes without buffering them. */ +async function readText(c: Context, max: number): Promise { + const declared = Number(c.req.header('content-length') ?? NaN) + if (declared > max) throw new BodyTooLarge() + const body = c.req.raw.body + if (!body) return '' + const reader = body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + for (;;) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > max) { + await reader.cancel() + throw new BodyTooLarge() + } + chunks.push(value) + } + return new TextDecoder().decode(Buffer.concat(chunks)) +} + +type JsonBody = Record + +/** Parse a JSON body under the input rules (duplicate keys, unsafe integers, …). */ +async function readJson(c: Context, max: number): Promise { + let text: string + try { + text = await readText(c, max) + } catch (err) { + if (err instanceof BodyTooLarge) return jsonError(c, 413, `Body exceeds ${max} bytes`) + throw err + } + if (text.trim() === '') return {} + try { + const v = parseStrict(text, 128) + if (v === null || typeof v !== 'object' || Array.isArray(v)) + return jsonError(c, 400, 'Body must be a JSON object') + return v as JsonBody + } catch (err) { + if (err instanceof InputRuleError) return jsonError(c, 400, `${err.code}: ${err.message}`) + throw err + } +} + +async function readNdjson(c: Context): Promise { + try { + return await readText(c, MAX_BATCH_BYTES) + } catch (err) { + if (err instanceof BodyTooLarge) + return jsonError(c, 413, `Batches are limited to ${MAX_BATCH_BYTES} bytes`) + throw err + } +} + +const str = (v: unknown): string | null => (typeof v === 'string' ? v : null) + +/** Validate a full schema set. Returns an error message or null. */ +function checkSchemas(schemas: unknown): string | null { + if (!schemas || typeof schemas !== 'object' || Array.isArray(schemas)) + return '"schemas" must be an object of type → schema' + for (const [slug, body] of Object.entries(schemas)) { + if (!body || typeof body !== 'object' || Array.isArray(body)) + return `Schema "${slug}" must be an object` + const err = checkSchema(slug, body) + if (err) return err + } + return null +} + +/** The session named in the URL, if it belongs to this collection and caller. */ +async function ownSession( + c: Context, + access: CollectionAccess, + kind: SessionRow['kind'], +): Promise { + const session = await getSession(c.var.ports, c.req.param('sid') ?? '') + if (!session || session.collectionId !== access.collection.id || session.kind !== kind) { + return jsonError(c, 404, 'Session not found') + } + if (session.userId !== c.var.principal?.userId) return jsonError(c, 403, 'Not your session') + return session +} + +/** The schemas and metadata of a base version (for sessions that keep them). */ +async function baseInputs(c: Context, collectionId: string, baseHash: string | null) { + if (!baseHash) return { schemas: {}, metadata: null } + const repo = await c.var.ports.stores.forCollection(collectionId) + const root = await repo.root(baseHash) + const priv = root.private ? await repo.privateSet(root.private) : null + const hashes = new Map() + for (const [slug, t] of Object.entries(priv?.types ?? {})) hashes.set(slug, t.schema) + for (const [slug, t] of Object.entries(root.public.types)) hashes.set(slug, t.schema) + const schemas: Record> = {} + for (const [slug, h] of hashes) schemas[slug] = await repo.schema(h) + return { schemas, metadata: root.metadata } +} + +/** Check a requested base semver against the head. Null/undefined means "no check". */ +function baseConflict(requested: unknown, head: { semver: string } | null): boolean { + if (requested === null || requested === undefined) return false + if (typeof requested !== 'string') return true + return parseSemver(requested).semver !== (head?.semver ?? null) +} + +async function commitRoute(c: Context, session: SessionRow) { + const ports = c.var.ports + const body = await readJson(c, 64 * 1024) + if (body instanceof Response) return body + const wantsAsync = + ['true', '1'].includes(c.req.query('async') ?? '') || + body.async === true || + session.recordsReceived > ASYNC_ABOVE + if ( + !(await transition(ports, session.id, 'open', 'committing', { finalizeStartedAt: new Date() })) + ) { + const now = await getSession(ports, session.id) + if (now?.status === 'committed') return c.json(now.result ?? {}, 201) + return jsonError(c, 409, `Session is ${now?.status ?? 'gone'}`) + } + if (wantsAsync) { + await ports.jobs.enqueue({ type: 'push.commit', sessionId: session.id }) + return c.json( + { + session_id: session.id, + status: 'committing', + poll: `GET ${new URL(c.req.url).pathname.replace(/\/commit$/, '')}`, + }, + 202, + ) + } + const outcome = await finalizeSession(ports, session.id) + return c.json(outcome.body, outcome.status) +} + +function statusBody(s: SessionRow) { + return { + session_id: s.id, + status: s.status, + total_records: s.manifestReceived, + needed_records: Math.max(0, s.manifestNeeded - s.recordsReceived), + records_received: s.recordsReceived, + expires_at: s.expiresAt, + created_at: s.createdAt, + finalize_started_at: s.finalizeStartedAt, + result: s.result ?? null, + error: s.error ?? null, + } +} + +async function abandon(c: Context, session: SessionRow) { + await transition(c.var.ports, session.id, 'open', 'expired') + return c.json({ ok: true }) +} + +/** Declared files the caller must upload: those the collection's base doesn't already hold. */ +async function neededFiles( + c: Context, + access: CollectionAccess, + baseHash: string | null, + files: string[], +): Promise { + if (files.length === 0) return [] + const repo = await c.var.ports.stores.forCollection(access.collection.id) + const roots: (string | null)[] = [access.collection.publicFilesRoot] + if (baseHash) { + const root = await repo.root(baseHash) + roots.push(root.public.files.root) + if (root.private) roots.push((await repo.privateSet(root.private)).files.root) + } + const source = new RepoSource(fileTree, repo) + const needed: string[] = [] + for (const h of files) { + let have = false + for (const r of roots) if (r && (await getEntry(source, r, h))) have = true + if (!have) needed.push(h) + } + return needed +} + +export function pushRoutes() { + const app = new Hono() + + // --- Delta push --------------------------------------------------------------- + + app.post('/:owner/:slug/push', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const body = await readJson(c, MAX_OPEN_BYTES) + if (body instanceof Response) return body + const ports = c.var.ports + const head = await headBase(ports, access.collection.id) + if (baseConflict(body.base, head)) + return jsonError(c, 409, 'Version conflict', { currentVersion: head?.semver ?? null }) + + const base = await baseInputs(c, access.collection.id, head?.hash ?? null) + if (body.schemas !== undefined) { + const err = checkSchemas(body.schemas) + if (err) return jsonError(c, 422, err) + } + if ( + body.metadata !== undefined && + body.metadata !== null && + typeof body.metadata !== 'object' + ) { + return jsonError(c, 400, '"metadata" must be an object or null') + } + const patch = body.metadata_patch as Record | undefined + const metadata = + body.metadata !== undefined + ? (body.metadata as Record | null) + : patch + ? { ...(base.metadata ?? {}), ...patch } + : base.metadata + const files = (body.files ?? {}) as { add?: unknown; remove?: unknown } + const hexList = (v: unknown) => + Array.isArray(v) + ? v.filter((h): h is string => typeof h === 'string' && /^[0-9a-f]{64}$/.test(h)) + : [] + const inputs: SessionInputs = { + schemas: (body.schemas as SessionInputs['schemas'] | undefined) ?? base.schemas, + metadata, + files: { add: hexList(files.add), remove: hexList(files.remove) }, + } + const session = await createSession( + ports, + { + collectionId: access.collection.id, + userId: c.var.principal!.userId, + kind: 'delta', + baseVersionId: head?.id ?? null, + baseSemver: head?.semver ?? null, + message: str(body.message), + appId: str(body.app_id), + actorId: str(body.actor_id), + stripUnknownFields: body.strip_unknown_fields === true, + }, + inputs, + ) + return c.json({ + session_id: session.id, + base: head?.semver ?? null, + needed_files: await neededFiles( + c, + access, + head?.hash ?? null, + inputs.files && 'add' in inputs.files ? inputs.files.add : [], + ), + expires_at: session.expiresAt, + }) + }) + + app.post('/:owner/:slug/push/:sid/records', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'delta') + if (session instanceof Response) return session + const text = await readNdjson(c) + if (text instanceof Response) return text + const r = await ingestRecords(c.var.ports, session, text) + return r.ok + ? c.json({ received: r.received }) + : jsonError(c, r.status, r.error, { validationErrors: r.details, totalErrors: r.total }) + }) + + app.post('/:owner/:slug/push/:sid/deletes', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'delta') + if (session instanceof Response) return session + const text = await readNdjson(c) + if (text instanceof Response) return text + const r = await ingestDeletes(c.var.ports, session, text) + return r.ok + ? c.json({ received: r.received }) + : jsonError(c, r.status, r.error, { errors: r.details }) + }) + + app.post('/:owner/:slug/push/:sid/commit', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'delta') + if (session instanceof Response) return session + return commitRoute(c, session) + }) + + app.get('/:owner/:slug/push/:sid', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'delta') + if (session instanceof Response) return session + return c.json(statusBody(session)) + }) + + app.delete('/:owner/:slug/push/:sid', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'delta') + if (session instanceof Response) return session + return abandon(c, session) + }) + + // --- Negotiate (v1 compatibility) ---------------------------------------------------- + + /** Validate manifest entries and store them as a run; returns the needed hashes. */ + const ingestManifest = async ( + c: Context, + session: SessionRow, + entries: ManifestLine[], + ) => { + const ports = c.var.ports + const inputs = await loadInputs(ports, session.id) + for (const m of entries) { + if (!inputs.schemas[m.type]) + throw new ManifestError(`No schema defined for record type "${m.type}"`) + } + const head = await headBase(ports, session.collectionId) + const trees = await baseTrees(ports, session.collectionId, head?.hash ?? null) + const needed = await neededOf(trees, entries) + const seq = await nextRunSeq(ports, session.id) + if (seq === null) throw new ManifestError('Session is not open', 409) + const run = entries.map((m) => ({ + t: m.type, + k: m.id, + h: m.hash, + ...(m.private ? { p: true } : {}), + })) + const index = await writeRun(ports.stores.internal, session.id, seq, run) + await recordRun(ports, session.id, 'manifest', index, { + manifest: entries.length, + needed: needed.length, + }) + return needed + } + + app.post('/:owner/:slug/versions/negotiate', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const body = await readJson(c, 64 * 1024 * 1024) + if (body instanceof Response) return body + const err = checkSchemas(body.schemas) + if (err) return jsonError(c, 422, err) + const ports = c.var.ports + const head = await headBase(ports, access.collection.id) + if (baseConflict(body.base_version, head)) { + return jsonError(c, 409, 'Version conflict', { currentVersion: head?.semver ?? null }) + } + const inline = Array.isArray(body.manifest) ? body.manifest : [] + const chunked = body.manifest_expected !== undefined + if (chunked && inline.length > 0) { + return jsonError( + c, + 400, + 'Send either an inline `manifest` or `manifest_expected` with chunked upload, not both.', + ) + } + if (inline.length > MAX_INLINE_MANIFEST) { + return jsonError( + c, + 413, + `Inline manifests are limited to ${MAX_INLINE_MANIFEST} entries; set manifest_expected and upload chunks.`, + ) + } + const entries: ManifestLine[] = [] + for (const m of inline) { + const parsed = parseManifestLine(m) + if (typeof parsed === 'string') return jsonError(c, 400, `Invalid manifest entry: ${parsed}`) + entries.push(parsed) + } + const files = Array.isArray(body.files) + ? body.files.filter((h): h is string => typeof h === 'string' && /^[0-9a-f]{64}$/.test(h)) + : [] + const session = await createSession( + ports, + { + collectionId: access.collection.id, + userId: c.var.principal!.userId, + kind: 'negotiate', + baseVersionId: head?.id ?? null, + baseSemver: + typeof body.base_version === 'string' ? parseSemver(body.base_version).semver : null, + message: str(body.message), + appId: str(body.app_id), + actorId: str(body.actor_id), + stripUnknownFields: body.strip_unknown_fields === true, + manifestExpected: chunked ? Number(body.manifest_expected) : null, + }, + { + schemas: body.schemas as SessionInputs['schemas'], + metadata: (body.metadata as Record | undefined) ?? null, + files: { all: files }, + }, + ) + const neededFilesList = await neededFiles(c, access, head?.hash ?? null, files) + if (chunked) { + return c.json({ + session_id: session.id, + manifest_expected: session.manifestExpected, + manifest_received: 0, + needed_files: neededFilesList, + total_files: files.length, + already_have_files: files.length - neededFilesList.length, + next: `POST .../versions/negotiate/${session.id}/manifest`, + }) + } + let needed: string[] = [] + try { + if (entries.length > 0) needed = await ingestManifest(c, session, entries) + } catch (err) { + if (err instanceof ManifestError) return jsonError(c, err.status, err.message) + throw err + } + return c.json({ + session_id: session.id, + needed_records: needed, + needed_files: neededFilesList, + total_records: entries.length, + total_files: files.length, + already_have_records: entries.length - needed.length, + already_have_files: files.length - neededFilesList.length, + }) + }) + + app.post('/:owner/:slug/versions/negotiate/:sid/manifest', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'negotiate') + if (session instanceof Response) return session + if (session.manifestExpected === null) { + return jsonError( + c, + 400, + 'This session was opened with an inline manifest. Pass `manifest_expected` at negotiate time to upload the manifest in chunks.', + ) + } + const text = await readNdjson(c) + if (text instanceof Response) return text + const lines = text.split('\n').filter((l) => l.trim()) + if (lines.length > MAX_MANIFEST_CHUNK) + return jsonError( + c, + 400, + `Chunk too large. Maximum ${MAX_MANIFEST_CHUNK} manifest entries per request.`, + ) + const entries: ManifestLine[] = [] + for (const line of lines) { + let v: unknown + try { + v = JSON.parse(line) + } catch { + return jsonError(c, 400, `Invalid JSONL line: ${line.slice(0, 100)}`) + } + const parsed = parseManifestLine(v) + if (typeof parsed === 'string') + return jsonError(c, 400, `Invalid manifest entry: ${line.slice(0, 100)}`, { + details: [parsed], + }) + entries.push(parsed) + } + try { + const needed = await ingestManifest(c, session, entries) + const now = await getSession(c.var.ports, session.id) + return c.json({ + received: entries.length, + needed_records: needed, + manifest_received: now!.manifestReceived, + manifest_expected: session.manifestExpected, + }) + } catch (err) { + if (err instanceof ManifestError) return jsonError(c, err.status, err.message) + throw err + } + }) + + app.post('/:owner/:slug/versions/negotiate/:sid/records', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'negotiate') + if (session instanceof Response) return session + if (session.status !== 'open') return jsonError(c, 409, `Session is ${session.status}`) + const text = await readNdjson(c) + if (text instanceof Response) return text + const ports = c.var.ports + const inputs = await loadInputs(ports, session.id) + const prepared = await prepareRecords(ports, session.collectionId, inputs, text, { + stripUnknownFields: session.stripUnknownFields, + }) + if ('errors' in prepared) { + return jsonError(c, 422, 'Schema validation failed', { + validationErrors: prepared.errors, + totalErrors: prepared.total, + }) + } + if (prepared.entries.length === 0) return jsonError(c, 400, 'Empty batch') + // Keep the format 1 hash next to the v2 one, so a v1 manifest entry matches either. + const datas = new Map( + text + .split('\n') + .filter((l) => l.trim()) + .map((l) => { + const r = JSON.parse(l) as { id: string; type: string; data: unknown } + return [`${r.type}\u0000${r.id}`, r.data] as const + }), + ) + const entries = prepared.entries.map((e) => withLegacyHash(e, datas.get(`${e.t}\u0000${e.k}`))) + const seq = await nextRunSeq(ports, session.id) + if (seq === null) return jsonError(c, 409, 'Session is not open') + const index = await writeRun(ports.stores.internal, session.id, seq, entries) + await recordRun(ports, session.id, 'records', index, { records: entries.length }) + const now = await getSession(ports, session.id) + return c.json({ + received: entries.length, + remaining: Math.max(0, now!.manifestNeeded - now!.recordsReceived), + }) + }) + + app.post('/:owner/:slug/versions/negotiate/:sid/commit', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'negotiate') + if (session instanceof Response) return session + return commitRoute(c, session) + }) + + app.get('/:owner/:slug/versions/negotiate/:sid', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'negotiate') + if (session instanceof Response) return session + return c.json(statusBody(session)) + }) + + app.delete('/:owner/:slug/versions/negotiate/:sid', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const session = await ownSession(c, access, 'negotiate') + if (session instanceof Response) return session + return abandon(c, session) + }) + + return app +} + +class ManifestError extends Error { + constructor( + message: string, + readonly status: 400 | 409 = 400, + ) { + super(message) + } +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 6766858..3efc56a 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -1,10 +1,12 @@ /** * The Hono app. Runtime-agnostic: each entry (Node, Worker) supplies a function - * that builds the ports and config for a request, and everything below reads - * them from the context. + * that builds the ports, config and authenticator for a request, and everything + * below reads them from the context. */ import { type Context, Hono } from 'hono' +import type { Principal } from './api/access.js' +import { pushRoutes } from './api/push.js' import type { Ports } from './ports.js' export interface AppConfig { @@ -14,27 +16,35 @@ export interface AppConfig { deployment: string } +export type Authenticate = (req: Request, ports: Ports) => Promise + export type AppEnv = { Bindings: Record Variables: { ports: Ports config: AppConfig + principal: Principal | null } } /** - * Builds a request's ports and config. Runs per request: on Workers, bindings and - * the execution context belong to the invocation. + * Builds a request's ports, config and authenticator. Runs per request: on + * Workers, bindings and the execution context belong to the invocation. */ -export type Setup = (c: Context) => { ports: Ports; config: AppConfig } +export type Setup = (c: Context) => { + ports: Ports + config: AppConfig + authenticate: Authenticate +} export function createApp(setup: Setup) { const app = new Hono() app.use('*', async (c, next) => { - const { ports, config } = setup(c) + const { ports, config, authenticate } = setup(c) c.set('ports', ports) c.set('config', config) + c.set('principal', await authenticate(c.req.raw, ports)) await next() }) @@ -47,6 +57,8 @@ export function createApp(setup: Setup) { }), ) + app.route('/api/collections', pushRoutes()) + app.notFound((c) => c.json({ error: 'Not found', statusCode: 404 }, 404)) app.onError((err, c) => { console.error('[app]', err) diff --git a/packages/server/src/node/main.ts b/packages/server/src/node/main.ts index 556e12f..38a713c 100644 --- a/packages/server/src/node/main.ts +++ b/packages/server/src/node/main.ts @@ -100,7 +100,8 @@ kick = () => void runJobs() setInterval(kick, 5000).unref() const config = { appUrl, deployment: env.DEPLOYMENT ?? 'dev' } -const app = createApp(() => ({ ports, config })) +// Sign-in and API keys (better-auth + KF Auth) land next; until then every caller is anonymous. +const app = createApp(() => ({ ports, config, authenticate: async () => null })) serve({ port, diff --git a/packages/server/src/push/delta.ts b/packages/server/src/push/delta.ts new file mode 100644 index 0000000..4615003 --- /dev/null +++ b/packages/server/src/push/delta.ts @@ -0,0 +1,330 @@ +/** + * Delta push: the path at scale. A session names a base version and uploads + * upserts (records) and deletes; the commit costs O(changes). + * + * Record lines go through the input rules, schema validation and canonical + * hashing at upload, so commit never re-reads them. Large records are stored out + * of line in the collection's repository right away, so runs stay small. + */ +import { + type Change, + compileSchema, + InputRuleError, + parseRecordLine, + recordCanonical, + type RecordEntry, + sha256Hex, + stripToSchema, + utf8ByteLength, +} from '@underlay/core' +import { OUT_OF_LINE_BYTES } from '@underlay/repo' +import { eq } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Ports } from '../ports.js' +import { + type BaseVersion, + commitVersion, + type CommitResult, + type TypeInput, +} from '../versions/commit.js' +import { + compactRuns, + mergeRuns, + MERGE_FAN_IN, + type RunEntry, + type RunIndex, + writeRun, +} from './runs.js' +import { + loadInputs, + nextRunSeq, + recordRun, + schemaHashes, + type SessionInputs, + sessionRuns, + type SessionRow, + transition, +} from './session.js' + +export const MAX_BATCH_BYTES = 16 * 1024 * 1024 +export const MAX_BATCH_LINES = 10_000 +const MAX_REPORTED = 100 + +export interface LineError { + line: number + recordId?: string + type?: string + errors: string[] +} + +export type IngestResult = + | { ok: true; received: number } + | { ok: false; status: 400 | 409 | 422; error: string; details?: LineError[]; total?: number } + +const isPrivateSchema = (s: Record) => s.private === true + +function lines(text: string): string[] { + return text.split('\n').filter((l) => l.trim().length > 0) +} + +/** Parse, validate and hash one batch of record lines into run entries. */ +export async function prepareRecords( + ports: Ports, + collectionId: string, + inputs: SessionInputs, + text: string, + opts: { stripUnknownFields: boolean }, +): Promise<{ entries: RunEntry[] } | { errors: LineError[]; total: number }> { + const errors: LineError[] = [] + let total = 0 + const fail = (e: LineError) => { + total++ + if (errors.length < MAX_REPORTED) errors.push(e) + } + const entries: RunEntry[] = [] + const repo = await ports.stores.forCollection(collectionId) + const all = lines(text) + for (let i = 0; i < all.length; i++) { + let rec + try { + rec = parseRecordLine(all[i]!) + } catch (err) { + if (!(err instanceof InputRuleError)) throw err + fail({ line: i + 1, errors: [`${err.code}: ${err.message}`] }) + continue + } + const typeSchema = inputs.schemas[rec.type] + if (!typeSchema) { + fail({ + line: i + 1, + recordId: rec.id, + type: rec.type, + errors: [`No schema for type "${rec.type}"`], + }) + continue + } + let data = rec.data + let canonical = rec.canonical + const props = typeSchema.properties as Record | undefined + if (props && data !== null && typeof data === 'object' && !Array.isArray(data)) { + const extra = Object.keys(data).filter((k) => !(k in props)) + if (extra.length > 0) { + if (!opts.stripUnknownFields) { + fail({ + line: i + 1, + recordId: rec.id, + type: rec.type, + errors: [ + `Fields not in the schema: ${extra.join(', ')} (set strip_unknown_fields to drop them)`, + ], + }) + continue + } + data = stripToSchema(data as Record, props) + canonical = recordCanonical(rec.id, rec.type, data) + } + } + const errs = compileSchema(typeSchema)(data) + if (errs.length > 0) { + fail({ line: i + 1, recordId: rec.id, type: rec.type, errors: errs }) + continue + } + const hash = sha256Hex(canonical) + const size = utf8ByteLength(canonical) + const body = size > OUT_OF_LINE_BYTES ? await repo.putOutOfLine(hash, canonical) : canonical + const isPrivate = isPrivateSchema(typeSchema) || rec.private === true + entries.push({ + t: rec.type, + k: rec.id, + h: hash, + s: size, + b: body, + ...(isPrivate ? { p: true } : {}), + }) + } + return total > 0 ? { errors, total } : { entries } +} + +export async function ingestRecords( + ports: Ports, + session: SessionRow, + text: string, +): Promise { + const inputs = await loadInputs(ports, session.id) + if (lines(text).length > MAX_BATCH_LINES) { + return { ok: false, status: 400, error: `At most ${MAX_BATCH_LINES} records per batch` } + } + const prepared = await prepareRecords(ports, session.collectionId, inputs, text, { + stripUnknownFields: session.stripUnknownFields, + }) + if ('errors' in prepared) { + return { + ok: false, + status: 422, + error: 'Invalid records', + details: prepared.errors, + total: prepared.total, + } + } + if (prepared.entries.length === 0) return { ok: false, status: 400, error: 'Empty batch' } + const seq = await nextRunSeq(ports, session.id) + if (seq === null) return { ok: false, status: 409, error: 'Session is not open' } + const index = await writeRun(ports.stores.internal, session.id, seq, prepared.entries) + await recordRun(ports, session.id, 'records', index, { records: prepared.entries.length }) + return { ok: true, received: prepared.entries.length } +} + +export async function ingestDeletes( + ports: Ports, + session: SessionRow, + text: string, +): Promise { + const inputs = await loadInputs(ports, session.id) + const entries: RunEntry[] = [] + const errors: LineError[] = [] + const all = lines(text) + if (all.length > MAX_BATCH_LINES) + return { ok: false, status: 400, error: `At most ${MAX_BATCH_LINES} deletes per batch` } + all.forEach((l, i) => { + let v: { type?: unknown; id?: unknown } + try { + v = JSON.parse(l) as typeof v + } catch { + errors.push({ line: i + 1, errors: ['Invalid JSON'] }) + return + } + if (typeof v.type !== 'string' || typeof v.id !== 'string') { + errors.push({ line: i + 1, errors: ['Each line is {"type": …, "id": …}'] }) + } else if (!inputs.schemas[v.type]) { + errors.push({ line: i + 1, errors: [`No schema for type "${v.type}"`] }) + } else { + entries.push({ t: v.type, k: v.id, x: true }) + } + }) + if (errors.length > 0) + return { + ok: false, + status: 422, + error: 'Invalid deletes', + details: errors.slice(0, MAX_REPORTED), + total: errors.length, + } + if (entries.length === 0) return { ok: false, status: 400, error: 'Empty batch' } + const seq = await nextRunSeq(ports, session.id) + if (seq === null) return { ok: false, status: 409, error: 'Session is not open' } + const index = await writeRun(ports.stores.internal, session.id, seq, entries) + await recordRun(ports, session.id, 'deletes', index, {}) + return { ok: true, received: entries.length } +} + +export const toRecordEntry = (e: RunEntry): RecordEntry => ({ + key: e.k, + hash: e.h!, + size: e.s!, + body: e.b!, +}) + +/** The base a session commits on, from the collection head. */ +export async function headBase(ports: Ports, collectionId: string): Promise { + const [row] = await ports.db + .select({ v: schema.versions }) + .from(schema.collections) + .innerJoin(schema.versions, eq(schema.versions.id, schema.collections.headVersionId)) + .where(eq(schema.collections.id, collectionId)) + .limit(1) + if (!row) return null + const v = row.v + return { + id: v.id, + seq: v.seq, + semver: v.semver, + hash: v.hash, + publicRefsRoot: v.publicRefsRoot, + privateRefsRoot: v.privateRefsRoot, + } +} + +/** + * Change streams for a delta session: per type, the merged runs split into the + * public and private sets. An upsert goes to its set and becomes a delete in the + * other set (when that set has a tree for the type); a delete goes to both. + */ +async function typeInputsForDelta( + ports: Ports, + session: SessionRow, + inputs: SessionInputs, + runs: RunIndex[], + base: { + pub: Record + priv: Record + }, +): Promise { + const internal = ports.stores.internal + const hashes = schemaHashes(inputs.schemas) + return Object.entries(inputs.schemas).map(([slug, s]) => { + const privateType = isPrivateSchema(s) + const hasPub = !!base.pub[slug]?.root + const hasPriv = !!base.priv[slug]?.root + const stream = async function* ( + set: 'public' | 'private', + ): AsyncGenerator> { + for await (const e of mergeRuns(internal, session.id, runs, slug)) { + if (e.x) { + if (set === 'public' ? hasPub : hasPriv) yield { key: e.k, entry: null } + continue + } + const target = privateType || e.p ? 'private' : 'public' + if (target === set) yield { key: e.k, entry: toRecordEntry(e) } + else if (set === 'public' ? hasPub : hasPriv) yield { key: e.k, entry: null } + } + } + return { + slug, + schema: s, + schemaHash: hashes[slug]!, + public: runs.length === 0 || privateType ? null : stream('public'), + private: runs.length === 0 ? null : stream('private'), + } + }) +} + +/** Commit a delta session. Idempotent on the session status: only an open session commits. */ +export async function commitDeltaSession( + ports: Ports, + session: SessionRow, +): Promise { + const base = await headBase(ports, session.collectionId) + if ((base?.id ?? null) !== session.baseVersionId) { + return { status: 'base_moved', current: base?.semver ?? null } + } + const inputs = await loadInputs(ports, session.id) + let runs = await sessionRuns(ports, session.id) + if (runs.length > MERGE_FAN_IN) { + runs = await compactRuns(ports.stores.internal, session.id, runs, session.runs + 1) + } + const repo = await ports.stores.forCollection(session.collectionId) + const root = base ? await repo.root(base.hash) : null + const priv = root?.private ? await repo.privateSet(root.private) : null + const declared = 'all' in inputs.files ? null : inputs.files + return commitVersion(ports, { + collectionId: session.collectionId, + base, + types: await typeInputsForDelta(ports, session, inputs, runs, { + pub: root?.public.types ?? {}, + priv: priv?.types ?? {}, + }), + metadata: inputs.metadata, + ...(declared ? { declaredFiles: declared } : {}), + message: session.message, + pushedBy: session.userId, + appId: session.appId, + actorId: session.actorId, + validate: (s, data) => { + const errs = compileSchema(s)(data) + return errs.length > 0 ? errs : null + }, + }) +} + +export { transition } diff --git a/packages/server/src/push/finalize.ts b/packages/server/src/push/finalize.ts new file mode 100644 index 0000000..d813940 --- /dev/null +++ b/packages/server/src/push/finalize.ts @@ -0,0 +1,110 @@ +/** + * Finishing a push session: run the commit and record the outcome on the + * session, whether it runs inside the request or as a job. The session moves + * open → committing (by the caller) → committed | failed; `result` holds what a + * synchronous commit returns, `error` the rejection. + */ +import { and, eq, lt } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Ports } from '../ports.js' +import { MissingFilesError } from '../versions/file-refs.js' +import { commitDeltaSession } from './delta.js' +import { commitNegotiateSession } from './negotiate.js' +import { getSession, transition } from './session.js' + +export interface Outcome { + status: 201 | 409 | 422 | 400 + body: Record +} + +export async function finalizeSession(ports: Ports, sessionId: string): Promise { + const session = await getSession(ports, sessionId) + if (!session) return { status: 400, body: { error: 'Session not found', statusCode: 400 } } + if (session.status === 'committed') return { status: 201, body: session.result ?? {} } + if (session.status === 'failed') + return { + status: (session.error?.statusCode ?? 400) as Outcome['status'], + body: session.error ?? {}, + } + + let outcome: Outcome + try { + const r = await (session.kind === 'delta' + ? commitDeltaSession(ports, session) + : commitNegotiateSession(ports, session)) + switch (r.status) { + case 'committed': + outcome = { + status: 201, + body: { + semver: r.version.semver, + hash: r.version.hash, + recordCount: r.version.recordCount, + fileCount: r.version.fileCount, + changes: r.version.changes, + }, + } + break + case 'no_changes': + outcome = { + status: 409, + body: { + error: 'No changes detected', + message: 'The head version already has identical content.', + hash: r.versionHash, + statusCode: 409, + }, + } + break + case 'conflict': + case 'base_moved': + outcome = { status: 409, body: { error: 'Version conflict', statusCode: 409 } } + break + case 'manifest_error': + outcome = { status: 400, body: r.body } + break + case 'invalid': + outcome = { + status: 422, + body: { + error: 'Schema validation failed', + validationErrors: r.errors, + totalErrors: r.total, + statusCode: 422, + }, + } + break + } + } catch (err) { + if (!(err instanceof MissingFilesError)) throw err + outcome = { + status: 422, + body: { + error: 'Missing files', + filesNeeded: err.hashes.slice(0, 100).map((h) => `sha256:${h}`), + statusCode: 422, + }, + } + } + + const ok = outcome.status === 201 + await transition( + ports, + sessionId, + 'committing', + ok ? 'committed' : 'failed', + ok ? { result: outcome.body } : { error: outcome.body as never }, + ) + return outcome +} + +/** Mark sessions whose idle timeout passed as expired (their objects expire by lifecycle rule). */ +export async function expireSessions(ports: Ports): Promise { + await ports.db + .update(schema.pushSessions) + .set({ status: 'expired' }) + .where( + and(eq(schema.pushSessions.status, 'open'), lt(schema.pushSessions.expiresAt, new Date())), + ) +} diff --git a/packages/server/src/push/negotiate.ts b/packages/server/src/push/negotiate.ts new file mode 100644 index 0000000..dde2929 --- /dev/null +++ b/packages/server/src/push/negotiate.ts @@ -0,0 +1,363 @@ +/** + * Negotiate: the v1 push API, reimplemented on v2 storage so PubPub and the + * current CLI keep working (edge-redesign.md, Push APIs 1). + * + * The manifest is a full snapshot of (type, id, hash, private). A manifest + * entry is "present" only when the collection's base version has the same hash + * for that (type, id), in either set. Presence never comes from a global + * lookup, which closes the hash oracle structurally: a record or file the base + * doesn't have must be uploaded. + * + * Commit diffs the base trees against the manifest in one sorted pass per type: + * O(collection size), inherent to snapshots. Large pushers should use delta push. + * + * Format 1 hashes: a v1 client hashes data with integer-like keys differently + * (edge-redesign-build.md finding 6). Uploads compute both hashes, so a manifest + * entry matches an upload by either. + */ +import { + type Change, + compareUtf8, + compileSchema, + getEntry, + hasArrayIndexKey, + iterate, + legacyRecordHash, + type RecordEntry, + recordTree, +} from '@underlay/core' +import { type Repo, RepoSource } from '@underlay/repo' + +import type { Ports } from '../ports.js' +import { type CommitResult, commitVersion, type TypeInput } from '../versions/commit.js' +import { declaredFiles } from '../versions/file-refs.js' +import { headBase, toRecordEntry } from './delta.js' +import { compactRuns, mergeRuns, MERGE_FAN_IN, type RunEntry, type RunIndex } from './runs.js' +import { + loadInputs, + schemaHashes, + type SessionInputs, + sessionRuns, + type SessionRow, +} from './session.js' + +const HEX64 = /^[0-9a-f]{64}$/ +const isPrivateSchema = (s: Record) => s.private === true + +export interface ManifestLine { + id: string + type: string + hash: string + private?: boolean +} + +export function parseManifestLine(v: unknown): ManifestLine | string { + const m = v as Partial + if (!m || typeof m !== 'object') return 'not an object' + if (typeof m.id !== 'string' || m.id.length === 0) return 'id must be a non-empty string' + if (typeof m.type !== 'string') return 'type must be a string' + if (typeof m.hash !== 'string' || !HEX64.test(m.hash)) + return 'hash must be a lowercase hex sha256' + if (m.private !== undefined && typeof m.private !== 'boolean') return 'private must be a boolean' + return m as ManifestLine +} + +export interface BaseTrees { + repo: Repo + pub: Record + priv: Record +} + +export async function baseTrees( + ports: Ports, + collectionId: string, + baseHash: string | null, +): Promise { + const repo = await ports.stores.forCollection(collectionId) + if (!baseHash) return { repo, pub: {}, priv: {} } + const root = await repo.root(baseHash) + const priv = root.private ? await repo.privateSet(root.private) : null + return { repo, pub: root.public.types, priv: priv?.types ?? {} } +} + +/** + * Which manifest entries the server needs uploaded: those the base doesn't hold + * with the same hash. Entries are looked up in (type, id) order so consecutive + * keys share cached leaves. + */ +export async function neededOf(trees: BaseTrees, entries: ManifestLine[]): Promise { + const source = new RepoSource(recordTree, trees.repo) + const sorted = entries + .slice() + .sort((a, b) => compareUtf8(a.type, b.type) || compareUtf8(a.id, b.id)) + const needed: string[] = [] + for (const m of sorted) { + let present = false + for (const set of [trees.pub, trees.priv]) { + const root = set[m.type]?.root ?? null + if (!root) continue + if ((await getEntry(source, root, m.id))?.hash === m.hash) present = true + } + if (!present) needed.push(m.hash) + } + return needed +} + +/** Run entries for uploaded records: like delta uploads, plus the format 1 hash when it differs. */ +export function withLegacyHash(e: RunEntry, data: unknown): RunEntry { + if (!hasArrayIndexKey(data)) return e + const lh = legacyRecordHash(e.k, e.t, data) + return lh === e.h ? e : { ...e, lh } +} + +type Tagged = { key: string; set: 'public' | 'private'; hash: string } + +async function* baseEntries( + source: RepoSource, + pubRoot: string | null, + privRoot: string | null, +): AsyncGenerator { + const pub = iterate(source, pubRoot)[Symbol.asyncIterator]() + const priv = iterate(source, privRoot)[Symbol.asyncIterator]() + let a = await pub.next() + let b = await priv.next() + while (!a.done || !b.done) { + if (b.done || (!a.done && compareUtf8(a.value.key, b.value.key) <= 0)) { + yield { key: a.value!.key, set: 'public', hash: a.value!.hash } + a = await pub.next() + } else { + yield { key: b.value.key, set: 'private', hash: b.value.hash } + b = await priv.next() + } + } +} + +/** One step of the snapshot diff for a key. */ +interface Step { + key: string + manifest: RunEntry | null + base: Tagged | null + upload: RunEntry | null +} + +/** Zip the manifest, the base and the uploads of one type by id. */ +async function* zip( + manifest: AsyncIterable, + base: AsyncIterable, + uploads: AsyncIterable, +): AsyncGenerator { + const mi = manifest[Symbol.asyncIterator]() + const bi = base[Symbol.asyncIterator]() + const ui = uploads[Symbol.asyncIterator]() + let m = await mi.next() + let b = await bi.next() + let u = await ui.next() + for (;;) { + const keys = [ + m.done ? null : m.value.k, + b.done ? null : b.value.key, + u.done ? null : u.value.k, + ].filter((k): k is string => k !== null) + if (keys.length === 0) return + const key = keys.reduce((x, y) => (compareUtf8(x, y) <= 0 ? x : y)) + const step: Step = { key, manifest: null, base: null, upload: null } + if (!m.done && m.value.k === key) { + step.manifest = m.value + m = await mi.next() + } + if (!b.done && b.value.key === key) { + step.base = b.value + b = await bi.next() + } + if (!u.done && u.value.k === key) { + step.upload = u.value + u = await ui.next() + } + yield step + } +} + +/** A base record's body, for a record that moves between sets unchanged. */ +async function baseBody( + source: RepoSource, + root: string, + key: string, +): Promise { + let hash = root + for (;;) { + const node = await source.node(hash) + if (node.kind === 'leaf') { + const e = (await source.leafEntries(hash)).find((x) => x.key === key) + if (!e) throw new Error(`Base record ${key} vanished`) + return e + } + const child = node.children.find((c) => compareUtf8(key, c.lastKey) <= 0) + if (!child) throw new Error(`Base record ${key} vanished`) + hash = child.hash + } +} + +interface SnapshotPlan { + types: TypeInput[] + /** The first 100 missing hashes, and how many there are. */ + missing: string[] + missingCount: number + manifestCount: number +} + +/** Build per-type change streams from the snapshot diff, and count what's missing. */ +async function planSnapshot( + ports: Ports, + session: SessionRow, + inputs: SessionInputs, + trees: BaseTrees, + manifestRuns: RunIndex[], + recordRuns: RunIndex[], +): Promise { + const internal = ports.stores.internal + const source = new RepoSource(recordTree, trees.repo) + const hashes = schemaHashes(inputs.schemas) + const missing: string[] = [] + let manifestCount = 0 + let missingCount = 0 + + const stepsFor = (slug: string) => + zip( + mergeRuns(internal, session.id, manifestRuns, slug), + baseEntries(source, trees.pub[slug]?.root ?? null, trees.priv[slug]?.root ?? null), + mergeRuns(internal, session.id, recordRuns, slug), + ) + + const uploadFor = (st: Step) => + st.manifest && st.upload && (st.upload.h === st.manifest.h || st.upload.lh === st.manifest.h) + ? st.upload + : null + + // Pre-pass: everything in the manifest must be in the base or uploaded. + for (const slug of Object.keys(inputs.schemas)) { + for await (const st of stepsFor(slug)) { + if (!st.manifest) continue + manifestCount++ + if (!uploadFor(st) && st.base?.hash !== st.manifest.h) { + missingCount++ + if (missing.length < 100) missing.push(st.manifest.h!) + } + } + } + + const types: TypeInput[] = Object.entries(inputs.schemas).map(([slug, s]) => { + const privateType = isPrivateSchema(s) + const stream = async function* ( + set: 'public' | 'private', + ): AsyncGenerator> { + for await (const st of stepsFor(slug)) { + const inBase = st.base?.set === set + if (!st.manifest) { + if (inBase) yield { key: st.key, entry: null } // dropped from the snapshot + continue + } + const target = privateType || st.manifest.p ? 'private' : 'public' + if (target !== set) { + if (inBase) yield { key: st.key, entry: null } // left this set + continue + } + const upload = uploadFor(st) + const wanted = upload ? upload.h! : st.manifest.h! + if (inBase && st.base!.hash === wanted) continue // unchanged + if (upload) { + yield { key: st.key, entry: toRecordEntry(upload) } + } else { + // Not uploaded, so the base has it (the pre-pass checked) in the other + // set: the same record moving here. + const otherRoot = (st.base!.set === 'public' ? trees.pub : trees.priv)[slug]!.root! + yield { key: st.key, entry: await baseBody(source, otherRoot, st.key) } + } + } + } + return { + slug, + schema: s, + schemaHash: hashes[slug]!, + public: privateType ? null : stream('public'), + private: stream('private'), + } + }) + return { types, missing, missingCount, manifestCount } +} + +export async function commitNegotiateSession( + ports: Ports, + session: SessionRow, +): Promise< + | CommitResult + | { status: 'base_moved'; current: string | null } + | { status: 'manifest_error'; body: Record } +> { + const base = await headBase(ports, session.collectionId) + // v1 semantics: a null base_version means "commit on whatever the head is". + if (session.baseSemver !== null && (base?.id ?? null) !== session.baseVersionId) { + return { status: 'base_moved', current: base?.semver ?? null } + } + const inputs = await loadInputs(ports, session.id) + const compact = async (runs: RunIndex[]) => + runs.length > MERGE_FAN_IN + ? compactRuns(ports.stores.internal, session.id, runs, session.runs + 1000) + : runs + const manifestRuns = await compact(await sessionRuns(ports, session.id, 'manifest')) + const recordRuns = await compact(await sessionRuns(ports, session.id, 'records')) + const trees = await baseTrees(ports, session.collectionId, base?.hash ?? null) + + const plan = await planSnapshot(ports, session, inputs, trees, manifestRuns, recordRuns) + if (session.manifestExpected !== null && plan.manifestCount !== session.manifestExpected) { + return { + status: 'manifest_error', + body: { + error: 'Manifest incomplete', + message: `Expected ${session.manifestExpected} manifest entries but received ${plan.manifestCount}.`, + manifest_expected: session.manifestExpected, + manifest_received: plan.manifestCount, + statusCode: 400, + }, + } + } + if (plan.missing.length > 0) { + return { + status: 'manifest_error', + body: { + error: 'Missing records', + missing_hashes: plan.missing, + message: `${plan.missingCount} needed record(s) have not been submitted.`, + statusCode: 400, + }, + } + } + + // v1 merges the pushed metadata over the previous version's. + const prevMetadata = base ? (await trees.repo.root(base.hash)).metadata : null + const metadata = inputs.metadata ? { ...(prevMetadata ?? {}), ...inputs.metadata } : prevMetadata + + // Declared files are a full list in v1; turn it into adds and removes. + const all = 'all' in inputs.files ? inputs.files.all : [] + const before = await declaredFiles(trees.repo, base?.privateRefsRoot ?? null) + const now = new Set(all) + const declared = { + add: all.filter((h) => !before.has(h)), + remove: [...before].filter((h) => !now.has(h)), + } + + return commitVersion(ports, { + collectionId: session.collectionId, + base, + types: plan.types, + metadata, + declaredFiles: declared, + message: session.message, + pushedBy: session.userId, + appId: session.appId, + actorId: session.actorId, + validate: (s, data) => { + const errs = compileSchema(s)(data) + return errs.length > 0 ? errs : null + }, + }) +} diff --git a/packages/server/src/push/runs.ts b/packages/server/src/push/runs.ts new file mode 100644 index 0000000..30df360 --- /dev/null +++ b/packages/server/src/push/runs.ts @@ -0,0 +1,256 @@ +/** + * Sorted runs: how push sessions hold uploaded data until commit (the external + * sort of edge-redesign-build.md finding 11). + * + * Each upload batch becomes one run, sorted by (type, id). A run is a list of + * gzip blocks of about 1 MB raw each, in the platform's internal area: + * + * sessions//runs//.ndjson.gz + * sessions//runs//index.json [{first, last, count}, …] per block + * + * Blocks are fetched whole, so a reader holds one block per run, and the index + * lets a reader skip to the blocks covering one type. Merging is bounded by a + * fan-in; sessions with more runs are compacted first (compactRuns), merging + * groups into bigger runs written block by block, so memory never depends on + * push size. + */ +import { compareUtf8 } from '@underlay/core' +import { gunzipText, gzip, splitLines } from '@underlay/repo' + +import type { BlobStore } from '../ports.js' + +/** One entry of a run. `k` is the record id, `t` the type. */ +export interface RunEntry { + t: string + k: string + /** Record hash (records and manifest runs). */ + h?: string + /** Format 1 hash the client used, when it differs (negotiate compatibility). */ + lh?: string + /** Canonical record size in bytes. */ + s?: number + /** Pushed as private. */ + p?: boolean + /** The canonical record (records runs). */ + b?: string + /** A delete (delta deletes runs). */ + x?: boolean +} + +export interface RunIndex { + seq: number + blocks: { first: string; last: string; count: number }[] +} + +export const BLOCK_BYTES = 1024 * 1024 +export const MERGE_FAN_IN = 16 + +/** Sort key: type, then id, by UTF-8 bytes. A NUL separator keeps types apart. */ +export const runKey = (e: { t: string; k: string }) => `${e.t}\u0000${e.k}` +export const compareRunKeys = (a: RunEntry, b: RunEntry) => + compareUtf8(a.t, b.t) || compareUtf8(a.k, b.k) + +const runPrefix = (sessionId: string, seq: number) => `sessions/${sessionId}/runs/${seq}` + +/** Writes one run block by block. Entries must arrive sorted. */ +export class RunWriter { + readonly #blocks: RunIndex['blocks'] = [] + #lines: string[] = [] + #first: string | null = null + #last: string | null = null + #bytes = 0 + #pending: Promise[] = [] + + constructor( + readonly store: BlobStore, + readonly sessionId: string, + readonly seq: number, + ) {} + + add(e: RunEntry): void { + const key = runKey(e) + if (this.#last !== null && compareUtf8(this.#last, key) >= 0) { + throw new Error(`Run entries out of order at ${JSON.stringify(key)}`) + } + const line = JSON.stringify(e) + this.#first ??= key + this.#last = key + this.#lines.push(line) + this.#bytes += line.length + if (this.#bytes >= BLOCK_BYTES) this.#flushBlock() + } + + #flushBlock() { + if (this.#lines.length === 0) return + const n = this.#blocks.length + const text = this.#lines.join('\n') + '\n' + this.#blocks.push({ first: this.#first!, last: this.#last!, count: this.#lines.length }) + this.#lines = [] + this.#bytes = 0 + this.#first = null + const key = `${runPrefix(this.sessionId, this.seq)}/${n}.ndjson.gz` + this.#pending.push( + gzip(text).then((gz) => this.store.put(key, gz, { contentType: 'application/gzip' })), + ) + } + + /** Bounded memory: callers writing large runs await this between entries. */ + async drain(): Promise { + if (this.#pending.length >= 4) await Promise.all(this.#pending.splice(0)) + } + + async finish(): Promise { + this.#flushBlock() + await Promise.all(this.#pending.splice(0)) + const index: RunIndex = { seq: this.seq, blocks: this.#blocks } + await this.store.put( + `${runPrefix(this.sessionId, this.seq)}/index.json`, + JSON.stringify(index), + { + contentType: 'application/json', + }, + ) + return index + } +} + +/** Write an in-memory batch (one upload request) as a run. */ +export async function writeRun( + store: BlobStore, + sessionId: string, + seq: number, + entries: RunEntry[], +) { + entries.sort(compareRunKeys) + // Within one batch the last occurrence of a key wins. + const w = new RunWriter(store, sessionId, seq) + for (let i = 0; i < entries.length; i++) { + const next = entries[i + 1] + if (next && compareRunKeys(entries[i]!, next) === 0) continue + w.add(entries[i]!) + } + return w.finish() +} + +export async function readRunIndex( + store: BlobStore, + sessionId: string, + seq: number, +): Promise { + const obj = await store.get(`${runPrefix(sessionId, seq)}/index.json`) + if (!obj) throw new Error(`Run ${seq} of session ${sessionId} is missing`) + return JSON.parse(await obj.text()) as RunIndex +} + +/** + * Entries of one run in order, optionally limited to one type. Reads one block + * at a time and fetches the next while the current one is consumed. + */ +export async function* readRun( + store: BlobStore, + sessionId: string, + index: RunIndex, + type?: string, +): AsyncGenerator { + const blocks = index.blocks + .map((b, i) => ({ ...b, i })) + .filter( + (b) => + type === undefined || + (compareUtf8(b.first.split('\u0000')[0]!, type) <= 0 && + compareUtf8(b.last.split('\u0000')[0]!, type) >= 0), + ) + const load = async (i: number) => { + const obj = await store.get(`${runPrefix(sessionId, index.seq)}/${i}.ndjson.gz`) + if (!obj) throw new Error(`Run block ${index.seq}/${i} of session ${sessionId} is missing`) + return splitLines(await gunzipText(await obj.bytes())) + } + let next = blocks[0] ? load(blocks[0].i) : null + for (let j = 0; j < blocks.length; j++) { + const lines = await next! + next = blocks[j + 1] ? load(blocks[j + 1]!.i) : null + next?.catch(() => {}) + for (const line of lines) { + const e = JSON.parse(line) as RunEntry + if (type === undefined || e.t === type) yield e + } + } +} + +/** + * K-way merge of runs (at most MERGE_FAN_IN) by (type, id). On equal keys the + * run with the higher seq wins: later uploads replace earlier ones. + */ +export async function* mergeRuns( + store: BlobStore, + sessionId: string, + indexes: RunIndex[], + type?: string, +): AsyncGenerator { + if (indexes.length > MERGE_FAN_IN) + throw new Error(`mergeRuns: ${indexes.length} runs; compact first`) + const cursors = await Promise.all( + indexes.map(async (ix) => { + const it = readRun(store, sessionId, ix, type) + return { seq: ix.seq, it, head: await it.next() } + }), + ) + for (;;) { + let best: (typeof cursors)[number] | null = null + for (const c of cursors) { + if (c.head.done) continue + if (!best) { + best = c + continue + } + const cmp = compareRunKeys(c.head.value, best.head.value as RunEntry) + if (cmp < 0 || (cmp === 0 && c.seq > best.seq)) best = c + } + if (!best) return + const winner = best.head.value as RunEntry + // Advance every cursor sitting on the same key; the winner's entry is emitted. + for (const c of cursors) { + while (!c.head.done && compareRunKeys(c.head.value, winner) === 0) c.head = await c.it.next() + } + yield winner + } +} + +/** + * Merge groups of runs until at most MERGE_FAN_IN remain. Returns the new run + * list. Each merged run takes the highest seq of its group, so "later wins" is + * preserved; new seqs start at `nextSeq`. + */ +export async function compactRuns( + store: BlobStore, + sessionId: string, + indexes: RunIndex[], + nextSeq: number, +): Promise { + let runs = indexes.slice().sort((a, b) => a.seq - b.seq) + let seq = nextSeq + while (runs.length > MERGE_FAN_IN) { + // Group from the newest end, so the only group that can be a lone, + // unmerged run is the oldest one: it keeps its low seq and still loses to + // every merged run, which all get fresh seqs in age order. + const groups: RunIndex[][] = [] + for (let end = runs.length; end > 0; end -= MERGE_FAN_IN) { + groups.unshift(runs.slice(Math.max(0, end - MERGE_FAN_IN), end)) + } + const out: RunIndex[] = [] + for (const group of groups) { + if (group.length === 1) { + out.push(group[0]!) + continue + } + const w = new RunWriter(store, sessionId, seq++) + for await (const e of mergeRuns(store, sessionId, group)) { + w.add(e) + await w.drain() + } + out.push(await w.finish()) + } + runs = out + } + return runs +} diff --git a/packages/server/src/push/session.ts b/packages/server/src/push/session.ts new file mode 100644 index 0000000..87c550a --- /dev/null +++ b/packages/server/src/push/session.ts @@ -0,0 +1,139 @@ +/** + * Push sessions. A session collects uploads against a base version and ends in + * one commit. Its inputs whose size the user controls (schemas, metadata, + * declared files) are an object in the platform's internal area; uploads are + * sorted runs (runs.ts). SQLite holds only the session row, counters and the run + * list. + */ +import { hashSchema } from '@underlay/core' +import { and, asc, eq, sql } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Ports } from '../ports.js' +import { readRunIndex, type RunIndex } from './runs.js' + +/** Idle timeout: pushed back by every upload. Runs live in storage, so this can be generous. */ +export const SESSION_TTL_MS = 60 * 60 * 1000 + +export interface SessionInputs { + /** The full new type set: slug → schema. */ + schemas: Record> + /** The full new metadata. */ + metadata: Record | null + /** Declared files to add and remove (delta), or the full declared list (negotiate). */ + files: { add: string[]; remove: string[] } | { all: string[] } +} + +export type SessionRow = typeof schema.pushSessions.$inferSelect + +const inputsKey = (id: string) => `sessions/${id}/inputs.json` + +export async function createSession( + ports: Ports, + row: Omit, + inputs: SessionInputs, +): Promise { + const id = crypto.randomUUID() + // Inputs first: a session row never points at missing inputs. + await ports.stores.internal.put(inputsKey(id), JSON.stringify(inputs), { + contentType: 'application/json', + }) + const [session] = await ports.db + .insert(schema.pushSessions) + .values({ ...row, id, expiresAt: new Date(Date.now() + SESSION_TTL_MS) }) + .returning() + return session! +} + +export async function loadInputs(ports: Ports, sessionId: string): Promise { + const obj = await ports.stores.internal.get(inputsKey(sessionId)) + if (!obj) throw new Error(`Session ${sessionId} has no inputs`) + return JSON.parse(await obj.text()) as SessionInputs +} + +export async function getSession(ports: Ports, sessionId: string): Promise { + const [s] = await ports.db + .select() + .from(schema.pushSessions) + .where(eq(schema.pushSessions.id, sessionId)) + .limit(1) + return s ?? null +} + +/** Claim the next run number for an open session and push back its expiry, atomically. */ +export async function nextRunSeq(ports: Ports, sessionId: string): Promise { + const [row] = await ports.db + .update(schema.pushSessions) + .set({ + runs: sql`${schema.pushSessions.runs} + 1`, + expiresAt: new Date(Date.now() + SESSION_TTL_MS), + }) + .where(and(eq(schema.pushSessions.id, sessionId), eq(schema.pushSessions.status, 'open'))) + .returning({ runs: schema.pushSessions.runs }) + return row?.runs ?? null +} + +export async function recordRun( + ports: Ports, + sessionId: string, + kind: 'manifest' | 'records' | 'deletes', + index: RunIndex, + counters: { records?: number; manifest?: number; needed?: number }, +): Promise { + const count = index.blocks.reduce((n, b) => n + b.count, 0) + await ports.db.batch([ + ports.db.insert(schema.pushRuns).values({ + sessionId, + seq: index.seq, + kind, + objectKey: `sessions/${sessionId}/runs/${index.seq}`, + count, + firstKey: index.blocks[0]?.first ?? '', + lastKey: index.blocks[index.blocks.length - 1]?.last ?? '', + }), + ports.db + .update(schema.pushSessions) + .set({ + recordsReceived: sql`${schema.pushSessions.recordsReceived} + ${counters.records ?? 0}`, + manifestReceived: sql`${schema.pushSessions.manifestReceived} + ${counters.manifest ?? 0}`, + manifestNeeded: sql`${schema.pushSessions.manifestNeeded} + ${counters.needed ?? 0}`, + }) + .where(eq(schema.pushSessions.id, sessionId)), + ]) +} + +export async function sessionRuns( + ports: Ports, + sessionId: string, + kind?: 'manifest' | 'records' | 'deletes', +): Promise { + const rows = await ports.db + .select() + .from(schema.pushRuns) + .where( + kind + ? and(eq(schema.pushRuns.sessionId, sessionId), eq(schema.pushRuns.kind, kind)) + : eq(schema.pushRuns.sessionId, sessionId), + ) + .orderBy(asc(schema.pushRuns.seq)) + return Promise.all(rows.map((r) => readRunIndex(ports.stores.internal, sessionId, r.seq))) +} + +/** Move a session between statuses only from an expected one (compare-and-swap). */ +export async function transition( + ports: Ports, + sessionId: string, + from: schema.SessionStatus, + to: schema.SessionStatus, + extra: Partial = {}, +): Promise { + const rows = await ports.db + .update(schema.pushSessions) + .set({ status: to, ...extra }) + .where(and(eq(schema.pushSessions.id, sessionId), eq(schema.pushSessions.status, from))) + .returning({ id: schema.pushSessions.id }) + return rows.length === 1 +} + +export const schemaHashes = (schemas: Record) => + Object.fromEntries(Object.entries(schemas).map(([slug, s]) => [slug, hashSchema(s)])) diff --git a/packages/server/src/versions/commit.ts b/packages/server/src/versions/commit.ts index 1bca45c..fca41db 100644 --- a/packages/server/src/versions/commit.ts +++ b/packages/server/src/versions/commit.ts @@ -202,6 +202,16 @@ export async function commitVersion(ports: Ports, input: CommitInput): Promise t.slug)) let schemaChanged = false + // Schemas are repository objects too (schemas/.json). + await Promise.all( + input.types.map(async (t) => { + if ((await repo.putSchema(t.schema)) !== t.schemaHash) { + throw new Error(`Schema for ${t.slug} does not match its hash`) + } + }), + ) + sink.written.push(...input.types.map((t) => `schemas/${t.schemaHash}.json`)) + for (const t of input.types) { const pubBase = basePublic.types[t.slug] const privBase = basePrivate.types[t.slug] diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index 1670946..c4d2b89 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -65,6 +65,8 @@ const app = createApp((c) => { return { ports: makePorts(env, c.executionCtx as unknown as ExecutionContext), config: { appUrl: env.APP_URL, deployment: env.DEPLOYMENT }, + // Sign-in and API keys (better-auth + KF Auth) land next. + authenticate: async () => null, } }) diff --git a/packages/server/test/harness.ts b/packages/server/test/harness.ts index 8d78d1f..ac306c3 100644 --- a/packages/server/test/harness.ts +++ b/packages/server/test/harness.ts @@ -8,6 +8,7 @@ import { ed25519Signer, generateSigningKey, type Signer } from '@underlay/repo' import { MemoryBlobStore } from '@underlay/repo/blob/memory' import '../src/handlers.js' +import { createApp } from '../src/app.js' import { MemoryCache } from '../src/cache.js' import { openNodeDb } from '../src/db/node.js' import * as schema from '../src/db/schema.js' @@ -23,6 +24,15 @@ export async function cleanup(): Promise { export interface Harness { ports: Ports + /** The app, authenticating `x-test-user: ` as a signed-in user. */ + app: ReturnType + /** A user who is a member of the org that owns test collections. */ + member(id?: string): Promise + /** fetch against the app as a user (or anonymously). */ + request( + path: string, + init?: RequestInit & { user?: string; json?: unknown; ndjson?: unknown[] }, + ): Promise bucket: MemoryBlobStore signer: Signer /** Run queued jobs until none are ready. */ @@ -47,16 +57,57 @@ export async function harness(): Promise { waitUntil: (p) => void p.catch((err) => console.error(err)), } let orgMade = false + const ensureOrg = async () => { + if (orgMade) return + await db.insert(schema.organization).values({ id: 'org1', name: 'Org', slug: 'org' }) + orgMade = true + } + const app = createApp(() => ({ + ports, + config: { appUrl: 'http://test', deployment: 'test' }, + authenticate: async (req) => { + const user = req.headers.get('x-test-user') + return user ? { userId: user, scope: 'session', collectionIds: null } : null + }, + })) return { ports, + app, bucket, signer, drain: () => drainSqliteJobs(ports), - async collection(slug = 'c') { - if (!orgMade) { - await db.insert(schema.organization).values({ id: 'org1', name: 'Org', slug: 'org' }) - orgMade = true + async member(id = 'u1') { + await ensureOrg() + await db + .insert(schema.user) + .values({ id, name: id, email: `${id}@example.org` }) + .onConflictDoNothing() + await db.insert(schema.member).values({ organizationId: 'org1', userId: id, role: 'owner' }) + return id + }, + async request(path, init = {}) { + const { user, json, ndjson, ...rest } = init + const headers = new Headers(rest.headers) + if (user) headers.set('x-test-user', user) + let body = rest.body + if (json !== undefined) { + body = JSON.stringify(json) + headers.set('content-type', 'application/json') + } + if (ndjson !== undefined) { + body = ndjson.map((l) => JSON.stringify(l)).join('\n') + headers.set('content-type', 'application/x-ndjson') } + return app.fetch( + new Request(`http://test${path}`, { + ...rest, + headers, + ...(body !== undefined ? { body } : {}), + }), + ) + }, + async collection(slug = 'c') { + await ensureOrg() const [c] = await db .insert(schema.collections) .values({ organizationId: 'org1', slug, name: slug, privateSalt: newSalt() }) diff --git a/packages/server/test/push.test.ts b/packages/server/test/push.test.ts new file mode 100644 index 0000000..2ab8b6f --- /dev/null +++ b/packages/server/test/push.test.ts @@ -0,0 +1,324 @@ +import { getEntry, hashRecord, legacyRecordHash, recordTree } from '@underlay/core' +import { RepoSource } from '@underlay/repo' +import { eq } from 'drizzle-orm' +import { afterAll, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) + +const Author = { + type: 'object', + properties: { name: { type: 'string' }, born: { type: 'integer' } }, + required: ['name'], +} + +const rec = (id: string, data: Record, extra: Record = {}) => ({ + id, + type: 'Author', + data, + ...extra, +}) +const hashOf = (id: string, data: unknown) => hashRecord(id, 'Author', data).hash + +async function setup() { + const h = await harness() + const user = await h.member() + const c = await h.collection('authors') + return { h, user, c, base: '/api/collections/org/authors' } +} + +async function json(res: Response) { + return (await res.json()) as Record +} + +async function head(h: Harness, collectionId: string) { + const [c] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, collectionId)) + if (!c?.headVersionId) return null + const [v] = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, c.headVersionId)) + return v! +} + +describe('delta push', () => { + it('opens, uploads, deletes and commits', async () => { + const { h, user, c, base } = await setup() + let res = await h.request(`${base}/push`, { + method: 'POST', + user, + json: { schemas: { Author }, metadata: { title: 'Authors' } }, + }) + expect(res.status).toBe(200) + let sid = (await json(res)).session_id + res = await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [ + rec('ada', { name: 'Ada', born: 1815 }), + rec('alan', { name: 'Alan' }), + rec('kurt', { name: 'Kurt' }, { private: true }), + ], + }) + expect(await json(res)).toEqual({ received: 3 }) + res = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) + expect(res.status).toBe(201) + const v1 = await json(res) + expect(v1).toMatchObject({ semver: 'v1.0.0', recordCount: 3 }) + + // Second push: base required to match; update, delete, flip one to public. + res = await h.request(`${base}/push`, { method: 'POST', user, json: { base: 'v0.9.0' } }) + expect(res.status).toBe(409) + res = await h.request(`${base}/push`, { method: 'POST', user, json: { base: 'v1.0.0' } }) + sid = (await json(res)).session_id + await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [rec('ada', { name: 'Ada Lovelace', born: 1815 }), rec('kurt', { name: 'Kurt' })], + }) + await h.request(`${base}/push/${sid}/deletes`, { + method: 'POST', + user, + ndjson: [{ type: 'Author', id: 'alan' }], + }) + res = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) + expect(res.status).toBe(201) + const v2 = await head(h, c.id) + expect(v2).toMatchObject({ + semver: 'v1.1.0', + recordCount: 2, + publicRecordCount: 2, + hasPrivate: false, + }) + const repo = await h.ports.stores.forCollection(c.id) + const root = await repo.root(v2!.hash) + // Metadata was kept from the base. + expect(root.metadata).toEqual({ title: 'Authors' }) + const kurt = await getEntry( + new RepoSource(recordTree, repo), + root.public.types.Author!.root, + 'kurt', + ) + expect(kurt?.hash).toBe(hashOf('kurt', { name: 'Kurt' })) + }) + + it('reports invalid records by line, and the input rules', async () => { + const { h, user, base } = await setup() + const sid = ( + await json( + await h.request(`${base}/push`, { method: 'POST', user, json: { schemas: { Author } } }), + ) + ).session_id + const res = await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + body: [ + JSON.stringify(rec('ok', { name: 'Fine' })), + JSON.stringify(rec('bad', { born: 'x' })), + '{"id":"dup","type":"Author","data":{"name":"a","name":"b"}}', + '{"id":"big","type":"Author","data":{"name":"n","born":12345678901234567890}}', + JSON.stringify(rec('extra', { name: 'E', nickname: 'e' })), + ].join('\n'), + }) + expect(res.status).toBe(422) + const body = await json(res) + expect(body.totalErrors).toBe(4) + expect(body.validationErrors.map((e: { line: number }) => e.line)).toEqual([2, 3, 4, 5]) + expect(body.validationErrors[1].errors[0]).toMatch(/duplicate_key/) + expect(body.validationErrors[2].errors[0]).toMatch(/unsafe_integer/) + }) + + it('commits asynchronously as a job', async () => { + const { h, user, base } = await setup() + const sid = ( + await json( + await h.request(`${base}/push`, { method: 'POST', user, json: { schemas: { Author } } }), + ) + ).session_id + await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [rec('a', { name: 'A' })], + }) + const res = await h.request(`${base}/push/${sid}/commit?async=true`, { method: 'POST', user }) + expect(res.status).toBe(202) + expect((await json(await h.request(`${base}/push/${sid}`, { user }))).status).toBe('committing') + await h.drain() + const status = await json(await h.request(`${base}/push/${sid}`, { user })) + expect(status).toMatchObject({ status: 'committed', result: { semver: 'v1.0.0' } }) + }) + + it('keeps writes to members, and private collections hidden', async () => { + const { h, base } = await setup() + expect((await h.request(`${base}/push`, { method: 'POST', json: {} })).status).toBe(404) + await h.ports.db.update(schema.collections).set({ public: true }) + expect((await h.request(`${base}/push`, { method: 'POST', json: {} })).status).toBe(401) + expect( + (await h.request(`${base}/push`, { method: 'POST', user: 'stranger', json: {} })).status, + ).toBe(403) + }) +}) + +describe('negotiate (v1 compatibility)', () => { + const manifestOf = (records: ReturnType[]) => + records.map((r) => ({ + id: r.id, + type: r.type, + hash: hashOf(r.id, r.data), + ...(r.private ? { private: true } : {}), + })) + + async function push( + h: Harness, + user: string, + base: string, + records: ReturnType[], + baseVersion: string | null, + metadata?: object, + ) { + let res = await h.request(`${base}/versions/negotiate`, { + method: 'POST', + user, + json: { + base_version: baseVersion, + schemas: { Author }, + manifest: manifestOf(records), + ...(metadata ? { metadata } : {}), + }, + }) + expect(res.status).toBe(200) + const n = await json(res) + const needed = new Set(n.needed_records as string[]) + const upload = records.filter((r) => needed.has(hashOf(r.id, r.data))) + if (upload.length > 0) { + res = await h.request(`${base}/versions/negotiate/${n.session_id}/records`, { + method: 'POST', + user, + ndjson: upload, + }) + expect(res.status).toBe(200) + } + res = await h.request(`${base}/versions/negotiate/${n.session_id}/commit`, { + method: 'POST', + user, + }) + return { negotiate: n, status: res.status, body: await json(res), uploaded: upload.length } + } + + it('pushes snapshots, uploading only what the base lacks', async () => { + const { h, user, c, base } = await setup() + const v1 = [rec('a', { name: 'A' }), rec('b', { name: 'B' }), rec('c', { name: 'C' })] + const p1 = await push(h, user, base, v1, null, { title: 'T' }) + expect(p1).toMatchObject({ + status: 201, + uploaded: 3, + body: { semver: 'v1.0.0', recordCount: 3 }, + }) + + // Change b, drop c, add d, make a private: only b and d are uploaded. + const v2 = [ + rec('a', { name: 'A' }, { private: true }), + rec('b', { name: 'B2' }), + rec('d', { name: 'D' }), + ] + const p2 = await push(h, user, base, v2, 'v1.0.0') + expect(p2).toMatchObject({ + status: 201, + uploaded: 2, + body: { semver: 'v1.1.0', recordCount: 3 }, + }) + const v = await head(h, c.id) + expect(v).toMatchObject({ publicRecordCount: 2, hasPrivate: true }) + // v1 merges metadata over the previous version's. + const repo = await h.ports.stores.forCollection(c.id) + expect((await repo.root(v!.hash)).metadata).toEqual({ title: 'T' }) + + // The same snapshot again: no changes. + const p3 = await push(h, user, base, v2, 'v1.1.0') + expect(p3).toMatchObject({ status: 409, uploaded: 0, body: { error: 'No changes detected' } }) + }) + + it('refuses a commit with records still missing, or a short chunked manifest', async () => { + const { h, user, base } = await setup() + let res = await h.request(`${base}/versions/negotiate`, { + method: 'POST', + user, + json: { schemas: { Author }, manifest: manifestOf([rec('a', { name: 'A' })]) }, + }) + let sid = (await json(res)).session_id + res = await h.request(`${base}/versions/negotiate/${sid}/commit`, { method: 'POST', user }) + expect(res.status).toBe(400) + expect(await json(res)).toMatchObject({ + error: 'Missing records', + missing_hashes: [hashOf('a', { name: 'A' })], + }) + + res = await h.request(`${base}/versions/negotiate`, { + method: 'POST', + user, + json: { schemas: { Author }, manifest_expected: 2 }, + }) + sid = (await json(res)).session_id + res = await h.request(`${base}/versions/negotiate/${sid}/manifest`, { + method: 'POST', + user, + ndjson: manifestOf([rec('a', { name: 'A' })]), + }) + expect(await json(res)).toMatchObject({ + received: 1, + needed_records: [hashOf('a', { name: 'A' })], + }) + await h.request(`${base}/versions/negotiate/${sid}/records`, { + method: 'POST', + user, + ndjson: [rec('a', { name: 'A' })], + }) + res = await h.request(`${base}/versions/negotiate/${sid}/commit`, { method: 'POST', user }) + expect(await json(res)).toMatchObject({ + error: 'Manifest incomplete', + manifest_expected: 2, + manifest_received: 1, + }) + }) + + it('accepts format 1 hashes for records with integer-like keys', async () => { + const { h, user, c, base } = await setup() + const Scores = { type: 'object' } + const data = { 10: 'ten', 9: 'nine' } + const legacy = legacyRecordHash('s', 'Scores', data) + const v2hash = hashRecord('s', 'Scores', data).hash + expect(legacy).not.toBe(v2hash) + let res = await h.request(`${base}/versions/negotiate`, { + method: 'POST', + user, + json: { schemas: { Scores }, manifest: [{ id: 's', type: 'Scores', hash: legacy }] }, + }) + const n = await json(res) + expect(n.needed_records).toEqual([legacy]) + res = await h.request(`${base}/versions/negotiate/${n.session_id}/records`, { + method: 'POST', + user, + ndjson: [{ id: 's', type: 'Scores', data }], + }) + expect(res.status).toBe(200) + res = await h.request(`${base}/versions/negotiate/${n.session_id}/commit`, { + method: 'POST', + user, + }) + expect(res.status).toBe(201) + const v = await head(h, c.id) + const repo = await h.ports.stores.forCollection(c.id) + const root = await repo.root(v!.hash) + const entry = await getEntry( + new RepoSource(recordTree, repo), + root.public.types.Scores!.root, + 's', + ) + expect(entry?.hash).toBe(v2hash) + }) +}) diff --git a/packages/server/test/runs.test.ts b/packages/server/test/runs.test.ts new file mode 100644 index 0000000..4344432 --- /dev/null +++ b/packages/server/test/runs.test.ts @@ -0,0 +1,77 @@ +import { MemoryBlobStore } from '@underlay/repo/blob/memory' +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' + +import { + compactRuns, + MERGE_FAN_IN, + mergeRuns, + readRun, + type RunEntry, + RunWriter, + writeRun, +} from '../src/push/runs.js' + +async function collect(it: AsyncIterable): Promise { + const out: T[] = [] + for await (const x of it) out.push(x) + return out +} + +describe('sorted runs', () => { + it('writes blocks, reads them back in order, and filters by type', async () => { + const store = new MemoryBlobStore() + const entries: RunEntry[] = [] + for (let i = 0; i < 5000; i++) + entries.push({ t: i % 2 ? 'B' : 'A', k: `id${i}`, b: 'x'.repeat(500) }) + const ix = await writeRun(store, 's', 1, entries) + expect(ix.blocks.length).toBeGreaterThan(1) + const all = await collect(readRun(store, 's', ix)) + expect(all.length).toBe(5000) + expect(all.slice(0, 2500).every((e) => e.t === 'A')).toBe(true) + const onlyB = await collect(readRun(store, 's', ix, 'B')) + expect(onlyB.length).toBe(2500) + expect(onlyB.every((e) => e.t === 'B')).toBe(true) + }) + + it('refuses out-of-order writes', () => { + const w = new RunWriter(new MemoryBlobStore(), 's', 1) + w.add({ t: 'A', k: 'b' }) + expect(() => w.add({ t: 'A', k: 'a' })).toThrow(/out of order/) + }) + + it('merges and compacts any number of runs, later uploads winning (property)', async () => { + await fc.assert( + fc.asyncProperty( + fc.array( + fc.array(fc.tuple(fc.constantFrom('A', 'B', 'C'), fc.integer({ min: 0, max: 60 })), { + maxLength: 30, + }), + { + minLength: 1, + maxLength: 2 * MERGE_FAN_IN + 5, + }, + ), + async (batches) => { + const store = new MemoryBlobStore() + const want = new Map() + const indexes = [] + for (let seq = 1; seq <= batches.length; seq++) { + const entries = batches[seq - 1]!.map(([t, k]) => ({ t, k: `k${k}`, s: seq })) + for (const e of entries) want.set(`${e.t}\u0000${e.k}`, seq) + indexes.push(await writeRun(store, 's', seq, entries)) + } + const compacted = await compactRuns(store, 's', indexes, batches.length + 1) + expect(compacted.length).toBeLessThanOrEqual(MERGE_FAN_IN) + const merged = await collect(mergeRuns(store, 's', compacted)) + const got = new Map(merged.map((e) => [`${e.t}\u0000${e.k}`, e.s])) + expect(got).toEqual(want) + // Sorted by type, then id. + const keys = merged.map((e) => `${e.t}\u0000${e.k}`) + expect(keys).toEqual([...keys].sort()) + }, + ), + { numRuns: 60 }, + ) + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4b3e226..39f5624 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -249,6 +249,9 @@ importers: drizzle-kit: specifier: ^0.31.10 version: 0.31.10 + fast-check: + specifier: ^4.10.2 + version: 4.10.2 tsx: specifier: ^4.19.0 version: 4.21.0 From a2895e02f97c253e47af3379b2e87951ce3cea65 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 16:55:51 -0400 Subject: [PATCH 007/178] v2 auth: better-auth on SQLite with KF Auth, orgs and API keys Port of v1's better-auth setup to the v2 schema (D1/libsql, adapter without transactions): KF Auth OIDC sign-in, organizations with v1's server-controlled fields and slug rules, API keys with v1's scope clamping, personal org on sign-up. The authenticator reads Bearer keys (or ?token= on GET), then the session cookie; org-owned keys act as members of that org only. /api/auth/* is mounted; both entries build auth from v1's env names. --- packages/server/package.json | 2 + packages/server/src/api/access.ts | 6 +- packages/server/src/app.ts | 9 ++ packages/server/src/auth/auth.ts | 228 ++++++++++++++++++++++++++++++ packages/server/src/lib/slug.ts | 57 ++++++++ packages/server/src/node/main.ts | 25 +++- packages/server/src/worker.ts | 37 ++++- packages/server/test/auth.test.ts | 105 ++++++++++++++ packages/server/test/push.test.ts | 2 +- pnpm-lock.yaml | 6 + 10 files changed, 470 insertions(+), 7 deletions(-) create mode 100644 packages/server/src/auth/auth.ts create mode 100644 packages/server/src/lib/slug.ts create mode 100644 packages/server/test/auth.test.ts diff --git a/packages/server/package.json b/packages/server/package.json index 885f7c0..58b3ed8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -14,10 +14,12 @@ "dev:node": "tsx watch src/node/main.ts" }, "dependencies": { + "@better-auth/api-key": "^1.6.11", "@hono/node-server": "^1.19.14", "@libsql/client": "^0.18.0", "@underlay/core": "workspace:*", "@underlay/repo": "workspace:*", + "better-auth": "^1.6.11", "drizzle-orm": "^0.45.2", "hono": "^4.12.18" }, diff --git a/packages/server/src/api/access.ts b/packages/server/src/api/access.ts index 89daf30..4a2d909 100644 --- a/packages/server/src/api/access.ts +++ b/packages/server/src/api/access.ts @@ -18,6 +18,8 @@ export interface Principal { scope: 'session' | 'read' | 'write' | 'admin' /** Collections an API key is limited to; null when unscoped. */ collectionIds: string[] | null + /** Set for an API key owned by an organization: it acts as a member of that org only. */ + orgId?: string } export interface CollectionAccess { @@ -49,7 +51,9 @@ export async function collectionAccess( if (principal) { const keyCovers = principal.collectionIds === null || principal.collectionIds.includes(row.collection.id) - if (keyCovers) { + if (keyCovers && principal.orgId) { + isMember = principal.orgId === row.owner.id + } else if (keyCovers) { const [m] = await db .select({ id: schema.member.id }) .from(schema.member) diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 3efc56a..de49275 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -35,11 +35,20 @@ export type Setup = (c: Context) => { ports: Ports config: AppConfig authenticate: Authenticate + /** better-auth's own routes (/api/auth/*): sign-in, callbacks, sessions, keys, orgs. */ + authHandler?: (req: Request) => Promise } export function createApp(setup: Setup) { const app = new Hono() + app.on(['GET', 'POST'], '/api/auth/*', (c) => { + const { authHandler } = setup(c) + return authHandler + ? authHandler(c.req.raw) + : c.json({ error: 'Auth is not configured', statusCode: 404 }, 404) + }) + app.use('*', async (c, next) => { const { ports, config, authenticate } = setup(c) c.set('ports', ports) diff --git a/packages/server/src/auth/auth.ts b/packages/server/src/auth/auth.ts new file mode 100644 index 0000000..5afec93 --- /dev/null +++ b/packages/server/src/auth/auth.ts @@ -0,0 +1,228 @@ +/** + * Sign-in, sessions, organizations and API keys: better-auth, as in v1, over the + * v2 SQLite schema (D1 or libsql). The tables keep v1's fields so users, + * sessions and keys migrate across unchanged. + * + * KF Auth is the only sign-in method (OIDC via genericOAuth). The `kf_underlay` + * client is shared by www, next and staging; each host needs its callback + * registered in kf-auth (edge-redesign-build.md, finding 14). + * + * D1 has no interactive transactions, so the adapter runs without them. + */ +import { apiKey } from '@better-auth/api-key' +import { betterAuth } from 'better-auth' +import { drizzleAdapter } from 'better-auth/adapters/drizzle' +import { APIError } from 'better-auth/api' +import { genericOAuth } from 'better-auth/plugins' +import { organization } from 'better-auth/plugins/organization' +import { and, eq, ne } from 'drizzle-orm' + +import type { Principal } from '../api/access.js' +import type { Authenticate } from '../app.js' +import * as schema from '../db/schema.js' +import { defaultOrgSlugCandidate, validateSlug } from '../lib/slug.js' +import type { Db, Ports } from '../ports.js' + +export interface AuthConfig { + appUrl: string + /** better-auth secret (v1's SESSION_SECRET). */ + secret: string + oidc: { + /** Public issuer URL (browser redirects). */ + issuerUrl: string + /** Issuer URL for server-to-server calls; on Workers the same as issuerUrl. */ + internalUrl: string + clientId: string + clientSecret: string + } + /** Extra trusted origins (e.g. http:// variants in dev). */ + trustedOrigins?: string[] +} + +function assertValidSlug(slug: unknown) { + const err = validateSlug(slug) + if (err) throw new APIError('BAD_REQUEST', { message: err }) +} + +export function createAuth(db: Db, cfg: AuthConfig, waitUntil: (p: Promise) => void) { + const kf = cfg.oidc + return betterAuth({ + database: drizzleAdapter(db, { provider: 'sqlite', schema, transaction: false }), + baseURL: cfg.appUrl, + basePath: '/api/auth', + secret: cfg.secret, + trustedOrigins: [cfg.appUrl, ...(cfg.trustedOrigins ?? [])], + advanced: { + database: { generateId: () => crypto.randomUUID() }, + backgroundTasks: { handler: waitUntil }, + }, + plugins: [ + genericOAuth({ + config: [ + { + providerId: 'kf-auth', + authorizationUrl: `${kf.issuerUrl}/api/auth/oauth2/authorize`, + tokenUrl: `${kf.internalUrl}/api/auth/oauth2/token`, + userInfoUrl: `${kf.internalUrl}/api/auth/oauth2/userinfo`, + clientId: kf.clientId, + clientSecret: kf.clientSecret, + scopes: ['openid', 'profile', 'email', 'offline_access'], + pkce: true, + mapProfileToUser: (profile) => ({ + name: profile.name ?? profile.email?.split('@')[0] ?? 'User', + email: profile.email, + image: profile.picture ?? null, + }), + }, + ], + }), + organization({ + schema: { + organization: { + additionalFields: { + bio: { type: 'string', required: false, input: true }, + website: { type: 'string', required: false, input: true }, + avatarUrl: { type: 'string', required: false, input: true }, + // Server-controlled, as in v1: a caller must not claim another + // institution's NAAN or the default-org flag. + arkNaan: { type: 'string', required: false, input: false }, + kfOrgId: { type: 'string', required: false, input: false }, + isDefault: { type: 'boolean', required: false, input: false, defaultValue: false }, + }, + }, + }, + organizationHooks: { + beforeCreateOrganization: async ({ organization: org }) => { + assertValidSlug(org.slug) + }, + beforeUpdateOrganization: async ({ organization: org }) => { + if (org.slug !== undefined) assertValidSlug(org.slug) + }, + }, + }), + apiKey({ + defaultPrefix: 'ul', + customKeyGenerator: async ({ length }) => { + const { generateRandomString } = await import('better-auth/crypto') + return `ul_${generateRandomString(length, 'a-z', 'A-Z')}` + }, + enableMetadata: true, + // Don't make requests wait on the lastRequest write (v1 setting). + deferUpdates: true, + keyExpiration: { minExpiresIn: 0 }, + rateLimit: { enabled: false }, + permissions: { + defaultPermissions: async (_referenceId, ctx) => { + // metadata is client-controlled: 'admin' is clamped to write, as in v1. + const scope = ctx.body?.metadata?.scope + if (scope === 'write' || scope === 'admin') return { collections: ['write', 'read'] } + return { collections: ['read'] } + }, + }, + }), + ], + databaseHooks: { + account: { + create: { + after: async (account) => { + // One account per provider per user (v1). + await db + .delete(schema.account) + .where( + and( + eq(schema.account.userId, account.userId), + eq(schema.account.providerId, account.providerId), + ne(schema.account.id, account.id), + ), + ) + }, + }, + }, + user: { + create: { + after: async (user) => { + // Every user gets a personal (default) organization, as in v1. + let attempt = 0 + let slug = defaultOrgSlugCandidate(user.email, attempt) + for (;;) { + const [taken] = await db + .select({ id: schema.organization.id }) + .from(schema.organization) + .where(eq(schema.organization.slug, slug)) + .limit(1) + if (!taken) break + slug = defaultOrgSlugCandidate(user.email, ++attempt) + } + const orgId = crypto.randomUUID() + await db.batch([ + db + .insert(schema.organization) + .values({ id: orgId, name: user.name, slug, isDefault: true }), + db + .insert(schema.member) + .values({ organizationId: orgId, userId: user.id, role: 'owner' }), + ]) + }, + }, + }, + }, + }) +} + +export type Auth = ReturnType + +/** + * The request authenticator: API keys (Bearer, or ?token= on GET for share + * links), then the session cookie. An invalid Bearer key is anonymous here; the + * routes answer 401/403 as needed. + */ +export function authenticator(getAuth: (ports: Ports) => Auth): Authenticate { + return async (req, ports) => { + const auth = getAuth(ports) + const bearer = req.headers.get('authorization') + const queryToken = + req.method === 'GET' || req.method === 'HEAD' + ? new URL(req.url).searchParams.get('token') + : null + const key = bearer?.startsWith('Bearer ') ? bearer.slice(7) : queryToken + if (key) { + try { + const result = await auth.api.verifyApiKey({ body: { key } }) + if (result?.valid && result.key) { + const k = result.key as unknown as { + referenceId: string + permissions?: Record | null + metadata?: { collectionIds?: string[] } | null + } + const perms = k.permissions?.collections ?? [] + const scope = perms.includes('admin') + ? 'admin' + : perms.includes('write') + ? 'write' + : 'read' + const [org] = await ports.db + .select({ id: schema.organization.id }) + .from(schema.organization) + .where(eq(schema.organization.id, k.referenceId)) + .limit(1) + const principal: Principal = { + userId: k.referenceId, + scope, + collectionIds: k.metadata?.collectionIds?.length ? k.metadata.collectionIds : null, + ...(org ? { orgId: org.id } : {}), + } + return principal + } + } catch { + // Invalid or expired key: anonymous. + } + } + try { + const session = await auth.api.getSession({ headers: req.headers }) + if (session) return { userId: session.user.id, scope: 'session', collectionIds: null } + } catch { + // No session. + } + return null + } +} diff --git a/packages/server/src/lib/slug.ts b/packages/server/src/lib/slug.ts new file mode 100644 index 0000000..2fa7974 --- /dev/null +++ b/packages/server/src/lib/slug.ts @@ -0,0 +1,57 @@ +/** + * Account (organization) slug rules. Org slugs are the first path segment of + * every owner URL, so they must not collide with top-level routes. + */ + +export const RESERVED_SLUGS = new Set([ + 'explore', + 'docs', + 'connect', + 'blog', + 'dashboard', + 'settings', + 'api', + 'login', + 'signup', + 'admin', + 'about', + 'help', + 'support', + 'search', + 'new', + 'create', + 'edit', + 'delete', + '404', + '500', +]) + +export const SLUG_RE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/ + +/** Returns an error message, or null when the slug is acceptable. */ +export function validateSlug(slug: unknown): string | null { + if (!slug || typeof slug !== 'string') return 'Slug is required' + if (slug.length < 2) return 'Slug must be at least 2 characters' + if (slug.length > 64) return 'Slug must be at most 64 characters' + if (!SLUG_RE.test(slug)) { + return 'Slug must be lowercase alphanumeric with hyphens, and cannot start or end with a hyphen' + } + if (RESERVED_SLUGS.has(slug)) return 'That slug is reserved' + return null +} + +/** + * Candidate slug for a user's default org, derived from their email's local + * part. Falls back to `user` when the local part is too short or reserved + * (e.g. `admin@…`); `attempt` > 0 appends a numeric suffix for collisions. + */ +export function defaultOrgSlugCandidate(email: string, attempt = 0): string { + const local = (email.split('@')[0] ?? '') + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .slice(0, 30) + .replace(/^-+|-+$/g, '') + const base = local.length < 2 || RESERVED_SLUGS.has(local) ? 'user' : local + return attempt === 0 ? base : `${base}-${attempt}` +} diff --git a/packages/server/src/node/main.ts b/packages/server/src/node/main.ts index 38a713c..4129f74 100644 --- a/packages/server/src/node/main.ts +++ b/packages/server/src/node/main.ts @@ -10,6 +10,8 @@ * BLOB_URL_SECRET HMAC key for filesystem presigned URLs * REPO_PREFIX (repo), INTERNAL_PREFIX (internal) key prefixes in the platform bucket * SIGNING_KEY Ed25519 private key seed (base64url) that signs version logs + * SESSION_SECRET, OIDC_ISSUER_URL, OIDC_ISSUER_INTERNAL_URL, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET + * better-auth and KF Auth (same names as v1's .env files) */ import { serve } from '@hono/node-server' import { ed25519Signer, generateSigningKey, type Signer } from '@underlay/repo' @@ -18,6 +20,7 @@ import { S3BlobStore } from '@underlay/repo/blob/s3' import '../handlers.js' import { createApp } from '../app.js' +import { authenticator, createAuth } from '../auth/auth.js' import { MemoryCache } from '../cache.js' import { openNodeDb } from '../db/node.js' import { drainSqliteJobs, SqliteJobs } from '../jobs.js' @@ -100,8 +103,26 @@ kick = () => void runJobs() setInterval(kick, 5000).unref() const config = { appUrl, deployment: env.DEPLOYMENT ?? 'dev' } -// Sign-in and API keys (better-auth + KF Auth) land next; until then every caller is anonymous. -const app = createApp(() => ({ ports, config, authenticate: async () => null })) +const auth = createAuth( + db, + { + appUrl, + secret: env.SESSION_SECRET ?? 'dev-secret-change-me', + oidc: { + issuerUrl: env.OIDC_ISSUER_URL ?? 'http://localhost:3000', + internalUrl: env.OIDC_ISSUER_INTERNAL_URL ?? env.OIDC_ISSUER_URL ?? 'http://localhost:3000', + clientId: env.OIDC_CLIENT_ID ?? 'kf_underlay', + clientSecret: env.OIDC_CLIENT_SECRET ?? '', + }, + }, + ports.waitUntil, +) +const app = createApp(() => ({ + ports, + config, + authenticate: authenticator(() => auth), + authHandler: (req) => auth.handler(req), +})) serve({ port, diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index c4d2b89..4b21a19 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -14,6 +14,7 @@ import { S3BlobStore } from '@underlay/repo/blob/s3' import './handlers.js' import { createApp } from './app.js' +import { type Auth, authenticator, createAuth } from './auth/auth.js' import { CfCache } from './cache.js' import { openD1 } from './db/d1.js' import { QueueJobs, runJob } from './jobs.js' @@ -30,6 +31,10 @@ export interface Env { R2_ACCESS_KEY_ID: string R2_SECRET_ACCESS_KEY: string SIGNING_KEY: string + SESSION_SECRET: string + OIDC_ISSUER_URL: string + OIDC_CLIENT_ID: string + OIDC_CLIENT_SECRET: string REPO_PREFIX?: string INTERNAL_PREFIX?: string } @@ -60,13 +65,39 @@ function makePorts(env: Env, ctx: ExecutionContext): Ports { } } +// One better-auth instance per isolate and database binding. +const auths = new WeakMap() +function authFor(env: Env, ports: Ports): Auth { + let auth = auths.get(env.DB) + if (!auth) { + auth = createAuth( + ports.db, + { + appUrl: env.APP_URL, + secret: env.SESSION_SECRET, + oidc: { + issuerUrl: env.OIDC_ISSUER_URL, + // No private network on Workers: server-to-server calls use the public URL. + internalUrl: env.OIDC_ISSUER_URL, + clientId: env.OIDC_CLIENT_ID, + clientSecret: env.OIDC_CLIENT_SECRET, + }, + }, + ports.waitUntil, + ) + auths.set(env.DB, auth) + } + return auth +} + const app = createApp((c) => { const env = c.env as unknown as Env + const ports = makePorts(env, c.executionCtx as unknown as ExecutionContext) return { - ports: makePorts(env, c.executionCtx as unknown as ExecutionContext), + ports, config: { appUrl: env.APP_URL, deployment: env.DEPLOYMENT }, - // Sign-in and API keys (better-auth + KF Auth) land next. - authenticate: async () => null, + authenticate: authenticator(() => authFor(env, ports)), + authHandler: (req) => authFor(env, ports).handler(req), } }) diff --git a/packages/server/test/auth.test.ts b/packages/server/test/auth.test.ts new file mode 100644 index 0000000..f5be251 --- /dev/null +++ b/packages/server/test/auth.test.ts @@ -0,0 +1,105 @@ +import { afterAll, describe, expect, it } from 'vitest' + +import { createApp } from '../src/app.js' +import { authenticator, createAuth } from '../src/auth/auth.js' +import * as schema from '../src/db/schema.js' +import { cleanup, harness } from './harness.js' + +afterAll(cleanup) + +const Author = { type: 'object', properties: { name: { type: 'string' } } } + +describe('better-auth on SQLite', () => { + it('authenticates API keys with their scope and collection limits', async () => { + const h = await harness() + const user = await h.member('u1') + const c = await h.collection('authors') + await h.collection('other') + const auth = createAuth( + h.ports.db, + { + appUrl: 'http://test', + secret: 'test-secret-test-secret-test-secret', + oidc: { + issuerUrl: 'http://kf', + internalUrl: 'http://kf', + clientId: 'x', + clientSecret: 'y', + }, + }, + () => {}, + ) + const app = createApp(() => ({ + ports: h.ports, + config: { appUrl: 'http://test', deployment: 'test' }, + authenticate: authenticator(() => auth), + authHandler: (req) => auth.handler(req), + })) + const call = (path: string, key: string, body: unknown) => + app.fetch( + new Request(`http://test${path}`, { + method: 'POST', + headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + }), + ) + + const write = await auth.api.createApiKey({ + body: { userId: user, metadata: { scope: 'write' } }, + }) + const read = await auth.api.createApiKey({ body: { userId: user } }) + const scoped = await auth.api.createApiKey({ + body: { userId: user, metadata: { scope: 'write', collectionIds: [c.id] } }, + }) + expect(write.key).toMatch(/^ul_/) + + expect( + (await call('/api/collections/org/authors/push', write.key, { schemas: { Author } })).status, + ).toBe(200) + expect( + (await call('/api/collections/org/authors/push', read.key, { schemas: { Author } })).status, + ).toBe(403) + expect( + (await call('/api/collections/org/authors/push', 'ul_bogus', { schemas: { Author } })).status, + ).toBe(404) + expect( + (await call('/api/collections/org/authors/push', scoped.key, { schemas: { Author } })).status, + ).toBe(200) + // A key limited to one collection is a stranger everywhere else. + expect( + (await call('/api/collections/org/other/push', scoped.key, { schemas: { Author } })).status, + ).toBe(404) + + // The auth routes are mounted. + const res = await app.fetch(new Request('http://test/api/auth/get-session')) + expect(res.status).toBeLessThan(500) + }) + + it('gives every new user a personal organization', async () => { + const h = await harness() + const auth = createAuth( + h.ports.db, + { + appUrl: 'http://test', + secret: 'test-secret-test-secret-test-secret', + oidc: { + issuerUrl: 'http://kf', + internalUrl: 'http://kf', + clientId: 'x', + clientSecret: 'y', + }, + }, + () => {}, + ) + const ctx = await auth.$context + await ctx.internalAdapter.createUser({ + name: 'Ada', + email: 'ada@example.org', + emailVerified: true, + }) + const orgs = await h.ports.db.select().from(schema.organization) + expect(orgs.map((o) => [o.slug, o.isDefault])).toEqual([['ada', true]]) + const members = await h.ports.db.select().from(schema.member) + expect(members[0]).toMatchObject({ organizationId: orgs[0]!.id, role: 'owner' }) + }) +}) diff --git a/packages/server/test/push.test.ts b/packages/server/test/push.test.ts index 2ab8b6f..9d917a5 100644 --- a/packages/server/test/push.test.ts +++ b/packages/server/test/push.test.ts @@ -170,7 +170,7 @@ describe('negotiate (v1 compatibility)', () => { id: r.id, type: r.type, hash: hashOf(r.id, r.data), - ...(r.private ? { private: true } : {}), + ...((r as { private?: boolean }).private ? { private: true } : {}), })) async function push( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 39f5624..200ac53 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -221,6 +221,9 @@ importers: packages/server: dependencies: + '@better-auth/api-key': + specifier: ^1.6.11 + version: 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)))) '@hono/node-server': specifier: ^1.19.14 version: 1.19.14(hono@4.12.18) @@ -233,6 +236,9 @@ importers: '@underlay/repo': specifier: workspace:* version: link:../repo + better-auth: + specifier: ^1.6.11 + version: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) From 387ca6d79fdfa0fd1c8291b0076483af40e55c0d Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 17:02:13 -0400 Subject: [PATCH 008/178] v2 on workerd: D1-safe publish batch, wrangler config, smoke test Running the Worker under wrangler dev (workerd, local D1 and Queues, S3 API against the fake server) found that D1 batches only take query builders: the CAS publish is now INSERT ... SELECT from the collection row and conditional UPDATEs, built with Drizzle. scripts/smoke.ts pushes through a running deployment (delta sync and async commits, v1 negotiate with a format 1 hash) and passes on workerd. wrangler.jsonc defines the staging and next environments (placeholder D1 ids; nothing deployed). --- .gitignore | 4 + packages/repo/test/fake-s3-server.ts | 5 + packages/server/package.json | 3 +- packages/server/scripts/smoke.ts | 109 +++ packages/server/src/versions/publish.ts | 116 +++- packages/server/wrangler.jsonc | 89 +++ pnpm-lock.yaml | 841 ++++++++++++++++++++++++ 7 files changed, 1134 insertions(+), 33 deletions(-) create mode 100644 packages/repo/test/fake-s3-server.ts create mode 100644 packages/server/scripts/smoke.ts create mode 100644 packages/server/wrangler.jsonc diff --git a/.gitignore b/.gitignore index a3d73bf..60381a4 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ npm-debug.log* # pnpm store cache (created by in-container installs) .pnpm-store/ + +# Cloudflare local state and dev secrets (v2) +.wrangler/ +.dev.vars* diff --git a/packages/repo/test/fake-s3-server.ts b/packages/repo/test/fake-s3-server.ts new file mode 100644 index 0000000..0585117 --- /dev/null +++ b/packages/repo/test/fake-s3-server.ts @@ -0,0 +1,5 @@ +/** Run the fake S3 server standalone (for `wrangler dev` smoke tests): tsx test/fake-s3-server.ts [port] */ +import { startFakeS3 } from './fake-s3.js' + +const s3 = await startFakeS3(process.argv[3] ?? 'underlay-v2-staging') +console.log(`fake S3 at ${s3.url}`) diff --git a/packages/server/package.json b/packages/server/package.json index 58b3ed8..95b1c9e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -30,6 +30,7 @@ "fast-check": "^4.10.2", "tsx": "^4.19.0", "typescript": "^6.0.0", - "vitest": "^4.1.6" + "vitest": "^4.1.6", + "wrangler": "^4.147.0" } } diff --git a/packages/server/scripts/smoke.ts b/packages/server/scripts/smoke.ts new file mode 100644 index 0000000..27e8a76 --- /dev/null +++ b/packages/server/scripts/smoke.ts @@ -0,0 +1,109 @@ +/** + * End-to-end smoke test against a running deployment (local `wrangler dev`, the + * Node server, or staging): + * + * npx tsx scripts/smoke.ts + * + * Pushes with the delta API (sync and async commits) and the v1 negotiate API, + * then checks the head moved. Exits non-zero on any unexpected response. + */ +const [baseUrl, key, collection] = process.argv.slice(2) +if (!baseUrl || !key || !collection) { + console.error('usage: smoke.ts ') + process.exit(2) +} +const api = `${baseUrl.replace(/\/$/, '')}/api/collections/${collection}` +const auth = { authorization: `Bearer ${key}` } + +async function call(method: string, path: string, body?: unknown, ndjson = false) { + const res = await fetch(`${api}${path}`, { + method, + headers: { ...auth, 'content-type': ndjson ? 'application/x-ndjson' : 'application/json' }, + ...(body === undefined + ? {} + : { + body: ndjson + ? (body as unknown[]).map((l) => JSON.stringify(l)).join('\n') + : JSON.stringify(body), + }), + }) + const text = await res.text() + let json: any + try { + json = JSON.parse(text) + } catch { + json = text + } + console.log(`${method} ${path} → ${res.status}`, JSON.stringify(json).slice(0, 300)) + return { status: res.status, json } +} + +function expect(cond: unknown, what: string) { + if (!cond) { + console.error(`FAILED: ${what}`) + process.exit(1) + } +} + +const Author = { + type: 'object', + properties: { name: { type: 'string' }, born: { type: 'integer' } }, +} +const stamp = Date.now() + +// 1. Delta push, synchronous commit. +let r = await call('POST', '/push', { schemas: { Author }, metadata: { title: `Smoke ${stamp}` } }) +expect(r.status === 200, 'open delta session') +let sid = r.json.session_id +r = await call( + 'POST', + `/push/${sid}/records`, + [ + { id: `ada-${stamp}`, type: 'Author', data: { name: 'Ada', born: 1815 } }, + { id: `alan-${stamp}`, type: 'Author', data: { name: 'Alan', born: 1912 } }, + { id: `kurt-${stamp}`, type: 'Author', data: { name: 'Kurt' }, private: true }, + ], + true, +) +expect(r.status === 200 && r.json.received === 3, 'upload records') +r = await call('POST', `/push/${sid}/commit`) +expect(r.status === 201, 'sync commit') +const first = r.json.semver as string + +// 2. Delta push, async commit (a job on Queues / the jobs table). +r = await call('POST', '/push', { base: first }) +sid = r.json.session_id +await call('POST', `/push/${sid}/deletes`, [{ type: 'Author', id: `alan-${stamp}` }], true) +r = await call('POST', `/push/${sid}/commit?async=true`) +expect(r.status === 202, 'async commit accepted') +let status = '' +for (let i = 0; i < 60 && status !== 'committed' && status !== 'failed'; i++) { + await new Promise((res) => setTimeout(res, 1000)) + status = (await call('GET', `/push/${sid}`)).json.status +} +expect(status === 'committed', 'async commit finished') + +// 3. v1 negotiate: full snapshot with one change. +const records = [ + { id: `ada-${stamp}`, type: 'Author', data: { name: 'Ada Lovelace', born: 1815 } }, + { id: `grace-${stamp}`, type: 'Author', data: { name: 'Grace', born: 1906, 10: 'x', 9: 'y' } }, +] +const { hashRecord, legacyRecordHash } = await import('@underlay/core') +r = await call('POST', '/versions/negotiate', { + base_version: null, + schemas: { Author: { type: 'object' } }, + manifest: records.map((x) => ({ + id: x.id, + type: x.type, + hash: legacyRecordHash(x.id, x.type, x.data), + })), +}) +expect(r.status === 200, 'negotiate') +void hashRecord +const nsid = r.json.session_id +expect(r.json.needed_records.length === 2, 'both records needed') +r = await call('POST', `/versions/negotiate/${nsid}/records`, records, true) +expect(r.status === 200, 'negotiate upload') +r = await call('POST', `/versions/negotiate/${nsid}/commit`) +expect(r.status === 201 && r.json.recordCount === 2, 'negotiate commit') +console.log('\nsmoke OK') diff --git a/packages/server/src/versions/publish.ts b/packages/server/src/versions/publish.ts index 21ea37c..7bd6fe0 100644 --- a/packages/server/src/versions/publish.ts +++ b/packages/server/src/versions/publish.ts @@ -69,51 +69,103 @@ export async function publishVersion( ): Promise<{ ok: boolean; headVersionId: string | null }> { const v = p.version const now = Date.now() + // Literal values for INSERT … SELECT. Raw SQL bypasses column mapping, so JSON + // and booleans are given in their stored form. + const lit = (value: unknown) => sql`${value}` const versionExists = sql`EXISTS (SELECT 1 FROM ${schema.versions} WHERE ${schema.versions.id} = ${v.id})` - const headIsBase = sql`(SELECT ${schema.collections.headVersionId} FROM ${schema.collections} WHERE ${schema.collections.id} = ${v.collectionId}) IS ${p.baseVersionId}` + const headIsBase = and( + eq(schema.collections.id, v.collectionId), + sql`${schema.collections.headVersionId} IS ${p.baseVersionId}`, + ) + // Only query builders can go in a D1 batch (raw db.run() can't), so the + // conditions are INSERT … SELECT from the row they depend on, and UPDATE … WHERE. const statements = [ - db.run(sql` - INSERT INTO ${schema.versions} ( - id, collection_id, seq, semver, major, minor, patch, hash, base_semver, message, pushed_by, - app_id, actor_id, record_count, public_record_count, file_count, total_bytes, type_counts, - public_type_counts, has_private, public_refs_root, private_refs_root, changes, created_at - ) - SELECT ${v.id}, ${v.collectionId}, ${v.seq}, ${v.semver}, ${v.major}, ${v.minor}, ${v.patch}, - ${v.hash}, ${v.baseSemver}, ${v.message}, ${v.pushedBy}, ${v.appId}, ${v.actorId}, - ${v.recordCount}, ${v.publicRecordCount}, ${v.fileCount}, ${v.totalBytes}, - ${JSON.stringify(v.typeCounts)}, ${JSON.stringify(v.publicTypeCounts)}, ${v.hasPrivate ? 1 : 0}, - ${v.publicRefsRoot}, ${v.privateRefsRoot}, ${JSON.stringify(v.changes)}, ${now} - WHERE ${headIsBase} - `), - db.run(sql` - UPDATE ${schema.collections} - SET head_version_id = ${v.id}, updated_at = ${now}, - public_files_root = ${p.collectionUpdate.publicFilesRoot}, - summary = ${p.collectionUpdate.summary ? JSON.stringify(p.collectionUpdate.summary) : null} - WHERE id = ${v.collectionId} AND head_version_id IS ${p.baseVersionId} AND ${versionExists} - `), + // 1. The version row, only if the head is still the base: selected from the + // collection row, filtered by that condition. + db.insert(schema.versions).select( + db + .select({ + id: lit(v.id).as('id'), + collectionId: lit(v.collectionId).as('collection_id'), + seq: lit(v.seq).as('seq'), + semver: lit(v.semver).as('semver'), + major: lit(v.major).as('major'), + minor: lit(v.minor).as('minor'), + patch: lit(v.patch).as('patch'), + hash: lit(v.hash).as('hash'), + legacyHash: lit(null).as('legacy_hash'), + legacyPublicHash: lit(null).as('legacy_public_hash'), + baseSemver: lit(v.baseSemver).as('base_semver'), + message: lit(v.message).as('message'), + pushedBy: lit(v.pushedBy).as('pushed_by'), + appId: lit(v.appId).as('app_id'), + actorId: lit(v.actorId).as('actor_id'), + signature: lit(null).as('signature'), + recordCount: lit(v.recordCount).as('record_count'), + publicRecordCount: lit(v.publicRecordCount).as('public_record_count'), + fileCount: lit(v.fileCount).as('file_count'), + totalBytes: lit(v.totalBytes).as('total_bytes'), + typeCounts: lit(JSON.stringify(v.typeCounts)).as('type_counts'), + publicTypeCounts: lit(JSON.stringify(v.publicTypeCounts)).as( + 'public_type_counts', + ), + hasPrivate: lit(v.hasPrivate ? 1 : 0).as('has_private'), + publicRefsRoot: lit(v.publicRefsRoot).as('public_refs_root'), + privateRefsRoot: lit(v.privateRefsRoot).as('private_refs_root'), + changes: lit(JSON.stringify(v.changes)).as('changes'), + createdAt: lit(now).as('created_at'), + }) + .from(schema.collections) + .where(headIsBase) as never, + ), + // 2. Move the head, only if it is still the base and the row from 1 exists. + db + .update(schema.collections) + .set({ + headVersionId: v.id, + updatedAt: new Date(now), + publicFilesRoot: p.collectionUpdate.publicFilesRoot, + summary: p.collectionUpdate.summary, + }) + .where(and(headIsBase, versionExists)), ...p.schemaHashes.map((h) => - db.run(sql`INSERT OR IGNORE INTO ${schema.schemas} (hash, created_at) VALUES (${h}, ${now})`), + db.insert(schema.schemas).values({ hash: h }).onConflictDoNothing(), ), + // 3. Schema usage, only if the row from 1 exists. ...p.usage.flatMap((u) => { const out = [] if (u.wasOpen) { out.push( - db.run(sql` - UPDATE ${schema.schemaUsage} SET to_seq = ${v.seq} - WHERE collection_id = ${v.collectionId} AND type_slug = ${u.typeSlug} AND "set" = ${u.set} - AND to_seq IS NULL AND ${versionExists} - `), + db + .update(schema.schemaUsage) + .set({ toSeq: v.seq }) + .where( + and( + eq(schema.schemaUsage.collectionId, v.collectionId), + eq(schema.schemaUsage.typeSlug, u.typeSlug), + eq(schema.schemaUsage.set, u.set), + isNull(schema.schemaUsage.toSeq), + versionExists, + ), + ), ) } if (u.schemaHash !== null) { out.push( - db.run(sql` - INSERT INTO ${schema.schemaUsage} (schema_hash, collection_id, type_slug, "set", from_seq, to_seq) - SELECT ${u.schemaHash}, ${v.collectionId}, ${u.typeSlug}, ${u.set}, ${v.seq}, NULL - WHERE ${versionExists} - `), + db.insert(schema.schemaUsage).select( + db + .select({ + schemaHash: lit(u.schemaHash).as('schema_hash'), + collectionId: lit(v.collectionId).as('collection_id'), + typeSlug: lit(u.typeSlug).as('type_slug'), + set: lit(u.set).as('set'), + fromSeq: lit(v.seq).as('from_seq'), + toSeq: lit(null).as('to_seq'), + }) + .from(schema.versions) + .where(eq(schema.versions.id, v.id)) as never, + ), ) } return out diff --git a/packages/server/wrangler.jsonc b/packages/server/wrangler.jsonc new file mode 100644 index 0000000..a6e04da --- /dev/null +++ b/packages/server/wrangler.jsonc @@ -0,0 +1,89 @@ +// Underlay v2 on Cloudflare Workers: one Worker per deployment, each with its own +// D1 database, R2 bucket and queue (edge-redesign-build.md, "Deployment targets"). +// +// wrangler deploy --env staging → staging.underlay.org +// wrangler deploy --env next → next.underlay.org +// +// Secrets (wrangler secret put --env ): +// R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY bucket-scoped R2 S3 credentials +// SIGNING_KEY Ed25519 seed for version logs (one per deployment) +// SESSION_SECRET better-auth secret +// OIDC_CLIENT_SECRET kf_underlay client secret (shared with v1) +// +// D1 migrations: wrangler d1 migrations apply --env --remote +// The database ids below are placeholders until the databases are created. +{ + "$schema": "../../node_modules/wrangler/config-schema.json", + "name": "underlay-v2", + "main": "src/worker.ts", + "compatibility_date": "2026-09-30", + "compatibility_flags": ["nodejs_compat"], + "limits": { "cpu_ms": 300000 }, + "observability": { "enabled": true }, + "env": { + "staging": { + "name": "underlay-v2-staging", + "routes": [{ "pattern": "staging.underlay.org", "custom_domain": true }], + "vars": { + "APP_URL": "https://staging.underlay.org", + "DEPLOYMENT": "staging", + "R2_ENDPOINT": "https://b66a0000000000000000000000000000.r2.cloudflarestorage.com", + "R2_BUCKET": "underlay-v2-staging", + "OIDC_ISSUER_URL": "https://auth.knowledgefutures.org", + "OIDC_CLIENT_ID": "kf_underlay" + }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "underlay-v2-staging", + "database_id": "00000000-0000-0000-0000-000000000000", + "migrations_dir": "drizzle" + } + ], + "queues": { + "producers": [{ "binding": "JOBS", "queue": "underlay-v2-staging-jobs" }], + "consumers": [ + { + "queue": "underlay-v2-staging-jobs", + "max_batch_size": 10, + "max_retries": 10, + "dead_letter_queue": "underlay-v2-staging-dead" + } + ] + }, + "triggers": { "crons": ["*/10 * * * *"] } + }, + "next": { + "name": "underlay-v2-next", + "routes": [{ "pattern": "next.underlay.org", "custom_domain": true }], + "vars": { + "APP_URL": "https://next.underlay.org", + "DEPLOYMENT": "next", + "R2_ENDPOINT": "https://b66a0000000000000000000000000000.r2.cloudflarestorage.com", + "R2_BUCKET": "underlay-v2-next", + "OIDC_ISSUER_URL": "https://auth.knowledgefutures.org", + "OIDC_CLIENT_ID": "kf_underlay" + }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "underlay-v2-next", + "database_id": "00000000-0000-0000-0000-000000000000", + "migrations_dir": "drizzle" + } + ], + "queues": { + "producers": [{ "binding": "JOBS", "queue": "underlay-v2-next-jobs" }], + "consumers": [ + { + "queue": "underlay-v2-next-jobs", + "max_batch_size": 10, + "max_retries": 10, + "dead_letter_queue": "underlay-v2-next-dead" + } + ] + }, + "triggers": { "crons": ["*/10 * * * *"] } + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 200ac53..5253cd4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -267,6 +267,9 @@ importers: vitest: specifier: ^4.1.6 version: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + wrangler: + specifier: ^4.147.0 + version: 4.147.0(@cloudflare/workers-types@5.20261003.1)(@types/node@25.6.2) packages: @@ -644,6 +647,49 @@ packages: '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} + '@cloudflare/kv-asset-handler@0.5.0': + resolution: {integrity: sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==} + engines: {node: '>=22.0.0'} + + '@cloudflare/unenv-preset@2.16.2': + resolution: {integrity: sha512-JBP1+Z7ZSNG/d4mRP+y8VC5dka3tZVMLEZRvS+rzQ4DGV1EoxRFQckcJTTkXbHSQiTj0DtNI01Zwb/V2fX0mvQ==} + peerDependencies: + unenv: 2.0.0-rc.24 + workerd: '>1.20260305.0 <2.0.0-0' + peerDependenciesMeta: + workerd: + optional: true + + '@cloudflare/workerd-darwin-64@1.20261001.1': + resolution: {integrity: sha512-4cgSgDf28JSw/P5Dj5GCS59hzVqS5XnmGAWNkvYHLI6ODU9idGaMMNEuhJXDkEG/lsABmlVlnCgsdh2VbKWepw==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20261001.1': + resolution: {integrity: sha512-8ulAWruEVouNmEIsQsy9WSSCS9zkLu93W2MTwp5esiFyoPp05BNSVFIFsYSPi1pkFmBBd7fsnwMpbLkaJLaVPQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20261001.1': + resolution: {integrity: sha512-kZbTZJGrhsMOdqZ2BIybjaBRLZjYsRLWj7mcNw/6Y3hodOhp5MrNzcz4iWGwNVWwyIV+7Pl+/LX5VcgnRqdHOg==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20261001.1': + resolution: {integrity: sha512-oOk3Zj6k/8oP0FJgZBWDn7+BqbsqIMEMaV95pulHPVbwVu4wYoLfQq2hp0vkbCNsCFLqZb7cqixXdrwD54ZIow==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20261001.1': + resolution: {integrity: sha512-uRxm5W4VyBkoSaoP1BfOuH0873tE+vxsknF4sab6/5YIRvIAufChq3QDp+zlAn67PuGPGcn7W8QraA0t4ucmOQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + '@cloudflare/workers-types@5.20261003.1': resolution: {integrity: sha512-Uii0J5qUd/R6lIDK/u0Sw8dEi7oI3ljXw7EDybuPntoXg2Yy5Y9H9UeNQ5R87oQ/Q4jq6uEJmYOOFQ3Z7C94lA==} @@ -665,6 +711,10 @@ packages: '@codemirror/view@6.42.1': resolution: {integrity: sha512-ToN3oFc0nsxNUYVF5P0ztLgbC4UPPjPtA9aKYhkOKQaZASpOUo6ISXyQLP66ctVwlDc+j6Jv0uK5IFALkiXztg==} + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + '@csstools/color-helpers@6.0.2': resolution: {integrity: sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==} engines: {node: '>=20.19.0'} @@ -704,6 +754,9 @@ packages: '@drizzle-team/brocli@0.10.2': resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} + '@emnapi/runtime@1.11.3': + resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} + '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} deprecated: 'Merged into tsx: https://tsx.is' @@ -724,6 +777,12 @@ packages: cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/android-arm64@0.18.20': resolution: {integrity: sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==} engines: {node: '>=12'} @@ -742,6 +801,12 @@ packages: cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm@0.18.20': resolution: {integrity: sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==} engines: {node: '>=12'} @@ -760,6 +825,12 @@ packages: cpu: [arm] os: [android] + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-x64@0.18.20': resolution: {integrity: sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==} engines: {node: '>=12'} @@ -778,6 +849,12 @@ packages: cpu: [x64] os: [android] + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/darwin-arm64@0.18.20': resolution: {integrity: sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==} engines: {node: '>=12'} @@ -796,6 +873,12 @@ packages: cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-x64@0.18.20': resolution: {integrity: sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==} engines: {node: '>=12'} @@ -814,6 +897,12 @@ packages: cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/freebsd-arm64@0.18.20': resolution: {integrity: sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==} engines: {node: '>=12'} @@ -832,6 +921,12 @@ packages: cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-x64@0.18.20': resolution: {integrity: sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==} engines: {node: '>=12'} @@ -850,6 +945,12 @@ packages: cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/linux-arm64@0.18.20': resolution: {integrity: sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==} engines: {node: '>=12'} @@ -868,6 +969,12 @@ packages: cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm@0.18.20': resolution: {integrity: sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==} engines: {node: '>=12'} @@ -886,6 +993,12 @@ packages: cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-ia32@0.18.20': resolution: {integrity: sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==} engines: {node: '>=12'} @@ -904,6 +1017,12 @@ packages: cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-loong64@0.18.20': resolution: {integrity: sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==} engines: {node: '>=12'} @@ -922,6 +1041,12 @@ packages: cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-mips64el@0.18.20': resolution: {integrity: sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==} engines: {node: '>=12'} @@ -940,6 +1065,12 @@ packages: cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-ppc64@0.18.20': resolution: {integrity: sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==} engines: {node: '>=12'} @@ -958,6 +1089,12 @@ packages: cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-riscv64@0.18.20': resolution: {integrity: sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==} engines: {node: '>=12'} @@ -976,6 +1113,12 @@ packages: cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-s390x@0.18.20': resolution: {integrity: sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==} engines: {node: '>=12'} @@ -994,6 +1137,12 @@ packages: cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-x64@0.18.20': resolution: {integrity: sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==} engines: {node: '>=12'} @@ -1012,6 +1161,12 @@ packages: cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/netbsd-arm64@0.25.12': resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} engines: {node: '>=18'} @@ -1024,6 +1179,12 @@ packages: cpu: [arm64] os: [netbsd] + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + '@esbuild/netbsd-x64@0.18.20': resolution: {integrity: sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==} engines: {node: '>=12'} @@ -1042,6 +1203,12 @@ packages: cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + '@esbuild/openbsd-arm64@0.25.12': resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} engines: {node: '>=18'} @@ -1054,6 +1221,12 @@ packages: cpu: [arm64] os: [openbsd] + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-x64@0.18.20': resolution: {integrity: sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==} engines: {node: '>=12'} @@ -1072,6 +1245,12 @@ packages: cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/openharmony-arm64@0.25.12': resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} engines: {node: '>=18'} @@ -1084,6 +1263,12 @@ packages: cpu: [arm64] os: [openharmony] + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + '@esbuild/sunos-x64@0.18.20': resolution: {integrity: sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==} engines: {node: '>=12'} @@ -1102,6 +1287,12 @@ packages: cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/win32-arm64@0.18.20': resolution: {integrity: sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==} engines: {node: '>=12'} @@ -1120,6 +1311,12 @@ packages: cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-ia32@0.18.20': resolution: {integrity: sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==} engines: {node: '>=12'} @@ -1138,6 +1335,12 @@ packages: cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-x64@0.18.20': resolution: {integrity: sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==} engines: {node: '>=12'} @@ -1156,6 +1359,12 @@ packages: cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@exodus/bytes@1.15.1': resolution: {integrity: sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -1198,6 +1407,168 @@ packages: '@hyperjump/uri@1.3.8': resolution: {integrity: sha512-pQ1IFUIdUrMzj6TjG0wyaAGZ0Go2bi4eFS8EydFLuBKGYpU0sTjd/R1td+lqhn3hCq7G17qZp975WL87MU84dw==} + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.35.4': + resolution: {integrity: sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.35.4': + resolution: {integrity: sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [darwin] + + '@img/sharp-freebsd-wasm32@0.35.4': + resolution: {integrity: sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==} + engines: {node: '>=20.9.0'} + os: [freebsd] + + '@img/sharp-libvips-darwin-arm64@1.3.3': + resolution: {integrity: sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.3.3': + resolution: {integrity: sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.3.3': + resolution: {integrity: sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.3.3': + resolution: {integrity: sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.3.3': + resolution: {integrity: sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.3.3': + resolution: {integrity: sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.3.3': + resolution: {integrity: sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.3.3': + resolution: {integrity: sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': + resolution: {integrity: sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.3.3': + resolution: {integrity: sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.35.4': + resolution: {integrity: sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.35.4': + resolution: {integrity: sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==} + engines: {node: '>=20.9.0'} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.35.4': + resolution: {integrity: sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==} + engines: {node: '>=20.9.0'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.35.4': + resolution: {integrity: sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==} + engines: {node: '>=20.9.0'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.35.4': + resolution: {integrity: sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==} + engines: {node: '>=20.9.0'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.35.4': + resolution: {integrity: sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.35.4': + resolution: {integrity: sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.35.4': + resolution: {integrity: sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.35.4': + resolution: {integrity: sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==} + engines: {node: '>=20.9.0'} + + '@img/sharp-webcontainers-wasm32@0.35.4': + resolution: {integrity: sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==} + engines: {node: '>=20.9.0'} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.35.4': + resolution: {integrity: sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.35.4': + resolution: {integrity: sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==} + engines: {node: ^20.9.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.35.4': + resolution: {integrity: sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [win32] + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -1214,6 +1585,9 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + '@lezer/common@1.5.2': resolution: {integrity: sha512-sxQE460fPZyU3sdc8lafxiPwJHBzZRy/udNFynGQky1SePYBdhkBl1kOagA9uT3pxR8K09bOrmTUqA9wb/PjSQ==} @@ -1545,6 +1919,15 @@ packages: cpu: [x64] os: [win32] + '@poppinss/colors@4.1.6': + resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} + + '@poppinss/dumper@0.6.5': + resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} + + '@poppinss/exception@1.2.3': + resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + '@rolldown/pluginutils@1.0.0-rc.3': resolution: {integrity: sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q==} @@ -1712,6 +2095,10 @@ packages: resolution: {integrity: sha512-tpmmG+/xRE2Kn9RpflU3AIyZv08v10+E1ZrJCx7z6+/91zHVxy0M73kC1LT4/8PbYNt85ywyC8+n+D99JdMcGA==} engines: {node: '>=20'} + '@sindresorhus/is@7.2.0': + resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} + engines: {node: '>=18'} + '@smithy/chunked-blob-reader-native@4.2.3': resolution: {integrity: sha512-jA5k5Udn7Y5717L86h4EIv06wIr3xn8GM1qHRi/Nf31annXcXHJjBKvgztnbn2TxH3xWrPBfgwHsOwZf0UmQWw==} engines: {node: '>=18.0.0'} @@ -1936,6 +2323,9 @@ packages: resolution: {integrity: sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g==} engines: {node: '>=18.0.0'} + '@speed-highlight/core@1.2.24': + resolution: {integrity: sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -2296,6 +2686,9 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + blake3-wasm@2.1.5: + resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} + bowser@2.14.1: resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} @@ -2512,6 +2905,9 @@ packages: resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} engines: {node: '>=18'} + error-stack-parser-es@1.0.5: + resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} + es-module-lexer@2.1.0: resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} @@ -2530,6 +2926,11 @@ packages: engines: {node: '>=18'} hasBin: true + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + escalade@3.2.0: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} @@ -2689,6 +3090,10 @@ packages: just-curry-it@5.3.0: resolution: {integrity: sha512-silMIRiFjUWlfaDhkgSzpuAyQ6EX/o09Eu8ZBfmFwQMbax7+LQzeIU2CBrICT6Ne4l86ITCGvUCBpCubWYy0Yw==} + kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + kysely@0.28.17: resolution: {integrity: sha512-nbD8lB9EB3wNdMhOCdx5Li8DxnLbvKByylRLcJ1h+4SkrowVeECAyZlyiKMThF7xFdRz0jSQ2MoJr+wXux2y0Q==} engines: {node: '>=20.0.0'} @@ -2816,6 +3221,10 @@ packages: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} + miniflare@5.20261001.0-alpha: + resolution: {integrity: sha512-GaimS5mSIOMyvd16ga+e1/QkI8cmp3z35QPHFex0DktqNQf2RVZQwMPQXdyoUreUiFP/0Gpe2MoQInTyMAD5xA==} + engines: {node: '>=22.0.0'} + minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} @@ -2896,6 +3305,9 @@ packages: resolution: {integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==} engines: {node: '>=14.0.0'} + path-to-regexp@6.3.0: + resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} @@ -3005,12 +3417,26 @@ packages: engines: {node: '>=10'} hasBin: true + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + set-cookie-parser@2.7.2: resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} set-cookie-parser@3.1.0: resolution: {integrity: sha512-kjnC1DXBHcxaOaOXBHBeRtltsDG2nUiUni+jP92M9gYdW12rsmx92UsfpH7o5tDRs7I1ZZPSQJQGv3UaRfCiuw==} + sharp@0.35.4: + resolution: {integrity: sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==} + engines: {node: '>=20.9.0'} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -3088,6 +3514,10 @@ packages: style-mod@4.1.3: resolution: {integrity: sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==} + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + symbol-tree@3.2.4: resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} @@ -3179,6 +3609,13 @@ packages: resolution: {integrity: sha512-uZsKNuzQxDMUY6M3pIMvy5tvlGmtq8XJ2oLAkfRKGNu+1VQAIvLy2xIVG5ATZl5wDXl/tddByAWCizRbOme+TA==} engines: {node: '>=20.18.1'} + undici@7.29.1: + resolution: {integrity: sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==} + engines: {node: '>=20.18.1'} + + unenv@2.0.0-rc.24: + resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + update-browserslist-db@1.2.3: resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} hasBin: true @@ -3301,6 +3738,21 @@ packages: engines: {node: '>=8'} hasBin: true + workerd@1.20261001.1: + resolution: {integrity: sha512-d/SIYHFO0PT/wiFZg8in4NpRIxYuFwslX1HdylOtWkBIIUmSpkGFhK820cV84XACFylwJ48xuRoWW/8DWDPsPQ==} + engines: {node: '>=16'} + hasBin: true + + wrangler@4.147.0: + resolution: {integrity: sha512-pQYRoiq8PTAxphaG69z8+GC1DkSGd19EDZehQ8zxjo/Ko3mRB6Qs1mTrd8ZuKAarLklIjTqr1lUdCK9r4q2hUg==} + engines: {node: '>=22.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^5.20261001.1 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + wrap-ansi@10.0.0: resolution: {integrity: sha512-SGcvg80f0wUy2/fXES19feHMz8E0JoXv2uNgHOu4Dgi2OrCy1lqwFYEJz1BLbDI0exjPMe/ZdzZ/YpGECBG/aQ==} engines: {node: '>=20'} @@ -3324,6 +3776,18 @@ packages: utf-8-validate: optional: true + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + xml-name-validator@5.0.0: resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} engines: {node: '>=18'} @@ -3343,6 +3807,12 @@ packages: engines: {node: '>= 14.6'} hasBin: true + youch-core@0.3.3: + resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} + + youch@4.1.0-beta.10: + resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} + zod-openapi@5.4.6: resolution: {integrity: sha512-P2jsOOBAq/6hCwUsMCjUATZ8szkMsV5VAwZENfyxp2Hc/XPJQpVwAgevWZc65xZauCwWB9LAn7zYeiCJFAEL+A==} engines: {node: '>=20'} @@ -4040,6 +4510,29 @@ snapshots: '@cfworker/json-schema@4.1.1': {} + '@cloudflare/kv-asset-handler@0.5.0': {} + + '@cloudflare/unenv-preset@2.16.2(unenv@2.0.0-rc.24)(workerd@1.20261001.1)': + dependencies: + unenv: 2.0.0-rc.24 + optionalDependencies: + workerd: 1.20261001.1 + + '@cloudflare/workerd-darwin-64@1.20261001.1': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20261001.1': + optional: true + + '@cloudflare/workerd-linux-64@1.20261001.1': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20261001.1': + optional: true + + '@cloudflare/workerd-windows-64@1.20261001.1': + optional: true + '@cloudflare/workers-types@5.20261003.1': {} '@codemirror/autocomplete@6.20.2': @@ -4085,6 +4578,10 @@ snapshots: style-mod: 4.1.3 w3c-keyname: 2.2.8 + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + '@csstools/color-helpers@6.0.2': {} '@csstools/css-calc@3.2.1(@csstools/css-parser-algorithms@4.0.0(@csstools/css-tokenizer@4.0.0))(@csstools/css-tokenizer@4.0.0)': @@ -4111,6 +4608,11 @@ snapshots: '@drizzle-team/brocli@0.10.2': {} + '@emnapi/runtime@1.11.3': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild-kit/core-utils@3.3.2': dependencies: esbuild: 0.18.20 @@ -4127,6 +4629,9 @@ snapshots: '@esbuild/aix-ppc64@0.27.7': optional: true + '@esbuild/aix-ppc64@0.28.1': + optional: true + '@esbuild/android-arm64@0.18.20': optional: true @@ -4136,6 +4641,9 @@ snapshots: '@esbuild/android-arm64@0.27.7': optional: true + '@esbuild/android-arm64@0.28.1': + optional: true + '@esbuild/android-arm@0.18.20': optional: true @@ -4145,6 +4653,9 @@ snapshots: '@esbuild/android-arm@0.27.7': optional: true + '@esbuild/android-arm@0.28.1': + optional: true + '@esbuild/android-x64@0.18.20': optional: true @@ -4154,6 +4665,9 @@ snapshots: '@esbuild/android-x64@0.27.7': optional: true + '@esbuild/android-x64@0.28.1': + optional: true + '@esbuild/darwin-arm64@0.18.20': optional: true @@ -4163,6 +4677,9 @@ snapshots: '@esbuild/darwin-arm64@0.27.7': optional: true + '@esbuild/darwin-arm64@0.28.1': + optional: true + '@esbuild/darwin-x64@0.18.20': optional: true @@ -4172,6 +4689,9 @@ snapshots: '@esbuild/darwin-x64@0.27.7': optional: true + '@esbuild/darwin-x64@0.28.1': + optional: true + '@esbuild/freebsd-arm64@0.18.20': optional: true @@ -4181,6 +4701,9 @@ snapshots: '@esbuild/freebsd-arm64@0.27.7': optional: true + '@esbuild/freebsd-arm64@0.28.1': + optional: true + '@esbuild/freebsd-x64@0.18.20': optional: true @@ -4190,6 +4713,9 @@ snapshots: '@esbuild/freebsd-x64@0.27.7': optional: true + '@esbuild/freebsd-x64@0.28.1': + optional: true + '@esbuild/linux-arm64@0.18.20': optional: true @@ -4199,6 +4725,9 @@ snapshots: '@esbuild/linux-arm64@0.27.7': optional: true + '@esbuild/linux-arm64@0.28.1': + optional: true + '@esbuild/linux-arm@0.18.20': optional: true @@ -4208,6 +4737,9 @@ snapshots: '@esbuild/linux-arm@0.27.7': optional: true + '@esbuild/linux-arm@0.28.1': + optional: true + '@esbuild/linux-ia32@0.18.20': optional: true @@ -4217,6 +4749,9 @@ snapshots: '@esbuild/linux-ia32@0.27.7': optional: true + '@esbuild/linux-ia32@0.28.1': + optional: true + '@esbuild/linux-loong64@0.18.20': optional: true @@ -4226,6 +4761,9 @@ snapshots: '@esbuild/linux-loong64@0.27.7': optional: true + '@esbuild/linux-loong64@0.28.1': + optional: true + '@esbuild/linux-mips64el@0.18.20': optional: true @@ -4235,6 +4773,9 @@ snapshots: '@esbuild/linux-mips64el@0.27.7': optional: true + '@esbuild/linux-mips64el@0.28.1': + optional: true + '@esbuild/linux-ppc64@0.18.20': optional: true @@ -4244,6 +4785,9 @@ snapshots: '@esbuild/linux-ppc64@0.27.7': optional: true + '@esbuild/linux-ppc64@0.28.1': + optional: true + '@esbuild/linux-riscv64@0.18.20': optional: true @@ -4253,6 +4797,9 @@ snapshots: '@esbuild/linux-riscv64@0.27.7': optional: true + '@esbuild/linux-riscv64@0.28.1': + optional: true + '@esbuild/linux-s390x@0.18.20': optional: true @@ -4262,6 +4809,9 @@ snapshots: '@esbuild/linux-s390x@0.27.7': optional: true + '@esbuild/linux-s390x@0.28.1': + optional: true + '@esbuild/linux-x64@0.18.20': optional: true @@ -4271,12 +4821,18 @@ snapshots: '@esbuild/linux-x64@0.27.7': optional: true + '@esbuild/linux-x64@0.28.1': + optional: true + '@esbuild/netbsd-arm64@0.25.12': optional: true '@esbuild/netbsd-arm64@0.27.7': optional: true + '@esbuild/netbsd-arm64@0.28.1': + optional: true + '@esbuild/netbsd-x64@0.18.20': optional: true @@ -4286,12 +4842,18 @@ snapshots: '@esbuild/netbsd-x64@0.27.7': optional: true + '@esbuild/netbsd-x64@0.28.1': + optional: true + '@esbuild/openbsd-arm64@0.25.12': optional: true '@esbuild/openbsd-arm64@0.27.7': optional: true + '@esbuild/openbsd-arm64@0.28.1': + optional: true + '@esbuild/openbsd-x64@0.18.20': optional: true @@ -4301,12 +4863,18 @@ snapshots: '@esbuild/openbsd-x64@0.27.7': optional: true + '@esbuild/openbsd-x64@0.28.1': + optional: true + '@esbuild/openharmony-arm64@0.25.12': optional: true '@esbuild/openharmony-arm64@0.27.7': optional: true + '@esbuild/openharmony-arm64@0.28.1': + optional: true + '@esbuild/sunos-x64@0.18.20': optional: true @@ -4316,6 +4884,9 @@ snapshots: '@esbuild/sunos-x64@0.27.7': optional: true + '@esbuild/sunos-x64@0.28.1': + optional: true + '@esbuild/win32-arm64@0.18.20': optional: true @@ -4325,6 +4896,9 @@ snapshots: '@esbuild/win32-arm64@0.27.7': optional: true + '@esbuild/win32-arm64@0.28.1': + optional: true + '@esbuild/win32-ia32@0.18.20': optional: true @@ -4334,6 +4908,9 @@ snapshots: '@esbuild/win32-ia32@0.27.7': optional: true + '@esbuild/win32-ia32@0.28.1': + optional: true + '@esbuild/win32-x64@0.18.20': optional: true @@ -4343,6 +4920,9 @@ snapshots: '@esbuild/win32-x64@0.27.7': optional: true + '@esbuild/win32-x64@0.28.1': + optional: true + '@exodus/bytes@1.15.1(@noble/hashes@2.2.0)': optionalDependencies: '@noble/hashes': 2.2.0 @@ -4384,6 +4964,112 @@ snapshots: '@hyperjump/uri@1.3.8': {} + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.3.3 + optional: true + + '@img/sharp-darwin-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.3.3 + optional: true + + '@img/sharp-freebsd-wasm32@0.35.4': + dependencies: + '@img/sharp-wasm32': 0.35.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-darwin-x64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-arm@1.3.3': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-s390x@1.3.3': + optional: true + + '@img/sharp-libvips-linux-x64@1.3.3': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.3.3': + optional: true + + '@img/sharp-linux-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.3.3 + optional: true + + '@img/sharp-linux-arm@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.3.3 + optional: true + + '@img/sharp-linux-ppc64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.3.3 + optional: true + + '@img/sharp-linux-riscv64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.3.3 + optional: true + + '@img/sharp-linux-s390x@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.3.3 + optional: true + + '@img/sharp-linux-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.3.3 + optional: true + + '@img/sharp-linuxmusl-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 + optional: true + + '@img/sharp-linuxmusl-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 + optional: true + + '@img/sharp-wasm32@0.35.4': + dependencies: + '@emnapi/runtime': 1.11.3 + optional: true + + '@img/sharp-webcontainers-wasm32@0.35.4': + dependencies: + '@img/sharp-wasm32': 0.35.4 + optional: true + + '@img/sharp-win32-arm64@0.35.4': + optional: true + + '@img/sharp-win32-ia32@0.35.4': + optional: true + + '@img/sharp-win32-x64@0.35.4': + optional: true + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -4403,6 +5089,11 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + '@lezer/common@1.5.2': {} '@lezer/highlight@1.2.3': @@ -4597,6 +5288,18 @@ snapshots: '@oxlint/binding-win32-x64-msvc@1.77.0': optional: true + '@poppinss/colors@4.1.6': + dependencies: + kleur: 4.1.5 + + '@poppinss/dumper@0.6.5': + dependencies: + '@poppinss/colors': 4.1.6 + '@sindresorhus/is': 7.2.0 + supports-color: 10.2.2 + + '@poppinss/exception@1.2.3': {} + '@rolldown/pluginutils@1.0.0-rc.3': {} '@rollup/rollup-android-arm-eabi@4.60.3': @@ -4701,6 +5404,8 @@ snapshots: '@scalar/validation@0.6.0': {} + '@sindresorhus/is@7.2.0': {} + '@smithy/chunked-blob-reader-native@4.2.3': dependencies: '@smithy/util-base64': 4.3.2 @@ -5049,6 +5754,8 @@ snapshots: dependencies: tslib: 2.8.1 + '@speed-highlight/core@1.2.24': {} + '@standard-schema/spec@1.1.0': {} '@tailwindcss/node@4.3.0': @@ -5361,6 +6068,8 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + blake3-wasm@2.1.5: {} + bowser@2.14.1: {} browserslist@4.28.2: @@ -5475,6 +6184,8 @@ snapshots: environment@1.1.0: {} + error-stack-parser-es@1.0.5: {} + es-module-lexer@2.1.0: {} esbuild@0.18.20: @@ -5560,6 +6271,35 @@ snapshots: '@esbuild/win32-ia32': 0.27.7 '@esbuild/win32-x64': 0.27.7 + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + escalade@3.2.0: {} estree-walker@3.0.3: @@ -5716,6 +6456,8 @@ snapshots: just-curry-it@5.3.0: {} + kleur@4.1.5: {} + kysely@0.28.17: {} libsql@0.5.29: @@ -5829,6 +6571,19 @@ snapshots: mimic-response@3.1.0: {} + miniflare@5.20261001.0-alpha(@types/node@25.6.2): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + sharp: 0.35.4(@types/node@25.6.2) + undici: 7.29.1 + workerd: 1.20261001.1 + ws: 8.21.0 + youch: 4.1.0-beta.10 + transitivePeerDependencies: + - '@types/node' + - bufferutil + - utf-8-validate + minimist@1.2.8: {} mkdirp-classic@0.5.3: {} @@ -5913,6 +6668,8 @@ snapshots: path-expression-matcher@1.5.0: {} + path-to-regexp@6.3.0: {} + pathe@2.0.3: {} picocolors@1.1.1: {} @@ -6039,10 +6796,45 @@ snapshots: semver@7.8.0: {} + semver@7.8.5: {} + set-cookie-parser@2.7.2: {} set-cookie-parser@3.1.0: {} + sharp@0.35.4(@types/node@25.6.2): + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.35.4 + '@img/sharp-darwin-x64': 0.35.4 + '@img/sharp-freebsd-wasm32': 0.35.4 + '@img/sharp-libvips-darwin-arm64': 1.3.3 + '@img/sharp-libvips-darwin-x64': 1.3.3 + '@img/sharp-libvips-linux-arm': 1.3.3 + '@img/sharp-libvips-linux-arm64': 1.3.3 + '@img/sharp-libvips-linux-ppc64': 1.3.3 + '@img/sharp-libvips-linux-riscv64': 1.3.3 + '@img/sharp-libvips-linux-s390x': 1.3.3 + '@img/sharp-libvips-linux-x64': 1.3.3 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 + '@img/sharp-linux-arm': 0.35.4 + '@img/sharp-linux-arm64': 0.35.4 + '@img/sharp-linux-ppc64': 0.35.4 + '@img/sharp-linux-riscv64': 0.35.4 + '@img/sharp-linux-s390x': 0.35.4 + '@img/sharp-linux-x64': 0.35.4 + '@img/sharp-linuxmusl-arm64': 0.35.4 + '@img/sharp-linuxmusl-x64': 0.35.4 + '@img/sharp-webcontainers-wasm32': 0.35.4 + '@img/sharp-win32-arm64': 0.35.4 + '@img/sharp-win32-ia32': 0.35.4 + '@img/sharp-win32-x64': 0.35.4 + '@types/node': 25.6.2 + siginfo@2.0.0: {} signal-exit@4.1.0: {} @@ -6118,6 +6910,8 @@ snapshots: style-mod@4.1.3: {} + supports-color@10.2.2: {} + symbol-tree@3.2.4: {} tagged-tag@1.0.0: {} @@ -6215,6 +7009,12 @@ snapshots: undici@7.27.2: {} + undici@7.29.1: {} + + unenv@2.0.0-rc.24: + dependencies: + pathe: 2.0.3 + update-browserslist-db@1.2.3(browserslist@4.28.2): dependencies: browserslist: 4.28.2 @@ -6295,6 +7095,32 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + workerd@1.20261001.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20261001.1 + '@cloudflare/workerd-darwin-arm64': 1.20261001.1 + '@cloudflare/workerd-linux-64': 1.20261001.1 + '@cloudflare/workerd-linux-arm64': 1.20261001.1 + '@cloudflare/workerd-windows-64': 1.20261001.1 + + wrangler@4.147.0(@cloudflare/workers-types@5.20261003.1)(@types/node@25.6.2): + dependencies: + '@cloudflare/kv-asset-handler': 0.5.0 + '@cloudflare/unenv-preset': 2.16.2(unenv@2.0.0-rc.24)(workerd@1.20261001.1) + blake3-wasm: 2.1.5 + esbuild: 0.28.1 + miniflare: 5.20261001.0-alpha(@types/node@25.6.2) + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.24 + workerd: 1.20261001.1 + optionalDependencies: + '@cloudflare/workers-types': 5.20261003.1 + fsevents: 2.3.3 + transitivePeerDependencies: + - '@types/node' + - bufferutil + - utf-8-validate + wrap-ansi@10.0.0: dependencies: ansi-styles: 6.2.3 @@ -6311,6 +7137,8 @@ snapshots: ws@8.20.0: {} + ws@8.21.0: {} + xml-name-validator@5.0.0: {} xml-naming@0.1.0: {} @@ -6322,6 +7150,19 @@ snapshots: yaml@2.9.0: optional: true + youch-core@0.3.3: + dependencies: + '@poppinss/exception': 1.2.3 + error-stack-parser-es: 1.0.5 + + youch@4.1.0-beta.10: + dependencies: + '@poppinss/colors': 4.1.6 + '@poppinss/dumper': 0.6.5 + '@speed-highlight/core': 1.2.24 + cookie: 1.1.1 + youch-core: 0.3.3 + zod-openapi@5.4.6(zod@4.4.3): dependencies: zod: 4.4.3 From 0126b2eb9ab8f8c70f80c20e0e21d890454becd6 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 17:07:54 -0400 Subject: [PATCH 009/178] v2 read path: collections, versions, records, manifest, diff, files Read endpoints with v1's shapes (docs/v1-read-api.md records what the UI and clients rely on): - version views decide privacy once (sets the caller may read); - records page by key or by offset in O(height), across both sets for owners (binary search on ranks) and across types, with a (type, id) cursor; no offset cap; - NDJSON streaming; manifests from nodes only, with ?since= deltas; diff via subtree-skipping tree diff, O(changes); file listings from the visible file trees; - collections list with SQL-side tag/owner filters and facets, detail, account pages and /api/context; - versions carry public file counts and bytes so non-owners never see private sizes. --- docs/v1-read-api.md | 199 ++++++++ packages/server/drizzle/0000_init.sql | 2 + .../server/drizzle/meta/0000_snapshot.json | 18 +- .../server/drizzle/meta/0001_snapshot.json | 20 +- packages/server/drizzle/meta/_journal.json | 4 +- packages/server/src/api/collections.ts | 371 ++++++++++++++ packages/server/src/api/versions.ts | 473 ++++++++++++++++++ packages/server/src/app.ts | 7 + packages/server/src/db/schema.ts | 3 + packages/server/src/ports.ts | 2 + packages/server/src/stores.ts | 1 + packages/server/src/versions/commit.ts | 2 + packages/server/src/versions/publish.ts | 4 + packages/server/src/versions/view.ts | 221 ++++++++ packages/server/test/read.test.ts | 217 ++++++++ packages/server/test/view.test.ts | 83 +++ 16 files changed, 1622 insertions(+), 5 deletions(-) create mode 100644 docs/v1-read-api.md create mode 100644 packages/server/src/api/collections.ts create mode 100644 packages/server/src/api/versions.ts create mode 100644 packages/server/src/versions/view.ts create mode 100644 packages/server/test/read.test.ts create mode 100644 packages/server/test/view.test.ts diff --git a/docs/v1-read-api.md b/docs/v1-read-api.md new file mode 100644 index 0000000..2806c02 --- /dev/null +++ b/docs/v1-read-api.md @@ -0,0 +1,199 @@ +# v1 read API: what v2 must match + +An inventory of v1's read endpoints (repo root `src/api/*`) and what the existing React UI +(`src/routes/**/*.data.ts`, `src/components/*`) actually reads, for the v2 read path. It was +taken from `main` at `7f6e1c6` on 2026-10-03. + +Within each section, "Response" is the shape and "UI reads" is what the UI uses. Where they +differ, only the UI fields are essential. + +## Cross-cutting + +- **Errors**: `{ error, statusCode }`. Uncaught errors give 500 `Internal server error`. +- **Missing vs hidden**: an invisible collection returns 404 `Collection not found`, the same as + a nonexistent one. +- **Auth, in order**: + - `Authorization: Bearer `: an invalid key is 401; + - `?token=` on GET/HEAD: invalid falls back to anonymous; + - the session cookie; + - anonymous for GETs. +- **Scope**: `permissions.collections` (admin > write > read). `metadata.collectionIds` limits a + key to those collections. +- **Owner access** means org membership, and a key's collection list (if any) covering the + collection. **Visible** means public or owner. +- **Rate limit**: a 60 s window. 60 requests per IP when anonymous, 5,000 per user. Sends the + `X-RateLimit-*` headers; over the limit, 429 with `Retry-After`. +- **Semver params** accept `1.0.0`, `v1.0.0` or `1`. They're stored as `v1.2.3`. +- **Share links**: + - A view link is a read key with `{scope:'read', collectionIds:[id], linkShare:true}`, valid 30 + days, used as `//?token=ul_…`. + - An agent link is a 1-hour write key at `/agent/`, which serves an HTML page. + - The UI builds API URLs with `apiUrlBuilder`, which forwards `?token=`. + +## SSR context + +`GET /api/context` uses the session cookie only and never returns 401. + +```ts +{ currentUser: null | { id, slug /*default org*/, displayName, avatarUrl, kfRole, + defaultOrg: {slug, displayName} | null, + orgs: [{ organizationId, slug, displayName, role, isDefault }] }, + mirrorConfig: { enabled, upstream, nodeName, syncSchedule }, + kfAccountUrl, kfAuthUrl } +``` + +- **UI reads**: + - `currentUser.id/slug/displayName/avatarUrl/kfRole/orgs[].slug/isDefault`; + - `isOwner` = `kfRole==='admin' || slug===owner || orgs.some(o => o.slug===owner)`. +- **SSR fetches** go to `http://127.0.0.1:${PORT}` (`fetchBase`). On Workers they must go + in-process (build doc finding 9). + +## Collections + +**`GET /api/collections`** +- **Query**: `q`, `owner`, `tag`, `sort` (`name`, `records`, `featured`, or default updatedAt), + `mine=true` (needs a session), `limit` (≤100, default 50), `offset`. +- **Response**: + ``` + { collections: [{ id, slug, name, public, ownerSlug, ownerName, createdAt, updatedAt, + description, tags, latestVersion /*semver*/, recordCount, fileCount, totalBytes, lastPushAt }], + facets: { owners: [{slug,name,count}], tags: [{name,count}] }, + featuredTags: string[], featuredCollections: [same item] } + ``` +- **UI reads** (explore and dashboard): `ownerSlug, slug, name, public, description, tags, + latestVersion, recordCount, totalBytes, lastPushAt, updatedAt`, plus the facets and featured + fields. +- **v1 bugs**: sorting, tag filters and facets run in memory over a window. The home page reads + `semver`, which doesn't exist. + +**`GET /api/collections/:owner/:slug`** +- **Response**: + ``` + { id, slug, name, public, ownerSlug, ownerName, createdAt, updatedAt, description, ark, + versionCount, latestVersion: null | { semver, hash, message, metadata, appId, pushedBy, + baseSemver, recordCount, fileCount, totalBytes, createdAt, typeCounts: [{type,count}] } } + ``` +- **UI reads**: + - collection fields: `public, id, ownerSlug, ownerName, description, versionCount, ark, name, + slug`; + - version fields: `latestVersion.semver`, `.metadata.readme/description/tags`, and the + overview's `semver, recordCount, fileCount, totalBytes, createdAt, typeCounts` (array or + object), `message, baseSemver, appId, pushedBy, hash`. +- Non-owners lose `pushedBy/actorId/signature` and private types in `typeCounts`. + +**`GET /api/accounts/:owner/collections`** +- **Response**: `[{ id, slug, name, public, createdAt, updatedAt }]`. Members also see private + collections; an unknown org gives `[]`. + +**`GET /api/collections/:owner/:slug/export?version=v1.2.3`** +- Returns `--.tar.gz` containing: + - `manifest.json`: `{collection:{owner,slug,name,description}, version:{semver,hash,message,recordCount,fileCount,totalBytes,createdAt}, schemas, files_missing}`; + - `records/.ndjson`; + - `files/`. + +## Versions + +**`GET .../versions?limit&offset`** +- **Response**: a bare array, newest first: + `[{ semver, hash, message, appId, actorId? (owner), recordCount, fileCount, totalBytes, createdAt, ark }]`. +- **UI reads**: `semver, message, recordCount, fileCount, totalBytes, createdAt, hash, ark`. The + version picker uses `?limit=20` and accepts an array or `{versions}`. + +**`GET .../versions/latest`, `GET .../versions/:n`** +- **Response**: + ``` + { semver, major, minor, patch, hash, baseSemver, message, metadata, pushedBy, appId, actorId, + recordCount, fileCount, typeCounts: {type: n}, totalBytes, createdAt, + schemas: { slug: JSONSchema }, ark } + ``` +- **UI reads**: `semver, schemas` (its keys, and `[t].properties` for table columns), + `recordCount, fileCount, totalBytes, createdAt, appId, hash, ark, message, metadata, + typeCounts, baseSemver, pushedBy`. + +**`GET .../versions/:n/records?type&limit(≤2000, default 100)&offset(≤10000)&after|cursor`** +- **Response**: + `{ records: [{ id, type, data, hash, ark? }], pagination: { limit, hasMore, nextCursor, total } }`. +- **Cursor**: `base64url(JSON {r:[recordId, recordHash]})`, or a bare id. +- **UI reads**: `records[].id/.data/.hash/.ark` and `pagination.total`. The UI pages by offset; + past offset 10k, v1 answers 400. + +**`GET .../versions/:n/records.ndjson?type&after`** +- Lines are `{id,type,data,hash}`. `X-Underlay-Record-Count` gives the total. Gzip is applied + when accepted. + +**`GET .../versions/:n/files`** +- **Response**: a bare array, `[{ hash, size, mimeType, createdAt, references: [{recordId,type,field}] }]`. +- **UI reads**: `hash, mimeType, size, references[].type/.recordId`. + +**`GET .../versions/:n/manifest?since&limit(≤100k)&cursor`** +- **Full**: + `{ semver, hash, schemas: {slug: schemaHash}, records: [{id,type,hash,private?}], files: string[], pagination }`. +- **Delta**: + `{ semver, hash, since, schemas, delta: {added, updated (+previousHash), removed}, files, pagination, truncated }`. +- Used by the CLI pull and mirror sync, not the UI. + +**`GET .../versions/:n/diff?from&limit(≤5000)&cursor`** +- **Response**: + `{ from, to, added: [{id,type,data}], updated: [{id,type,data}], removed: string[], pagination, meta: {schemaChanged, metadataChanged, filesAdded, filesRemoved} }`. +- **UI reads**: `added, updated, removed, meta.*`. It also reads `meta.readmeChanged`, which v1 + never returned. + +## Schemas + +- **`GET /api/schemas?q|label|slug|schema_hash&limit&offset`** + - Returns `[{ id, schema, schemaHash, createdAt, labels: string[] }]`. With `schema_hash` it + returns one object plus `usageCount`. + - A schema is visible when it's non-private in a public collection, or in one of the caller's + orgs. +- **`GET /api/schemas/:id`** + - Returns + `{ id, schema, schemaHash, createdAt, labels: [{label, createdAt}], usage: [{slug, semver, collection: "owner/slug"}] }`. +- **`GET /api/collections/:owner/:slug/schemas?version&raw`** + - Returns + `{ version, semver, schemas: [{ slug, schemaId, schemaHash, schema (+ 'x-underlay-labels') }] }`. + +## Records, files, accounts + +- **`GET /api/records/:hash/provenance`** + - Returns + `{ hash, recordId, type, data, size, createdAt, firstSeen, references: [{owner, collection, collectionName, semver, versionCreatedAt}] }`. + - References are public collections only. +- **`POST /api/records/batch`**: `{hashes}` in, NDJSON out. +- **Files**: + - `HEAD|GET /api/collections/:owner/:slug/files/:hash`: GET is a 302 to a presigned URL with + `attachment`. + - `GET /api/collections/files/:hash`: the same, for any collection the caller can read. + - `POST .../files/presign {hashes}` returns `{hash: url|null}`. +- **Accounts**: + - `GET /api/accounts/:slug` returns the org row, plus `displayName` and `arkShoulder`. + - `GET /api/accounts/:slug/members` returns `[{role, slug, displayName}]`. + - `GET /api/accounts/me` returns + `{id, name, email, image, slug, displayName, createdAt, orgs:[{organizationId, role, slug, name, isDefault}]}`. +- **ARK**: + - `GET /api/ark/resolve?path=ark:…` returns `{type:'redirect', url, metadata}`, or 404 + `{type:'not_found'}`. + - `GET .../ark` returns `{enabled, customUrl, arkUrl, shoulder, arkId}`. + - `GET .../ark/record-types` returns `[{recordType, redirectUrlField}]`. +- **Webhooks**: + - `GET .../webhooks` returns + `{webhooks:[{id,url,bumpFilter,enabled,createdAt,lastDeliveryAt}]}`. + - `GET .../webhooks/:id/deliveries` returns `{deliveries:[…]}`. +- **Health**: `GET /api/health` returns `{status:'ok', timestamp}`. +- **KF summary**: `GET /api/kf/summary?kf_org_id`, with the KF internal key, returns per-org + collection stats. + +## v1 behaviour v2 deliberately changes or fixes + +- **Records order**: + - With `?type=`, records are in id order. + - Without it, they're in (type, id) order, and the cursor becomes (type, id) (edge-redesign.md + Read path). + - Offsets have no 10k cap: they cost O(height) in v2. +- **Non-owner hashes and counts**: + - Non-owners get the public set's real contents, and its counts are exact, not upper bounds. + - A version's `hash` is the v2 version hash for everyone. There's no separate public hash. +- **Field-level privacy is gone**, so nothing is stripped from records. +- **Manifest `files` and version `/files`** come from the set's file tree. They're + privacy-correct and paginated. +- **Inconsistent v1 fields** (`typeCounts` array vs object, `labels` list vs objects) are kept + where the UI depends on them. diff --git a/packages/server/drizzle/0000_init.sql b/packages/server/drizzle/0000_init.sql index b8bb218..c404185 100644 --- a/packages/server/drizzle/0000_init.sql +++ b/packages/server/drizzle/0000_init.sql @@ -389,6 +389,8 @@ CREATE TABLE `versions` ( `public_record_count` integer NOT NULL, `file_count` integer NOT NULL, `total_bytes` integer NOT NULL, + `public_file_count` integer DEFAULT 0 NOT NULL, + `public_total_bytes` integer DEFAULT 0 NOT NULL, `type_counts` text NOT NULL, `public_type_counts` text NOT NULL, `has_private` integer DEFAULT false NOT NULL, diff --git a/packages/server/drizzle/meta/0000_snapshot.json b/packages/server/drizzle/meta/0000_snapshot.json index 5fbdbb5..fad99ad 100644 --- a/packages/server/drizzle/meta/0000_snapshot.json +++ b/packages/server/drizzle/meta/0000_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "fb1d3f77-7095-4ca8-8fe8-45e5eb96f631", + "id": "546a7fbc-8404-4b9e-a3d6-af1a1f02737a", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { "account": { @@ -2495,6 +2495,22 @@ "notNull": true, "autoincrement": false }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, "type_counts": { "name": "type_counts", "type": "text", diff --git a/packages/server/drizzle/meta/0001_snapshot.json b/packages/server/drizzle/meta/0001_snapshot.json index 1e88ce5..61b93db 100644 --- a/packages/server/drizzle/meta/0001_snapshot.json +++ b/packages/server/drizzle/meta/0001_snapshot.json @@ -1,6 +1,6 @@ { - "id": "153626ea-7e2f-4690-a012-0514c340dcdb", - "prevId": "fb1d3f77-7095-4ca8-8fe8-45e5eb96f631", + "id": "89c64381-abbc-4f02-8032-90d5d70523a9", + "prevId": "546a7fbc-8404-4b9e-a3d6-af1a1f02737a", "version": "6", "dialect": "sqlite", "tables": { @@ -2495,6 +2495,22 @@ "notNull": true, "autoincrement": false }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, "type_counts": { "name": "type_counts", "type": "text", diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index bc977ad..0cc11d7 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -5,14 +5,14 @@ { "idx": 0, "version": "6", - "when": 1791059939117, + "when": 1791061421563, "tag": "0000_init", "breakpoints": true }, { "idx": 1, "version": "6", - "when": 1791059939881, + "when": 1791061422322, "tag": "0001_platform_location", "breakpoints": true } diff --git a/packages/server/src/api/collections.ts b/packages/server/src/api/collections.ts new file mode 100644 index 0000000..0052f95 --- /dev/null +++ b/packages/server/src/api/collections.ts @@ -0,0 +1,371 @@ +/** + * Collection, account and app-context reads (v1 shapes: docs/v1-read-api.md). + * + * GET /api/context + * GET /api/collections list and explore + * GET /api/collections/:owner/:slug detail + * GET /api/accounts/:owner/collections + * GET /api/accounts/:slug, /api/accounts/:slug/members + * + * Everything here is SQLite rows: no repository reads. Lists show what a + * caller may see: public counts to non-members, full counts to members. + */ +import { and, asc, count, desc, eq, inArray, like, or, sql } from 'drizzle-orm' +import { Hono } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import type { Db } from '../ports.js' +import { jsonError, requireCollection } from './access.js' + +type VersionRow = typeof schema.versions.$inferSelect + +/** Org ids the principal belongs to (empty for anonymous or collection-scoped keys). */ +async function memberOrgIds( + db: Db, + userId: string | undefined, + scoped: boolean, +): Promise { + if (!userId || scoped) return [] + const rows = await db + .select({ id: schema.member.organizationId }) + .from(schema.member) + .where(eq(schema.member.userId, userId)) + return rows.map((r) => r.id) +} + +/** A version as a caller sees it in lists and the collection detail. */ +export function versionSummary(v: VersionRow, owner: boolean) { + return { + semver: v.semver, + major: v.major, + minor: v.minor, + patch: v.patch, + hash: v.hash, + baseSemver: v.baseSemver, + message: v.message, + appId: v.appId, + ...(owner ? { pushedBy: v.pushedBy, actorId: v.actorId } : {}), + recordCount: owner ? v.recordCount : v.publicRecordCount, + fileCount: owner ? v.fileCount : v.publicFileCount, + totalBytes: owner ? v.totalBytes : v.publicTotalBytes, + typeCounts: owner ? v.typeCounts : v.publicTypeCounts, + createdAt: v.createdAt, + ark: null as string | null, + } +} + +export function collectionRoutes() { + const app = new Hono() + + app.get('/api/context', async (c) => { + const { db } = c.var.ports + const p = c.var.principal + let currentUser = null + if (p && p.scope === 'session') { + const [u] = await db.select().from(schema.user).where(eq(schema.user.id, p.userId)).limit(1) + if (u) { + const orgs = await db + .select({ org: schema.organization, role: schema.member.role }) + .from(schema.member) + .innerJoin(schema.organization, eq(schema.organization.id, schema.member.organizationId)) + .where(eq(schema.member.userId, u.id)) + const def = orgs.find((o) => o.org.isDefault) ?? null + currentUser = { + id: u.id, + slug: def?.org.slug ?? null, + displayName: u.name ?? def?.org.name ?? null, + avatarUrl: u.image ?? null, + kfRole: null, + defaultOrg: def ? { slug: def.org.slug, displayName: def.org.name } : null, + orgs: orgs.map((o) => ({ + organizationId: o.org.id, + slug: o.org.slug, + displayName: o.org.name, + role: o.role, + isDefault: o.org.isDefault, + })), + } + } + } + return c.json({ + currentUser, + mirrorConfig: { enabled: false, upstream: '', nodeName: '', syncSchedule: '' }, + kfAccountUrl: c.var.config.kfAccountUrl ?? '', + kfAuthUrl: c.var.config.kfAuthUrl ?? '', + }) + }) + + app.get('/api/collections', async (c) => { + const { db } = c.var.ports + const p = c.var.principal + const q = c.req.query('q') + const owner = c.req.query('owner') + const tag = c.req.query('tag') + const sort = c.req.query('sort') + const mine = c.req.query('mine') === 'true' + const limit = Math.min(100, Math.max(1, Number(c.req.query('limit') ?? 50) || 50)) + const offset = Math.max(0, Number(c.req.query('offset') ?? 0) || 0) + const scoped = !!p?.collectionIds + if (mine && (!p || scoped)) + return jsonError(c, 401, 'Unauthorized — mine=true requires a session') + const orgIds = await memberOrgIds(db, p?.userId, scoped) + + const visible = mine + ? orgIds.length > 0 + ? inArray(schema.collections.organizationId, orgIds) + : sql`0` + : eq(schema.collections.public, true) + const conds = [visible, sql`${schema.collections.deletedAt} IS NULL`] + if (q) conds.push(like(schema.collections.name, `%${q}%`)) + if (owner) conds.push(eq(schema.organization.slug, owner)) + if (tag) { + conds.push( + sql`EXISTS (SELECT 1 FROM json_each(${schema.collections.summary}, '$.tags') WHERE value = ${tag})`, + ) + } + const where = and(...conds) + const order = + sort === 'name' + ? [asc(schema.collections.name)] + : sort === 'records' + ? [desc(schema.versions.publicRecordCount)] + : [desc(schema.collections.updatedAt)] + + const rows = await db + .select({ c: schema.collections, owner: schema.organization, v: schema.versions }) + .from(schema.collections) + .innerJoin(schema.organization, eq(schema.organization.id, schema.collections.organizationId)) + .leftJoin(schema.versions, eq(schema.versions.id, schema.collections.headVersionId)) + .where(where) + .orderBy(...order) + .limit(limit) + .offset(offset) + + const item = (r: (typeof rows)[number]) => { + const isOwner = orgIds.includes(r.c.organizationId) + const v = r.v ? versionSummary(r.v, isOwner) : null + return { + id: r.c.id, + slug: r.c.slug, + name: r.c.name, + public: r.c.public, + ownerSlug: r.owner.slug, + ownerName: r.owner.name, + createdAt: r.c.createdAt, + updatedAt: r.c.updatedAt, + description: r.c.summary?.description ?? null, + tags: r.c.summary?.tags ?? [], + latestVersion: v?.semver ?? null, + recordCount: v?.recordCount ?? null, + fileCount: v?.fileCount ?? null, + totalBytes: v?.totalBytes ?? null, + lastPushAt: v?.createdAt ?? null, + } + } + + // Facets over everything visible (not just this page). + const ownerFacets = await db + .select({ slug: schema.organization.slug, name: schema.organization.name, count: count() }) + .from(schema.collections) + .innerJoin(schema.organization, eq(schema.organization.id, schema.collections.organizationId)) + .where(and(visible, sql`${schema.collections.deletedAt} IS NULL`)) + .groupBy(schema.organization.id) + .orderBy(desc(count())) + .limit(50) + const tagFacets = (await db.all(sql` + SELECT t.value AS name, count(*) AS count + FROM ${schema.collections} c, json_each(c.summary, '$.tags') t + WHERE ${mine ? (orgIds.length ? inArray(sql`c.organization_id`, orgIds) : sql`0`) : sql`c.public = 1`} + AND c.deleted_at IS NULL + GROUP BY t.value ORDER BY count DESC LIMIT 50 + `)) as { name: string; count: number }[] + + const settings = await db + .select() + .from(schema.instanceSettings) + .where( + inArray(schema.instanceSettings.key, [ + 'explore_featured_tags', + 'explore_featured_collections', + ]), + ) + const setting = (k: string) => settings.find((s) => s.key === k)?.value + const featuredTags = Array.isArray(setting('explore_featured_tags')) + ? (setting('explore_featured_tags') as string[]) + : [] + const featuredRefs = Array.isArray(setting('explore_featured_collections')) + ? (setting('explore_featured_collections') as string[]) + : [] + let featuredCollections: ReturnType[] = [] + if (featuredRefs.length > 0) { + const pairs = featuredRefs.map((r) => r.split('/')).filter((x) => x.length === 2) + const featured = await db + .select({ c: schema.collections, owner: schema.organization, v: schema.versions }) + .from(schema.collections) + .innerJoin( + schema.organization, + eq(schema.organization.id, schema.collections.organizationId), + ) + .leftJoin(schema.versions, eq(schema.versions.id, schema.collections.headVersionId)) + .where( + and( + eq(schema.collections.public, true), + or( + ...pairs.map(([o, s]) => + and(eq(schema.organization.slug, o!), eq(schema.collections.slug, s!)), + ), + ), + ), + ) + featuredCollections = featuredRefs + .map((ref) => featured.find((f) => `${f.owner.slug}/${f.c.slug}` === ref)) + .filter((f): f is (typeof featured)[number] => !!f) + .map(item) + } + + return c.json({ + collections: rows.map(item), + facets: { owners: ownerFacets, tags: tagFacets }, + featuredTags, + featuredCollections, + }) + }) + + app.get('/api/collections/:owner/:slug', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const { db } = c.var.ports + const col = access.collection + const [versionCount] = await db + .select({ n: count() }) + .from(schema.versions) + .where(eq(schema.versions.collectionId, col.id)) + let latestVersion = null + if (col.headVersionId) { + const [v] = await db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, col.headVersionId)) + .limit(1) + if (v) { + const repo = await c.var.ports.stores.forCollection(col.id) + const root = await repo.root(v.hash) + const s = versionSummary(v, access.isMember) + latestVersion = { + ...s, + metadata: root.metadata, + typeCounts: Object.entries(s.typeCounts).map(([type, n]) => ({ type, count: n })), + } + } + } + return c.json({ + id: col.id, + slug: col.slug, + name: col.name, + public: col.public, + ownerSlug: access.owner.slug, + ownerName: access.owner.name, + createdAt: col.createdAt, + updatedAt: col.updatedAt, + description: col.summary?.description ?? null, + ark: null, + versionCount: versionCount?.n ?? 0, + latestVersion, + }) + }) + + app.get('/api/accounts/:owner/collections', async (c) => { + const { db } = c.var.ports + const p = c.var.principal + const [org] = await db + .select() + .from(schema.organization) + .where(eq(schema.organization.slug, c.req.param('owner'))) + .limit(1) + if (!org) return c.json([]) + const orgIds = await memberOrgIds(db, p?.userId, !!p?.collectionIds) + const member = orgIds.includes(org.id) + const rows = await db + .select({ + id: schema.collections.id, + slug: schema.collections.slug, + name: schema.collections.name, + public: schema.collections.public, + createdAt: schema.collections.createdAt, + updatedAt: schema.collections.updatedAt, + }) + .from(schema.collections) + .where( + and( + eq(schema.collections.organizationId, org.id), + sql`${schema.collections.deletedAt} IS NULL`, + member ? undefined : eq(schema.collections.public, true), + ), + ) + .orderBy(desc(schema.collections.updatedAt)) + return c.json(rows) + }) + + app.get('/api/accounts/:slug', async (c) => { + const { db } = c.var.ports + const [org] = await db + .select() + .from(schema.organization) + .where(eq(schema.organization.slug, c.req.param('slug'))) + .limit(1) + if (!org) return jsonError(c, 404, 'Not found') + // Public profile only (v1 returned the whole row, including kfOrgId). + return c.json({ + id: org.id, + name: org.name, + slug: org.slug, + displayName: org.name, + avatarUrl: org.avatarUrl, + logo: org.logo, + bio: org.bio, + website: org.website, + createdAt: org.createdAt, + isDefault: org.isDefault, + arkNaan: org.arkNaan, + arkShoulder: null, + }) + }) + + app.get('/api/accounts/:slug/members', async (c) => { + const { db } = c.var.ports + const [org] = await db + .select() + .from(schema.organization) + .where(eq(schema.organization.slug, c.req.param('slug'))) + .limit(1) + if (!org) return jsonError(c, 404, 'Not found') + const members = await db + .select({ role: schema.member.role, userId: schema.member.userId, name: schema.user.name }) + .from(schema.member) + .innerJoin(schema.user, eq(schema.user.id, schema.member.userId)) + .where(eq(schema.member.organizationId, org.id)) + const defaults = await db + .select({ userId: schema.member.userId, slug: schema.organization.slug }) + .from(schema.member) + .innerJoin(schema.organization, eq(schema.organization.id, schema.member.organizationId)) + .where( + and( + inArray( + schema.member.userId, + members.map((m) => m.userId), + ), + eq(schema.organization.isDefault, true), + ), + ) + return c.json( + members.map((m) => ({ + role: m.role, + slug: defaults.find((d) => d.userId === m.userId)?.slug ?? null, + displayName: m.name, + })), + ) + }) + + return app +} diff --git a/packages/server/src/api/versions.ts b/packages/server/src/api/versions.ts new file mode 100644 index 0000000..e33308e --- /dev/null +++ b/packages/server/src/api/versions.ts @@ -0,0 +1,473 @@ +/** + * Version reads (v1 shapes: docs/v1-read-api.md), mounted at /api/collections. + * + * GET /:owner/:slug/versions list (SQLite) + * GET /:owner/:slug/versions/:n detail (+ root metadata, visible schemas) + * GET /:owner/:slug/versions/:n/records page by key or offset; O(height) seeks + * GET /:owner/:slug/versions/:n/records.ndjson stream + * GET /:owner/:slug/versions/:n/manifest ids and hashes from nodes; ?since= delta + * GET /:owner/:slug/versions/:n/diff ?from=; O(changes) + * GET /:owner/:slug/versions/:n/files from the visible file trees + * + * Privacy is decided once: the view holds the sets the caller may read. + * Records list order: by id within a type; without ?type=, types in slug order + * then id, with a (type, id) cursor (a deliberate change from v1). + */ +import { + compareUtf8, + diffTrees, + fileTree, + iterate, + type RecordEntry, + recordTree, +} from '@underlay/core' +import { RepoSource } from '@underlay/repo' +import { desc, eq, inArray } from 'drizzle-orm' +import { type Context, Hono } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import { + findVersion, + getRecord, + loadView, + type TypeView, + typeRecords, + type VersionView, +} from '../versions/view.js' +import { type CollectionAccess, jsonError, requireCollection } from './access.js' +import { versionSummary } from './collections.js' + +const encodeCursor = (t: string, k: string) => + Buffer.from(JSON.stringify({ t, k })).toString('base64url') +function decodeCursor( + s: string | undefined, + type: string | undefined, +): { t: string; k: string } | null { + if (!s) return null + try { + const v = JSON.parse(Buffer.from(s, 'base64url').toString()) as { + t?: unknown + k?: unknown + r?: unknown + } + if (typeof v.t === 'string' && typeof v.k === 'string') return { t: v.t, k: v.k } + // v1 cursors: {r: [recordId, recordHash]} within a type. + if (Array.isArray(v.r) && typeof v.r[0] === 'string' && type) return { t: type, k: v.r[0] } + } catch { + // Not a cursor: a bare record id within ?type= (v1 accepted that). + } + return type ? { t: type, k: s } : null +} + +const clamp = (v: string | undefined, def: number, max: number) => + Math.min(max, Math.max(1, Number(v ?? def) || def)) + +async function viewFor( + c: Context, + access: CollectionAccess, +): Promise { + const n = c.req.param('n') ?? 'latest' + const v = await findVersion( + c.var.ports.db, + access.collection.id, + n, + access.collection.headVersionId, + ) + if (!v) return jsonError(c, 404, n === 'latest' ? 'No versions' : 'Version not found') + const repo = await c.var.ports.stores.forCollection(access.collection.id) + return loadView(repo, v, access.isMember) +} + +/** Add the record hash to a canonical record line: `{"id",…,"data":…,"hash":"…"}`. */ +const withHash = (body: string, hash: string) => `${body.slice(0, -1)},"hash":"${hash}"}` + +function recordJson(e: RecordEntry & { type: string }) { + const r = JSON.parse(e.body!) as { id: string; type: string; data: unknown } + return { id: r.id, type: r.type, data: r.data, hash: e.hash } +} + +/** Records across visible types from a (type, id) position, in (type, id) order. */ +async function* allRecords( + view: VersionView, + from: { t: string; k: string } | null, + offset: number, + bodies: boolean, +) { + let skip = offset + for (const t of view.types) { + if (from && compareUtf8(t.slug, from.t) < 0) continue + if (skip >= t.count) { + skip -= t.count + continue + } + const opts: { after?: string; offset?: number; bodies: boolean } = { bodies } + if (from && t.slug === from.t) opts.after = from.k + else if (skip > 0) opts.offset = skip + skip = 0 + yield* typeRecords(view, t, opts) + } +} + +export function versionRoutes() { + const app = new Hono() + + app.get('/:owner/:slug/versions', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const limit = clamp(c.req.query('limit'), 50, 100) + const offset = Math.max(0, Number(c.req.query('offset') ?? 0) || 0) + const rows = await c.var.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.collectionId, access.collection.id)) + .orderBy(desc(schema.versions.seq)) + .limit(limit) + .offset(offset) + return c.json(rows.map((v) => versionSummary(v, access.isMember))) + }) + + app.get('/:owner/:slug/versions/:n', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const view = await viewFor(c, access) + if (view instanceof Response) return view + const schemas: Record = {} + await Promise.all( + view.types.map(async (t) => (schemas[t.slug] = await view.repo.schema(t.schemaHash))), + ) + return c.json({ + ...versionSummary(view.version, view.owner), + metadata: view.root.metadata, + typeCounts: Object.fromEntries(view.types.map((t) => [t.slug, t.count])), + schemas, + }) + }) + + app.get('/:owner/:slug/versions/:n/records', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const view = await viewFor(c, access) + if (view instanceof Response) return view + const type = c.req.query('type') + const limit = clamp(c.req.query('limit'), 100, 2000) + const cursor = decodeCursor(c.req.query('cursor') ?? c.req.query('after'), type) + const offset = cursor ? 0 : Math.max(0, Number(c.req.query('offset') ?? 0) || 0) + let source: AsyncIterable + let total: number + if (type) { + const t = view.types.find((x) => x.slug === type) + if (!t) + return c.json({ + records: [], + pagination: { limit, hasMore: false, nextCursor: null, total: 0 }, + }) + total = t.count + const opts: { after?: string; offset?: number; bodies: boolean } = { bodies: true } + if (cursor) opts.after = cursor.k + else opts.offset = offset + source = typeRecords(view, t, opts) + } else { + total = view.types.reduce((n, t) => n + t.count, 0) + source = allRecords(view, cursor, offset, true) + } + const page: (RecordEntry & { type: string })[] = [] + let hasMore = false + for await (const e of source) { + if (page.length === limit) { + hasMore = true + break + } + page.push(e) + } + const last = page[page.length - 1] + return c.json({ + records: page.map(recordJson), + pagination: { + limit, + hasMore, + nextCursor: hasMore && last ? encodeCursor(last.type, last.key) : null, + total, + }, + }) + }) + + app.get('/:owner/:slug/versions/:n/records.ndjson', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const view = await viewFor(c, access) + if (view instanceof Response) return view + const type = c.req.query('type') + const after = c.req.query('after') + const types = type ? view.types.filter((t) => t.slug === type) : view.types + const count = types.reduce((n, t) => n + t.count, 0) + const enc = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + try { + for (const t of types) { + const opts = after && type ? { after, bodies: true } : { bodies: true } + let batch: string[] = [] + for await (const e of typeRecords(view, t, opts)) { + batch.push(withHash(e.body!, e.hash)) + if (batch.length >= 512) { + controller.enqueue(enc.encode(batch.join('\n') + '\n')) + batch = [] + } + } + if (batch.length) controller.enqueue(enc.encode(batch.join('\n') + '\n')) + } + controller.close() + } catch (err) { + controller.error(err) + } + }, + }) + return new Response(stream, { + headers: { 'content-type': 'application/x-ndjson', 'x-underlay-record-count': String(count) }, + }) + }) + + app.get('/:owner/:slug/versions/:n/manifest', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const view = await viewFor(c, access) + if (view instanceof Response) return view + const limit = clamp(c.req.query('limit'), 10_000, 100_000) + const cursor = decodeCursor(c.req.query('cursor'), undefined) + const schemas = Object.fromEntries(view.types.map((t) => [t.slug, t.schemaHash])) + const since = c.req.query('since') + + if (since) { + const from = await findVersion(c.var.ports.db, access.collection.id, since, null) + if (!from) return jsonError(c, 404, `Version ${since} not found`) + const fromView = await loadView(view.repo, from, view.owner) + const delta = { added: [] as object[], updated: [] as object[], removed: [] as object[] } + let n = 0 + let next: string | null = null + let last: { type: string; key: string } | null = null + for await (const d of diffAll(fromView, view, cursor)) { + // Resuming skips up to and including the cursor: it names the last entry returned. + if (n === limit) { + next = encodeCursor(last!.type, last!.key) + break + } + n++ + last = d + if (!d.before) delta.added.push({ id: d.key, type: d.type, hash: d.after!.hash }) + else if (!d.after) delta.removed.push({ id: d.key, type: d.type, hash: d.before.hash }) + else + delta.updated.push({ + id: d.key, + type: d.type, + hash: d.after.hash, + previousHash: d.before.hash, + }) + } + return c.json({ + semver: view.version.semver, + hash: view.version.hash, + since: from.semver, + schemas, + delta, + files: await visibleFiles(view, 100_000), + pagination: { limit, hasMore: next !== null, nextCursor: next }, + truncated: next !== null, + }) + } + + const records: object[] = [] + let next: string | null = null + // Nodes only: ids and hashes, no bodies. + // Resuming starts after the cursor, so it names the last entry returned. + for await (const e of allRecords(view, cursor, 0, false)) { + if (records.length === limit) { + const last = records[records.length - 1] as { id: string; type: string } + next = encodeCursor(last.type, last.id) + break + } + records.push({ + id: e.key, + type: e.type, + hash: e.hash, + ...(e.set === 'private' ? { private: true } : {}), + }) + } + return c.json({ + semver: view.version.semver, + hash: view.version.hash, + schemas, + records, + files: cursor ? [] : await visibleFiles(view, 100_000), + pagination: { limit, hasMore: next !== null, nextCursor: next }, + }) + }) + + app.get('/:owner/:slug/versions/:n/diff', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const view = await viewFor(c, access) + if (view instanceof Response) return view + const limit = clamp(c.req.query('limit'), 500, 5000) + const cursor = decodeCursor(c.req.query('cursor'), undefined) + const fromParam = c.req.query('from') + const from = fromParam + ? await findVersion(c.var.ports.db, access.collection.id, fromParam, null) + : null + if (fromParam && !from) return jsonError(c, 404, `Version ${fromParam} not found`) + const fromView = from ? await loadView(view.repo, from, view.owner) : null + const added: object[] = [] + const updated: object[] = [] + const removed: string[] = [] + let n = 0 + let next: string | null = null + let last: { type: string; key: string } | null = null + const body = async (side: VersionView, d: { type: string; key: string }) => { + const t = side.types.find((x) => x.slug === d.type)! + const r = await getRecord(side, t, d.key) + return { id: d.key, type: d.type, data: (JSON.parse(r!.body!) as { data: unknown }).data } + } + for await (const d of diffAll(fromView, view, cursor)) { + if (n === limit) { + next = encodeCursor(last!.type, last!.key) + break + } + n++ + last = d + if (!d.before) added.push(await body(view, d)) + else if (!d.after) removed.push(d.key) + else updated.push(await body(view, d)) + } + const schemaChanged = + !fromView || + JSON.stringify(fromView.types.map((t) => [t.slug, t.schemaHash])) !== + JSON.stringify(view.types.map((t) => [t.slug, t.schemaHash])) + const fileDelta = fromView ? await fileCounts(fromView, view) : { added: 0, removed: 0 } + return c.json({ + from: from?.semver ?? null, + to: view.version.semver, + added, + updated, + removed, + pagination: { limit, hasMore: next !== null, nextCursor: next }, + meta: { + schemaChanged, + metadataChanged: + JSON.stringify(fromView?.root.metadata ?? null) !== JSON.stringify(view.root.metadata), + filesAdded: fileDelta.added, + filesRemoved: fileDelta.removed, + }, + }) + }) + + app.get('/:owner/:slug/versions/:n/files', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const view = await viewFor(c, access) + if (view instanceof Response) return view + const hashes = await visibleFiles(view, 10_000) + const rows = hashes.length + ? await c.var.ports.db.select().from(schema.files).where(inHashes(hashes)) + : [] + const byHash = new Map(rows.map((r) => [r.hash, r])) + // References aren't indexed per file in v2; the listing returns none. + return c.json( + hashes.map((h) => ({ + hash: h, + size: byHash.get(h)?.size ?? null, + mimeType: byHash.get(h)?.mimeType ?? null, + createdAt: byHash.get(h)?.createdAt ?? null, + references: [], + })), + ) + }) + + return app +} + +const inHashes = (hashes: string[]) => inArray(schema.files.hash, hashes) + +/** File hashes in the sets this view may read (deduplicated, sorted). */ +async function visibleFiles(view: VersionView, max: number): Promise { + const source = new RepoSource(fileTree, view.repo) + const roots = [view.public.files.root, view.private?.files.root ?? null] + const out = new Set() + for (const r of roots) { + for await (const f of iterate(source, r)) { + out.add(f.key) + if (out.size >= max) break + } + } + return [...out].sort() +} + +async function fileCounts(a: VersionView, b: VersionView) { + const source = new RepoSource(fileTree, b.repo) + let added = 0 + let removed = 0 + for (const [x, y] of [ + [a.public.files.root, b.public.files.root], + [a.private?.files.root ?? null, b.private?.files.root ?? null], + ] as const) { + for await (const d of diffTrees(source, x, y)) { + if (!d.before) added++ + else if (!d.after) removed++ + } + } + return { added, removed } +} + +type TypedDiff = { + type: string + key: string + before: RecordEntry | null + after: RecordEntry | null +} + +/** + * Differences between two views, by (type, id), resuming after a cursor. A + * record moving between sets with the same hash isn't a change to a reader who + * sees both; to a public reader it appears or disappears. + */ +async function* diffAll( + from: VersionView | null, + to: VersionView, + cursor: { t: string; k: string } | null, +): AsyncGenerator { + const source = new RepoSource(recordTree, to.repo) + const slugs = new Set([...(from?.types.map((t) => t.slug) ?? []), ...to.types.map((t) => t.slug)]) + for (const slug of [...slugs].sort(compareUtf8)) { + if (cursor && compareUtf8(slug, cursor.t) < 0) continue + const a = from?.types.find((t) => t.slug === slug) + const b = to.types.find((t) => t.slug === slug) + const changes = new Map() + const collect = async ( + x: TypeView | undefined, + y: TypeView | undefined, + set: 'public' | 'private', + ) => { + const xr = x?.[set]?.root ?? null + const yr = y?.[set]?.root ?? null + if (set === 'private' && !to.owner) return + for await (const d of diffTrees(source, xr, yr)) { + const prev = changes.get(d.key) + if (prev) { + // The same id changed in both sets: a move. Combine the halves. + const before = prev.before ?? d.before + const after = prev.after ?? d.after + if (before && after && before.hash === after.hash) changes.delete(d.key) + else changes.set(d.key, { type: slug, key: d.key, before, after }) + } else { + changes.set(d.key, { type: slug, key: d.key, before: d.before, after: d.after }) + } + } + } + await collect(a, b, 'public') + await collect(a, b, 'private') + const keys = [...changes.keys()].sort(compareUtf8) + for (const k of keys) { + if (cursor && slug === cursor.t && compareUtf8(k, cursor.k) <= 0) continue + yield changes.get(k)! + } + } +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index de49275..eb59216 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -6,7 +6,9 @@ import { type Context, Hono } from 'hono' import type { Principal } from './api/access.js' +import { collectionRoutes } from './api/collections.js' import { pushRoutes } from './api/push.js' +import { versionRoutes } from './api/versions.js' import type { Ports } from './ports.js' export interface AppConfig { @@ -14,6 +16,9 @@ export interface AppConfig { appUrl: string /** "staging", "next", "production" or "dev": shown in /api/health. */ deployment: string + /** KF Auth URLs the UI links to (account settings, sign-out). */ + kfAuthUrl?: string + kfAccountUrl?: string } export type Authenticate = (req: Request, ports: Ports) => Promise @@ -67,6 +72,8 @@ export function createApp(setup: Setup) { ) app.route('/api/collections', pushRoutes()) + app.route('/api/collections', versionRoutes()) + app.route('/', collectionRoutes()) app.notFound((c) => c.json({ error: 'Not found', statusCode: 404 }, 404)) app.onError((err, c) => { diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index e3ab113..5fc43f9 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -254,6 +254,9 @@ export const versions = sqliteTable( publicRecordCount: integer('public_record_count').notNull(), fileCount: integer('file_count').notNull(), totalBytes: integer('total_bytes').notNull(), + /** What non-owners see: the public set's file count and record + file bytes. */ + publicFileCount: integer('public_file_count').notNull().default(0), + publicTotalBytes: integer('public_total_bytes').notNull().default(0), typeCounts: json>('type_counts').notNull(), publicTypeCounts: json>('public_type_counts').notNull(), hasPrivate: bool('has_private').notNull().default(false), diff --git a/packages/server/src/ports.ts b/packages/server/src/ports.ts index 2407955..8551212 100644 --- a/packages/server/src/ports.ts +++ b/packages/server/src/ports.ts @@ -47,6 +47,8 @@ export interface Stores { * mirrored: push sessions, staging uploads, the reference log. */ internal: BlobStore + /** File bytes, by the `files.storage_key` (v1 keys are relative to the bucket root). */ + fileBytes: BlobStore } export interface Ports { diff --git a/packages/server/src/stores.ts b/packages/server/src/stores.ts index 6b511e0..3781c5a 100644 --- a/packages/server/src/stores.ts +++ b/packages/server/src/stores.ts @@ -70,6 +70,7 @@ export function createStores(db: Db, cache: Cache, platform: PlatformStorage): S return forLocation(row.locationId) }, internal: new PrefixedBlobStore(platform.bucket, platform.internalPrefix), + fileBytes: platform.bucket, } } diff --git a/packages/server/src/versions/commit.ts b/packages/server/src/versions/commit.ts index fca41db..9489d0d 100644 --- a/packages/server/src/versions/commit.ts +++ b/packages/server/src/versions/commit.ts @@ -389,6 +389,8 @@ export async function commitVersion(ports: Ports, input: CommitInput): Promise publicTypeCounts: Record hasPrivate: boolean @@ -106,6 +108,8 @@ export async function publishVersion( publicRecordCount: lit(v.publicRecordCount).as('public_record_count'), fileCount: lit(v.fileCount).as('file_count'), totalBytes: lit(v.totalBytes).as('total_bytes'), + publicFileCount: lit(v.publicFileCount).as('public_file_count'), + publicTotalBytes: lit(v.publicTotalBytes).as('public_total_bytes'), typeCounts: lit(JSON.stringify(v.typeCounts)).as('type_counts'), publicTypeCounts: lit(JSON.stringify(v.publicTypeCounts)).as( 'public_type_counts', diff --git a/packages/server/src/versions/view.ts b/packages/server/src/versions/view.ts new file mode 100644 index 0000000..ed9160d --- /dev/null +++ b/packages/server/src/versions/view.ts @@ -0,0 +1,221 @@ +/** + * What a caller can see of a version: the read path's single place for privacy. + * + * Authorization picks the sets (public, or public + private) once; everything + * after that is set-scoped reading of trees. Nothing filters individual records. + */ +import { + compareUtf8, + entryAt, + getEntry, + iterate, + type PrivateSetObject, + rankOf, + type RecordEntry, + recordTree, + type SetObject, + type TreeSummary, + type VersionRoot, +} from '@underlay/core' +import { type Repo, RepoSource } from '@underlay/repo' +import { and, desc, eq } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Db } from '../ports.js' +import { parseSemver } from './semver.js' + +export type VersionRow = typeof schema.versions.$inferSelect + +/** Find a version by semver ("v1.2.3", "1.2"), "latest", or v2 hash. */ +export async function findVersion( + db: Db, + collectionId: string, + n: string, + headVersionId: string | null, +): Promise { + let where + if (n === 'latest') { + if (!headVersionId) return null + where = eq(schema.versions.id, headVersionId) + } else if (n.startsWith('ulv2:')) { + where = and(eq(schema.versions.collectionId, collectionId), eq(schema.versions.hash, n)) + } else if (n.startsWith('private:') || n.startsWith('public:')) { + where = and( + eq(schema.versions.collectionId, collectionId), + n.startsWith('private:') + ? eq(schema.versions.legacyHash, n) + : eq(schema.versions.legacyPublicHash, n), + ) + } else { + where = and( + eq(schema.versions.collectionId, collectionId), + eq(schema.versions.semver, parseSemver(n).semver), + ) + } + const [v] = await db + .select() + .from(schema.versions) + .where(where) + .orderBy(desc(schema.versions.seq)) + .limit(1) + return v ?? null +} + +export interface TypeView { + slug: string + schemaHash: string + /** Trees this caller may read, per set. */ + public: TreeSummary | null + private: TreeSummary | null + /** Records visible to this caller. */ + count: number + bytes: number + /** True when the whole type is private. */ + privateType: boolean +} + +export interface VersionView { + version: VersionRow + repo: Repo + root: VersionRoot + public: SetObject + /** Only when the caller may read the private set. */ + private: PrivateSetObject | null + types: TypeView[] + owner: boolean +} + +export async function loadView( + repo: Repo, + version: VersionRow, + owner: boolean, +): Promise { + const root = await repo.root(version.hash) + const priv = owner && root.private ? await repo.privateSet(root.private) : null + const slugs = new Set([...Object.keys(root.public.types), ...Object.keys(priv?.types ?? {})]) + const types = [...slugs].sort(compareUtf8).map((slug): TypeView => { + const pub = root.public.types[slug] ?? null + const pri = priv?.types[slug] ?? null + return { + slug, + schemaHash: (pub ?? pri)!.schema, + public: pub, + private: pri, + count: (pub?.count ?? 0) + (pri?.count ?? 0), + bytes: (pub?.bytes ?? 0) + (pri?.bytes ?? 0), + privateType: !pub && !!pri, + } + }) + return { version, repo, root, public: root.public, private: priv, types, owner } +} + +export type VisibleRecord = RecordEntry & { type: string; set: 'public' | 'private' } + +/** Merge two sorted streams by key (keys are unique across sets). */ +async function* mergeById( + a: AsyncIterable, + b: AsyncIterable, +): AsyncGenerator<{ e: RecordEntry; set: 'public' | 'private' }> { + const ai = a[Symbol.asyncIterator]() + const bi = b[Symbol.asyncIterator]() + let x = await ai.next() + let y = await bi.next() + while (!x.done || !y.done) { + if (y.done || (!x.done && compareUtf8(x.value.key, y.value.key) < 0)) { + yield { e: x.value, set: 'public' } + x = await ai.next() + } else { + yield { e: y.value, set: 'private' } + y = await bi.next() + } + } +} + +/** + * Records of one type, in id order, from a position. `offset` seeks in + * O(height × log n) even across two sets, by binary search on ranks. + */ +export async function* typeRecords( + view: VersionView, + type: TypeView, + opts: { after?: string; offset?: number; bodies?: boolean } = {}, +): AsyncGenerator { + const source = new RepoSource(recordTree, view.repo) + const pubRoot = type.public?.root ?? null + const privRoot = view.owner ? (type.private?.root ?? null) : null + let after = opts.after + if (opts.offset && opts.offset > 0) { + if (opts.offset >= type.count) return + after = await keyBeforeOffset(source, pubRoot, privRoot, opts.offset) + } + const it = (root: string | null) => + iterate(source, root, { + payloads: opts.bodies ?? false, + ...(after !== undefined ? { after } : {}), + }) + for await (const { e, set } of mergeById(it(pubRoot), it(privRoot))) { + yield { ...e, type: type.slug, set } + } +} + +/** The key just before global position `offset` across two trees. */ +async function keyBeforeOffset( + source: RepoSource, + a: string | null, + b: string | null, + offset: number, +): Promise { + if (b === null) return (await entryAt(source, a, offset - 1))!.key + if (a === null) return (await entryAt(source, b, offset - 1))!.key + // The (offset-1)th key overall is the larger of the last keys taken from each + // tree; find how many come from `a` by binary search on ranks in `b`. + let lo = Math.max(0, offset - (await countOf(source, b))) + let hi = Math.min(offset, await countOf(source, a)) + while (lo < hi) { + const i = (lo + hi) >> 1 // take i from a, offset - i from b + const ka = (await entryAt(source, a, i))!.key + if ((await rankOf(source, b, ka)) + i < offset) lo = i + 1 + else hi = i + } + const fromA = lo + const lastA = fromA > 0 ? (await entryAt(source, a, fromA - 1))!.key : null + const lastB = offset - fromA > 0 ? (await entryAt(source, b, offset - fromA - 1))!.key : null + if (lastA === null) return lastB! + if (lastB === null) return lastA + return compareUtf8(lastA, lastB) > 0 ? lastA : lastB +} + +async function countOf(source: RepoSource, root: string): Promise { + const n = await source.node(root) + return n.kind === 'leaf' ? n.entries.length : n.children.reduce((s, c) => s + c.count, 0) +} + +/** One record by type and id, with its body. */ +export async function getRecord( + view: VersionView, + type: TypeView, + id: string, +): Promise { + const source = new RepoSource(recordTree, view.repo) + for (const [set, tree] of [ + ['public', type.public], + ['private', view.owner ? type.private : null], + ] as const) { + if (!tree?.root) continue + const hit = await getEntry(source, tree.root, id) + if (!hit) continue + const body = await bodyOf(source, tree.root, id) + return { ...hit, body, type: type.slug, set } + } + return null +} + +async function bodyOf(source: RepoSource, root: string, key: string): Promise { + let hash = root + for (;;) { + const node = await source.node(hash) + if (node.kind === 'leaf') + return (await source.leafEntries(hash)).find((e) => e.key === key)!.body! + hash = node.children.find((c) => compareUtf8(key, c.lastKey) <= 0)!.hash + } +} diff --git a/packages/server/test/read.test.ts b/packages/server/test/read.test.ts new file mode 100644 index 0000000..ce7ba8c --- /dev/null +++ b/packages/server/test/read.test.ts @@ -0,0 +1,217 @@ +import { afterAll, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) + +const Author = { type: 'object', properties: { name: { type: 'string' } } } +const Book = { type: 'object', properties: { title: { type: 'string' } } } +const Secret = { type: 'object', private: true, properties: { note: { type: 'string' } } } + +async function json(res: Response) { + expect(res.headers.get('content-type') ?? '').toMatch(/json/) + return (await res.json()) as any +} + +async function pushDelta( + h: Harness, + user: string, + base: string, + body: object, + records: object[], + deletes: object[] = [], +) { + const sid = (await json(await h.request(`${base}/push`, { method: 'POST', user, json: body }))) + .session_id + for (let i = 0; i < records.length; i += 5000) { + const r = await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: records.slice(i, i + 5000), + }) + expect(r.status).toBe(200) + } + if (deletes.length) + await h.request(`${base}/push/${sid}/deletes`, { method: 'POST', user, ndjson: deletes }) + const res = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) + expect(res.status).toBe(201) + return json(res) +} + +async function setup() { + const h = await harness() + const user = await h.member() + const c = await h.collection('lib') + await h.ports.db.update(schema.collections).set({ public: true }) + const base = '/api/collections/org/lib' + const authors = Array.from({ length: 1200 }, (_, i) => ({ + id: `a${String(i).padStart(4, '0')}`, + type: 'Author', + data: { name: `Author ${i}` }, + ...(i % 10 === 0 ? { private: true } : {}), + })) + const books = Array.from({ length: 300 }, (_, i) => ({ + id: `b${i}`, + type: 'Book', + data: { title: `Book ${i}` }, + })) + const secrets = [{ id: 's1', type: 'Secret', data: { note: 'hidden' } }] + await pushDelta( + h, + user, + base, + { + schemas: { Author, Book, Secret }, + metadata: { title: 'Library', description: 'Books', tags: ['lit'] }, + }, + [...authors, ...books, ...secrets], + ) + return { h, user, c, base } +} + +describe('read API', () => { + it('serves collections and versions with owner and public views', async () => { + const { h, user, base } = await setup() + const anon = await json(await h.request(base)) + expect(anon).toMatchObject({ + slug: 'lib', + ownerSlug: 'org', + description: 'Books', + versionCount: 1, + }) + expect(anon.latestVersion).toMatchObject({ + semver: 'v1.0.0', + recordCount: 1380, + metadata: { title: 'Library' }, + }) + expect(anon.latestVersion.typeCounts).toEqual([ + { type: 'Author', count: 1080 }, + { type: 'Book', count: 300 }, + ]) + const owner = await json(await h.request(base, { user })) + expect(owner.latestVersion.recordCount).toBe(1501) + + const list = await json(await h.request('/api/collections')) + expect(list.collections[0]).toMatchObject({ + slug: 'lib', + tags: ['lit'], + latestVersion: 'v1.0.0', + recordCount: 1380, + }) + expect(list.facets.tags).toEqual([{ name: 'lit', count: 1 }]) + + const v = await json(await h.request(`${base}/versions/v1.0.0`)) + expect(Object.keys(v.schemas)).toEqual(['Author', 'Book']) + expect(v.typeCounts).toEqual({ Author: 1080, Book: 300 }) + const vOwner = await json(await h.request(`${base}/versions/latest`, { user })) + expect(Object.keys(vOwner.schemas)).toEqual(['Author', 'Book', 'Secret']) + expect((await json(await h.request(`${base}/versions`)))[0]).toMatchObject({ semver: 'v1.0.0' }) + }) + + it('pages records by offset and by cursor, within and across types', async () => { + const { h, user, base } = await setup() + // Offset deep in a type (v1 refused offsets past 10k; here it's a seek). + let r = await json( + await h.request(`${base}/versions/latest/records?type=Author&offset=1000&limit=5`), + ) + expect(r.records.map((x: any) => x.id)).toEqual(['a1112', 'a1113', 'a1114', 'a1115', 'a1116']) + expect(r.pagination.total).toBe(1080) + r = await json( + await h.request(`${base}/versions/latest/records?type=Author&offset=1000&limit=5`, { user }), + ) + expect(r.records.map((x: any) => x.id)).toEqual(['a1000', 'a1001', 'a1002', 'a1003', 'a1004']) + + // Cursor paging across types visits every visible record once, in (type, id) order. + const seen: string[] = [] + let cursor: string | null = null + do { + const page: any = await json( + await h.request( + `${base}/versions/latest/records?limit=400${cursor ? `&cursor=${cursor}` : ''}`, + ), + ) + seen.push(...page.records.map((x: any) => `${x.type}/${x.id}`)) + cursor = page.pagination.nextCursor + } while (cursor) + expect(seen.length).toBe(1380) + expect(new Set(seen).size).toBe(1380) + expect(seen.some((s) => s.startsWith('Secret/'))).toBe(false) + // An offset across types lands in the second type. + r = await json(await h.request(`${base}/versions/latest/records?offset=1081&limit=1`)) + expect(r.records[0]).toMatchObject({ type: 'Book', id: 'b1' }) + expect(r.records[0].hash).toMatch(/^[0-9a-f]{64}$/) + }) + + it('streams NDJSON, and serves manifests (full and delta) and diffs', async () => { + const { h, user, base } = await setup() + const res = await h.request(`${base}/versions/latest/records.ndjson?type=Book`) + expect(res.headers.get('x-underlay-record-count')).toBe('300') + const lines = (await res.text()) + .trim() + .split('\n') + .map((l) => JSON.parse(l)) + expect(lines.length).toBe(300) + expect(lines[0]).toMatchObject({ id: 'b0', type: 'Book', data: { title: 'Book 0' } }) + + const pages: any[] = [] + let cursor: string | null = null + do { + const m: any = await json( + await h.request( + `${base}/versions/latest/manifest?limit=500${cursor ? `&cursor=${cursor}` : ''}`, + { user }, + ), + ) + pages.push(m) + cursor = m.pagination.nextCursor + } while (cursor) + const entries = pages.flatMap((p) => p.records) + expect(entries.length).toBe(1501) + expect(entries.filter((e) => e.private).length).toBe(121) + + // Second version: one update, one delete, one add. + await pushDelta( + h, + user, + base, + { base: 'v1.0.0' }, + [ + { id: 'b0', type: 'Book', data: { title: 'Changed' } }, + { id: 'b999', type: 'Book', data: { title: 'New' } }, + ], + [{ type: 'Book', id: 'b1' }], + ) + const delta = await json(await h.request(`${base}/versions/v1.1.0/manifest?since=v1.0.0`)) + expect(delta.delta.updated.map((x: any) => x.id)).toEqual(['b0']) + expect(delta.delta.removed.map((x: any) => x.id)).toEqual(['b1']) + expect(delta.delta.added.map((x: any) => x.id)).toEqual(['b999']) + const diff = await json(await h.request(`${base}/versions/v1.1.0/diff?from=v1.0.0`)) + expect(diff).toMatchObject({ + from: 'v1.0.0', + to: 'v1.1.0', + removed: ['b1'], + meta: { schemaChanged: false }, + }) + expect(diff.updated[0]).toMatchObject({ id: 'b0', data: { title: 'Changed' } }) + // Diff paging resumes without skipping or repeating. + const p1 = await json(await h.request(`${base}/versions/v1.1.0/diff?from=v1.0.0&limit=2`)) + const p2 = await json( + await h.request( + `${base}/versions/v1.1.0/diff?from=v1.0.0&limit=2&cursor=${p1.pagination.nextCursor}`, + ), + ) + const ids = [...p1.added, ...p1.updated, ...p2.added, ...p2.updated] + .map((x: any) => x.id) + .concat(p1.removed, p2.removed) + expect(ids.sort()).toEqual(['b0', 'b1', 'b999']) + }) + + it('hides private collections entirely', async () => { + const { h, base } = await setup() + await h.ports.db.update(schema.collections).set({ public: false }) + expect((await h.request(base)).status).toBe(404) + expect((await h.request(`${base}/versions/latest/records`)).status).toBe(404) + expect((await json(await h.request('/api/collections'))).collections).toEqual([]) + }) +}) diff --git a/packages/server/test/view.test.ts b/packages/server/test/view.test.ts new file mode 100644 index 0000000..18c5f94 --- /dev/null +++ b/packages/server/test/view.test.ts @@ -0,0 +1,83 @@ +import { + type Change, + compareUtf8, + hashRecord, + hashSchema, + type RecordEntry, + utf8ByteLength, +} from '@underlay/core' +import { afterAll, describe, expect, it } from 'vitest' + +import { commitVersion } from '../src/versions/commit.js' +import { getRecord, loadView, typeRecords } from '../src/versions/view.js' +import { cleanup, harness } from './harness.js' + +afterAll(cleanup) + +const Author = { type: 'object' } +const up = (id: string, n: number): Change => { + const { hash, canonical } = hashRecord(id, 'Author', { n }) + return { key: id, entry: { key: id, hash, size: utf8ByteLength(canonical), body: canonical } } +} + +async function collect(it: AsyncIterable): Promise { + const out: T[] = [] + for await (const x of it) out.push(x) + return out +} + +describe('version views', () => { + it('pages owners across both sets by offset and by key, and hides private records from others', async () => { + const h = await harness() + const c = await h.collection() + const ids = Array.from( + { length: 2500 }, + (_, i) => `id${String((i * 7919) % 2500).padStart(5, '0')}`, + ) + const isPrivate = (id: string) => Number(id.slice(2)) % 3 === 0 + const sort = (cs: Change[]) => cs.sort((a, b) => compareUtf8(a.key, b.key)) + const r = await commitVersion(h.ports, { + collectionId: c.id, + base: null, + types: [ + { + slug: 'Author', + schema: Author, + schemaHash: hashSchema(Author), + public: sort(ids.filter((id) => !isPrivate(id)).map((id, i) => up(id, i))), + private: sort(ids.filter(isPrivate).map((id, i) => up(id, i))), + }, + ], + metadata: null, + }) + if (r.status !== 'committed') throw new Error(r.status) + const repo = await h.ports.stores.forCollection(c.id) + const owner = await loadView(repo, r.version, true) + const reader = await loadView(repo, r.version, false) + const all = [...ids].sort(compareUtf8) + const pub = all.filter((id) => !isPrivate(id)) + expect(owner.types[0]!.count).toBe(2500) + expect(reader.types[0]!.count).toBe(pub.length) + + for (const offset of [0, 1, 833, 834, 1250, 2499, 2500]) { + const page = (await collect(typeRecords(owner, owner.types[0]!, { offset }))) + .slice(0, 3) + .map((e) => e.key) + expect(page).toEqual(all.slice(offset, offset + 3)) + } + for (const offset of [0, 5, 1000, pub.length - 1]) { + const page = (await collect(typeRecords(reader, reader.types[0]!, { offset }))) + .slice(0, 3) + .map((e) => e.key) + expect(page).toEqual(pub.slice(offset, offset + 3)) + } + const afterKey = await collect(typeRecords(owner, owner.types[0]!, { after: all[99]! })) + expect(afterKey[0]!.key).toBe(all[100]) + + const secret = all.find(isPrivate)! + expect((await getRecord(owner, owner.types[0]!, secret))?.set).toBe('private') + expect(await getRecord(reader, reader.types[0]!, secret)).toBe(null) + const visible = await getRecord(reader, reader.types[0]!, pub[0]!) + expect(JSON.parse(visible!.body!).id).toBe(pub[0]) + }) +}) From 5c8f6216651d80e0dd2f6cbdd5cecd962913c2f3 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 17:11:45 -0400 Subject: [PATCH 010/178] v2 files: direct uploads with verification, presigned downloads - Small uploads through the API (32 MB, held in memory to hash), with v1's inert-MIME rule; direct uploads to a staging key via presigned PUT or multipart part URLs, verified by a job that streams the object, checks hash and size, and copies it to the canonical repository key files/ (BlobStore.copy: S3 CopyObject, fs, memory). Staging keys can expire by lifecycle rule. - Proof of possession: an upload always carries bytes. - Downloads: 302 to a presigned attachment URL; non-members only for files in the cumulative public files tree, members also for the head's private files. HEAD reports size and type. - Known gap: copies past 5 GB need UploadPartCopy (marked TODO). --- packages/repo/src/blob/fs.ts | 21 ++- packages/repo/src/blob/memory.ts | 6 + packages/repo/src/blob/s3.ts | 16 ++ packages/repo/src/types.ts | 5 + packages/repo/test/blob.test.ts | 6 + packages/repo/test/fake-s3.ts | 12 +- packages/server/src/api/files.ts | 167 ++++++++++++++++++++ packages/server/src/app.ts | 2 + packages/server/src/files/files.ts | 243 +++++++++++++++++++++++++++++ packages/server/src/handlers.ts | 1 + packages/server/src/ports.ts | 4 + packages/server/src/stores.ts | 2 + packages/server/test/files.test.ts | 155 ++++++++++++++++++ packages/server/wrangler.jsonc | 32 ++-- 14 files changed, 654 insertions(+), 18 deletions(-) create mode 100644 packages/server/src/api/files.ts create mode 100644 packages/server/src/files/files.ts create mode 100644 packages/server/test/files.test.ts diff --git a/packages/repo/src/blob/fs.ts b/packages/repo/src/blob/fs.ts index 7446b31..44a4a23 100644 --- a/packages/repo/src/blob/fs.ts +++ b/packages/repo/src/blob/fs.ts @@ -8,7 +8,17 @@ */ import { createHmac, timingSafeEqual } from 'node:crypto' import { createReadStream } from 'node:fs' -import { mkdir, open, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' +import { + copyFile, + mkdir, + open, + readdir, + readFile, + rename, + rm, + stat, + writeFile, +} from 'node:fs/promises' import { dirname, join, relative, resolve, sep } from 'node:path' import { Readable } from 'node:stream' @@ -189,6 +199,15 @@ export class FsBlobStore implements BlobStore { await rm(this.#path(`_multipart/${uploadId}`), { recursive: true, force: true }) } + async copy(from: string, to: string): Promise { + const src = this.#path(from) + const dest = this.#path(to) + await mkdir(dirname(dest), { recursive: true }) + const tmp = `${dest}.${crypto.randomUUID()}.tmp` + await copyFile(src, tmp) + await rename(tmp, dest) + } + async abortMultipart(_key: string, uploadId: string): Promise { await rm(this.#path(`_multipart/${uploadId}`), { recursive: true, force: true }) } diff --git a/packages/repo/src/blob/memory.ts b/packages/repo/src/blob/memory.ts index e23f14a..2c4528a 100644 --- a/packages/repo/src/blob/memory.ts +++ b/packages/repo/src/blob/memory.ts @@ -104,4 +104,10 @@ export class MemoryBlobStore implements BlobStore { async abortMultipart(key: string, uploadId: string) { this.multipart.delete(`${key}#${uploadId}`) } + + async copy(from: string, to: string) { + const o = this.objects.get(from) + if (!o) throw new Error(`No such key ${from}`) + this.objects.set(to, { bytes: o.bytes.slice(), contentType: o.contentType }) + } } diff --git a/packages/repo/src/blob/s3.ts b/packages/repo/src/blob/s3.ts index 88740ff..0648c8b 100644 --- a/packages/repo/src/blob/s3.ts +++ b/packages/repo/src/blob/s3.ts @@ -237,6 +237,22 @@ export class S3BlobStore implements BlobStore { } } + async copy(from: string, to: string): Promise { + const source = `/${this.#base.split('/').pop()}/${encodeKey(from)}` + const res = await this.#aws.fetch(this.#url(to), { + method: 'PUT', + headers: { 'x-amz-copy-source': source }, + }) + const text = await res.text() + // Like CompleteMultipartUpload, CopyObject can answer 200 with an body. + if (!res.ok || text.includes('')) { + throw new S3Error( + res.ok ? 500 : res.status, + `CopyObject ${from} → ${to}: ${text.slice(0, 200)}`, + ) + } + } + async abortMultipart(key: string, uploadId: string): Promise { const res = await this.#aws.fetch(this.#url(key, { uploadId }), { method: 'DELETE' }) if (!res.ok && res.status !== 404) return this.#fail(res, `AbortMultipartUpload ${key}`) diff --git a/packages/repo/src/types.ts b/packages/repo/src/types.ts index 86dce9b..4b8ed2c 100644 --- a/packages/repo/src/types.ts +++ b/packages/repo/src/types.ts @@ -47,6 +47,8 @@ export interface BlobStore { parts: { partNumber: number; etag: string }[], ): Promise abortMultipart(key: string, uploadId: string): Promise + /** Server-side copy within the bucket (up to 5 GB on S3). */ + copy(from: string, to: string): Promise } /** A shared cache for immutable, hash-keyed bytes (nodes, roots, schemas, bodies). */ @@ -112,4 +114,7 @@ export class PrefixedBlobStore implements BlobStore { abortMultipart(key: string, uploadId: string) { return this.inner.abortMultipart(this.#k(key), uploadId) } + copy(from: string, to: string) { + return this.inner.copy(this.#k(from), this.#k(to)) + } } diff --git a/packages/repo/test/blob.test.ts b/packages/repo/test/blob.test.ts index 32da7c5..38f7b8e 100644 --- a/packages/repo/test/blob.test.ts +++ b/packages/repo/test/blob.test.ts @@ -46,6 +46,12 @@ function contract(name: string, make: () => Promise) { expect(text.length).toBe(100_000) }) + it('copies within the bucket', async () => { + await store.put('copy/src', 'copied bytes') + await store.copy('copy/src', 'copy/dest/x') + expect(await (await store.get('copy/dest/x'))!.text()).toBe('copied bytes') + }) + it('lists by prefix', async () => { await store.put('list/1', '1') await store.put('list/2', '2') diff --git a/packages/repo/test/fake-s3.ts b/packages/repo/test/fake-s3.ts index 9e593a6..a901834 100644 --- a/packages/repo/test/fake-s3.ts +++ b/packages/repo/test/fake-s3.ts @@ -86,7 +86,16 @@ export async function startFakeS3(bucket = 'test'): Promise { } const obj = objects.get(key) switch (req.method) { - case 'PUT': + case 'PUT': { + const copySource = req.headers['x-amz-copy-source'] + if (typeof copySource === 'string') { + const srcKey = decodeURIComponent(copySource.replace(/^\/[^/]+\//, '')) + const src = objects.get(srcKey) + if (!src) return void res.writeHead(404).end('NoSuchKey') + objects.set(key, { bytes: Buffer.from(src.bytes), contentType: src.contentType }) + res.writeHead(200).end('"x"') + return + } if (req.headers['if-none-match'] === '*' && obj) { res.writeHead(412).end('PreconditionFailed') return @@ -94,6 +103,7 @@ export async function startFakeS3(bucket = 'test'): Promise { objects.set(key, { bytes: body, contentType: req.headers['content-type'] }) res.writeHead(200, { etag: '"x"' }).end() return + } case 'HEAD': case 'GET': { if (!obj) diff --git a/packages/server/src/api/files.ts b/packages/server/src/api/files.ts new file mode 100644 index 0000000..595b9ba --- /dev/null +++ b/packages/server/src/api/files.ts @@ -0,0 +1,167 @@ +/** + * File routes, mounted at /api/collections. + * + * HEAD|GET /:owner/:slug/files/:hash 302 to a presigned URL (v1 contract) + * POST /:owner/:slug/files/presign {hashes} → {hash: url|null} (read-only) + * PUT /:owner/:slug/files/:hash small upload through the API + * POST /:owner/:slug/files/uploads start a direct upload → presigned PUT or parts + * POST /:owner/:slug/files/uploads/:id/complete + * GET /:owner/:slug/files/uploads/:id pending | verifying | verified | failed + */ +import { and, eq } from 'drizzle-orm' +import { Hono } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import { + canReadFile, + cleanHash, + completeUpload, + isHash, + presignDownload, + safeMimeType, + SMALL_UPLOAD_BYTES, + startUpload, + storeSmallFile, +} from '../files/files.js' +import { jsonError, requireCollection } from './access.js' + +export function fileRoutes() { + const app = new Hono() + + // Hono routes HEAD to GET handlers, so one handler serves both. + app.get('/:owner/:slug/files/:hash', async (c) => { + const head = c.req.method === 'HEAD' + const access = await requireCollection(c, 'read') + if (access instanceof Response) return head ? c.body(null, 404) : access + const hash = cleanHash(c.req.param('hash')) + if ( + !isHash(hash) || + !(await canReadFile(c.var.ports, access.collection, access.isMember, hash)) + ) { + return head ? c.body(null, 404) : jsonError(c, 404, 'File not found') + } + const [f] = await c.var.ports.db + .select() + .from(schema.files) + .where(eq(schema.files.hash, hash)) + .limit(1) + if (!f) return head ? c.body(null, 404) : jsonError(c, 404, 'File not found') + if (head) + return c.body(null, 200, { 'content-length': String(f.size), 'content-type': f.mimeType }) + return c.redirect((await presignDownload(c.var.ports, hash))!, 302) + }) + + app.post('/:owner/:slug/files/presign', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const body = (await c.req.json().catch(() => null)) as { hashes?: unknown } | null + if (!Array.isArray(body?.hashes) || body.hashes.length > 500) { + return jsonError(c, 400, '"hashes" must be an array of at most 500 file hashes') + } + const out: Record = {} + for (const requested of body.hashes) { + if (typeof requested !== 'string') continue + const hash = cleanHash(requested) + out[requested] = + isHash(hash) && (await canReadFile(c.var.ports, access.collection, access.isMember, hash)) + ? await presignDownload(c.var.ports, hash) + : null + } + return c.json(out) + }) + + app.put('/:owner/:slug/files/:hash', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const hash = cleanHash(c.req.param('hash')) + if (!isHash(hash)) return jsonError(c, 400, 'Not a sha256 file hash') + const declared = Number(c.req.header('content-length') ?? NaN) + if (declared > SMALL_UPLOAD_BYTES) { + return jsonError( + c, + 413, + `Uploads through the API are limited to ${SMALL_UPLOAD_BYTES} bytes; use POST .../files/uploads for larger files`, + ) + } + const type = c.req.header('content-type') ?? 'application/octet-stream' + let bytes: Uint8Array + let mime: string + if (type.startsWith('multipart/')) { + const form = await c.req.parseBody() + const file = form.file + if (!(file instanceof File)) return jsonError(c, 400, 'No file in multipart body') + bytes = new Uint8Array(await file.arrayBuffer()) + mime = file.type + } else { + bytes = new Uint8Array(await c.req.arrayBuffer()) + mime = type + } + if (bytes.byteLength > SMALL_UPLOAD_BYTES) + return jsonError(c, 413, `File exceeds ${SMALL_UPLOAD_BYTES} bytes`) + const result = await storeSmallFile(c.var.ports, hash, bytes, safeMimeType(mime)) + if (result === 'mismatch') return jsonError(c, 400, 'Hash mismatch', { expected: hash }) + return c.json({ hash, status: 'stored', size: bytes.byteLength }, 201) + }) + + app.post('/:owner/:slug/files/uploads', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const body = (await c.req.json().catch(() => null)) as { + hash?: unknown + size?: unknown + mimeType?: unknown + } | null + const hash = typeof body?.hash === 'string' ? cleanHash(body.hash) : '' + const size = Number(body?.size) + if (!isHash(hash)) return jsonError(c, 400, '"hash" must be a sha256 file hash') + if (!Number.isSafeInteger(size) || size < 0) + return jsonError(c, 400, '"size" must be a byte count') + const ticket = await startUpload(c.var.ports, access.collection.id, { + hash, + size, + mimeType: safeMimeType(typeof body?.mimeType === 'string' ? body.mimeType : undefined), + }) + return c.json(ticket, 201) + }) + + app.post('/:owner/:slug/files/uploads/:id/complete', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const [u] = await c.var.ports.db + .select() + .from(schema.fileUploads) + .where( + and( + eq(schema.fileUploads.id, c.req.param('id')), + eq(schema.fileUploads.collectionId, access.collection.id), + ), + ) + .limit(1) + if (!u) return jsonError(c, 404, 'Upload not found') + const body = (await c.req.json().catch(() => ({}))) as { + parts?: { partNumber: number; etag: string }[] + } + await completeUpload(c.var.ports, u, body.parts) + return c.json({ id: u.id, status: 'verifying' }, 202) + }) + + app.get('/:owner/:slug/files/uploads/:id', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const [u] = await c.var.ports.db + .select() + .from(schema.fileUploads) + .where( + and( + eq(schema.fileUploads.id, c.req.param('id')), + eq(schema.fileUploads.collectionId, access.collection.id), + ), + ) + .limit(1) + if (!u) return jsonError(c, 404, 'Upload not found') + return c.json({ id: u.id, hash: u.hash, size: u.size, status: u.status, error: u.error }) + }) + + return app +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index eb59216..b6f3c44 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -7,6 +7,7 @@ import { type Context, Hono } from 'hono' import type { Principal } from './api/access.js' import { collectionRoutes } from './api/collections.js' +import { fileRoutes } from './api/files.js' import { pushRoutes } from './api/push.js' import { versionRoutes } from './api/versions.js' import type { Ports } from './ports.js' @@ -71,6 +72,7 @@ export function createApp(setup: Setup) { }), ) + app.route('/api/collections', fileRoutes()) app.route('/api/collections', pushRoutes()) app.route('/api/collections', versionRoutes()) app.route('/', collectionRoutes()) diff --git a/packages/server/src/files/files.ts b/packages/server/src/files/files.ts new file mode 100644 index 0000000..400105f --- /dev/null +++ b/packages/server/src/files/files.ts @@ -0,0 +1,243 @@ +/** + * Files (edge-redesign.md, "Files"): SHA-256-of-bytes ids, bytes never served + * from the app's origin (presigned URLs on the bucket's domain), uploads + * verified before a file exists. + * + * Upload paths: + * - small: PUT through the API, hashed in memory (bounded by SMALL_UPLOAD_BYTES); + * - direct: a presigned PUT (or multipart part URLs) to a staging key, then a + * job streams the object, checks its hash, and copies it to the canonical + * repository key `files/` (mirrors need canonical keys; staging keys + * can then expire by lifecycle rule). + * + * Proof of possession: an upload always carries the bytes, even when the + * server already has the file, so "do you have X?" is never answered for free. + */ +import { createHash } from 'node:crypto' + +import { fileTree, getEntry } from '@underlay/core' +import { RepoSource } from '@underlay/repo' +import { and, eq } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import { registerJob } from '../jobs.js' +import type { Ports } from '../ports.js' + +/** PUT-through-the-API limit: the body is held in isolate memory to hash it. */ +export const SMALL_UPLOAD_BYTES = 32 * 1024 * 1024 +/** Single presigned PUT limit (S3/R2); larger files use multipart. */ +export const SINGLE_PUT_BYTES = 5 * 1024 * 1024 * 1024 +/** Server-side copy limit; past it the verified staging object would need UploadPartCopy. */ +export const COPY_LIMIT_BYTES = 5 * 1024 * 1024 * 1024 +export const PART_BYTES = 100 * 1024 * 1024 +export const PRESIGN_SECONDS = 300 + +// Types that render or run in a browser are stored as inert bytes (v1 rule). +const UNSAFE_MIME = /^(text\/html|application\/xhtml|image\/svg|text\/xml|application\/xml)/i +export const safeMimeType = (mime: string | undefined) => + !mime || UNSAFE_MIME.test(mime.trim()) ? 'application/octet-stream' : mime.trim() + +const HEX64 = /^[0-9a-f]{64}$/ +export const cleanHash = (h: string) => h.replace(/^sha256:/, '').toLowerCase() +export const isHash = (h: string) => HEX64.test(h) + +/** + * May this caller read this file through this collection? Non-members: the file + * is in the collection's cumulative public files tree. Members: also the head's + * file trees (both sets). + */ +export async function canReadFile( + ports: Ports, + collection: typeof schema.collections.$inferSelect, + member: boolean, + hash: string, +): Promise { + const repo = await ports.stores.forCollection(collection.id) + const source = new RepoSource(fileTree, repo) + if (await getEntry(source, collection.publicFilesRoot, hash)) return true + if (!member || !collection.headVersionId) return false + const [v] = await ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, collection.headVersionId)) + if (!v) return false + const root = await repo.root(v.hash) + if (await getEntry(source, root.public.files.root, hash)) return true + if (!root.private) return false + const priv = await repo.privateSet(root.private) + return !!(await getEntry(source, priv.files.root, hash)) +} + +export async function presignDownload(ports: Ports, hash: string): Promise { + const [f] = await ports.db.select().from(schema.files).where(eq(schema.files.hash, hash)).limit(1) + if (!f) return null + // Downloads are attachments: nothing renders on the bucket's domain either. + return ports.stores.fileBytes.presignGet(f.storageKey, { + expiresIn: PRESIGN_SECONDS, + disposition: `attachment; filename="${hash}"`, + contentType: f.mimeType, + }) +} + +/** Store verified bytes at the canonical key and record the file. Idempotent. */ +export async function storeSmallFile( + ports: Ports, + hash: string, + bytes: Uint8Array, + mime: string, +): Promise<'stored' | 'mismatch'> { + const actual = createHash('sha256').update(bytes).digest('hex') + if (actual !== hash) return 'mismatch' + const [existing] = await ports.db + .select() + .from(schema.files) + .where(eq(schema.files.hash, hash)) + .limit(1) + if (existing) return 'stored' + const key = ports.stores.canonicalFileKey(hash) + await ports.stores.fileBytes.put(key, bytes, { contentType: mime, ifAbsent: true }) + await ports.db + .insert(schema.files) + .values({ + hash, + size: bytes.byteLength, + mimeType: mime, + storageKey: key, + verifiedAt: new Date(), + }) + .onConflictDoNothing() + return 'stored' +} + +export interface UploadTicket { + id: string + url?: string + parts?: { partNumber: number; url: string }[] + expiresIn: number +} + +/** Start a direct upload to a staging key; returns where the client PUTs bytes. */ +export async function startUpload( + ports: Ports, + collectionId: string, + req: { hash: string; size: number; mimeType: string }, +): Promise { + const id = crypto.randomUUID() + const key = ports.stores.stagingKey(id) + const blobs = ports.stores.fileBytes + let multipartUploadId: string | null = null + const ticket: UploadTicket = { id, expiresIn: 3600 } + if (req.size <= SINGLE_PUT_BYTES) { + ticket.url = await blobs.presignPut(key, { expiresIn: 3600 }) + } else { + multipartUploadId = await blobs.createMultipart(key, req.mimeType) + const n = Math.ceil(req.size / PART_BYTES) + ticket.parts = await Promise.all( + Array.from({ length: n }, async (_, i) => ({ + partNumber: i + 1, + url: await blobs.presignPart(key, multipartUploadId!, i + 1, 3600), + })), + ) + } + await ports.db.insert(schema.fileUploads).values({ + id, + collectionId, + hash: req.hash, + size: req.size, + mimeType: req.mimeType, + storageKey: key, + multipartUploadId, + }) + return ticket +} + +export async function completeUpload( + ports: Ports, + upload: typeof schema.fileUploads.$inferSelect, + parts: { partNumber: number; etag: string }[] | undefined, +): Promise { + if (upload.multipartUploadId) { + if (!parts?.length) throw new Error('Multipart uploads complete with their parts') + await ports.stores.fileBytes.completeMultipart( + upload.storageKey, + upload.multipartUploadId, + parts, + ) + } + const claimed = await ports.db + .update(schema.fileUploads) + .set({ status: 'verifying' }) + .where(and(eq(schema.fileUploads.id, upload.id), eq(schema.fileUploads.status, 'pending'))) + .returning({ id: schema.fileUploads.id }) + if (claimed.length === 1) await ports.jobs.enqueue({ type: 'files.verify', uploadId: upload.id }) +} + +/** + * Verify a staged upload: stream it, hash it, and on a match make it a file. + * One job hashes at roughly 1 GB/s of CPU; files past what fits in one job's + * CPU budget need a resumable SHA-256 (edge-redesign.md, Files). + */ +export async function verifyUpload(ports: Ports, uploadId: string): Promise { + const [u] = await ports.db + .select() + .from(schema.fileUploads) + .where(eq(schema.fileUploads.id, uploadId)) + .limit(1) + if (!u || u.status !== 'verifying') return + const blobs = ports.stores.fileBytes + const fail = async (error: string) => { + await ports.db + .update(schema.fileUploads) + .set({ status: 'failed', error }) + .where(eq(schema.fileUploads.id, uploadId)) + await blobs.delete(u.storageKey) + } + const obj = await blobs.get(u.storageKey) + if (!obj) return fail('Nothing was uploaded') + const h = createHash('sha256') + let size = 0 + const reader = obj.body.getReader() + for (;;) { + const { done, value } = await reader.read() + if (done) break + h.update(value) + size += value.byteLength + } + const actual = h.digest('hex') + if (actual !== u.hash) return fail(`Hash mismatch: uploaded bytes hash to ${actual}`) + if (size !== u.size) return fail(`Size mismatch: ${size} bytes, declared ${u.size}`) + + const [existing] = await ports.db + .select() + .from(schema.files) + .where(eq(schema.files.hash, u.hash)) + .limit(1) + if (!existing) { + let storageKey = ports.stores.canonicalFileKey(u.hash) + if (size <= COPY_LIMIT_BYTES) { + await blobs.copy(u.storageKey, storageKey) + } else { + // TODO(files): UploadPartCopy for >5 GB. Until then the staging object is kept + // as the file; the staging lifecycle rule must not cover verified uploads. + storageKey = u.storageKey + } + await ports.db + .insert(schema.files) + .values({ hash: u.hash, size, mimeType: u.mimeType, storageKey, verifiedAt: new Date() }) + .onConflictDoNothing() + if (storageKey === u.storageKey) { + await ports.db + .update(schema.fileUploads) + .set({ status: 'verified' }) + .where(eq(schema.fileUploads.id, uploadId)) + return + } + } + await blobs.delete(u.storageKey) + await ports.db + .update(schema.fileUploads) + .set({ status: 'verified' }) + .where(eq(schema.fileUploads.id, uploadId)) +} + +registerJob('files.verify', async (job, ports) => verifyUpload(ports, String(job.uploadId))) diff --git a/packages/server/src/handlers.ts b/packages/server/src/handlers.ts index 19f55e5..d3153c5 100644 --- a/packages/server/src/handlers.ts +++ b/packages/server/src/handlers.ts @@ -10,6 +10,7 @@ import { readHead } from '@underlay/repo' import { and, asc, eq, gt } from 'drizzle-orm' import * as schema from './db/schema.js' +import './files/files.js' import { registerJob } from './jobs.js' import { appendVersionLog } from './versions/commit.js' diff --git a/packages/server/src/ports.ts b/packages/server/src/ports.ts index 8551212..baecfd3 100644 --- a/packages/server/src/ports.ts +++ b/packages/server/src/ports.ts @@ -49,6 +49,10 @@ export interface Stores { internal: BlobStore /** File bytes, by the `files.storage_key` (v1 keys are relative to the bucket root). */ fileBytes: BlobStore + /** The canonical key of a verified file in the platform repository (relative to fileBytes). */ + canonicalFileKey(hash: string): string + /** Where a direct upload is staged before verification (relative to fileBytes; expires by lifecycle rule). */ + stagingKey(uploadId: string): string } export interface Ports { diff --git a/packages/server/src/stores.ts b/packages/server/src/stores.ts index 3781c5a..a2658cf 100644 --- a/packages/server/src/stores.ts +++ b/packages/server/src/stores.ts @@ -71,6 +71,8 @@ export function createStores(db: Db, cache: Cache, platform: PlatformStorage): S }, internal: new PrefixedBlobStore(platform.bucket, platform.internalPrefix), fileBytes: platform.bucket, + canonicalFileKey: (hash) => [platform.repoPrefix, 'files', hash].filter(Boolean).join('/'), + stagingKey: (id) => [platform.internalPrefix, 'uploads', id].filter(Boolean).join('/'), } } diff --git a/packages/server/test/files.test.ts b/packages/server/test/files.test.ts new file mode 100644 index 0000000..7bbde5f --- /dev/null +++ b/packages/server/test/files.test.ts @@ -0,0 +1,155 @@ +import { createHash } from 'node:crypto' + +import { eq } from 'drizzle-orm' +import { afterAll, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { cleanup, harness } from './harness.js' + +afterAll(cleanup) + +const sha = (b: string | Uint8Array) => createHash('sha256').update(b).digest('hex') +const Doc = { type: 'object', properties: { title: { type: 'string' }, pdf: {} } } + +async function json(res: Response) { + return (await res.json()) as any +} + +describe('files', () => { + it('uploads small files through the API, verifying the hash', async () => { + const h = await harness() + const user = await h.member() + await h.collection('docs') + const base = '/api/collections/org/docs' + const bytes = 'hello file' + let res = await h.request(`${base}/files/${'0'.repeat(64)}`, { + method: 'PUT', + user, + body: bytes, + }) + expect(res.status).toBe(400) + res = await h.request(`${base}/files/sha256:${sha(bytes)}`, { + method: 'PUT', + user, + body: bytes, + headers: { 'content-type': 'text/html' }, + }) + expect(res.status).toBe(201) + const [f] = await h.ports.db.select().from(schema.files) + // HTML is stored as inert bytes; the key is the canonical repository key. + expect(f).toMatchObject({ + hash: sha(bytes), + size: 10, + mimeType: 'application/octet-stream', + storageKey: `repo/files/${sha(bytes)}`, + }) + expect(h.bucket.objects.has(`repo/files/${sha(bytes)}`)).toBe(true) + }) + + it('verifies direct uploads in a job and copies them to the canonical key', async () => { + const h = await harness() + const user = await h.member() + await h.collection('docs') + const base = '/api/collections/org/docs' + const good = new TextEncoder().encode('x'.repeat(100_000)) + let res = await h.request(`${base}/files/uploads`, { + method: 'POST', + user, + json: { hash: sha(good), size: good.length, mimeType: 'application/pdf' }, + }) + expect(res.status).toBe(201) + const ticket = await json(res) + expect(ticket.url).toBeTruthy() + // The client PUTs to the presigned URL; in tests, write the staging key directly. + await h.bucket.put(`internal/uploads/${ticket.id}`, good) + res = await h.request(`${base}/files/uploads/${ticket.id}/complete`, { + method: 'POST', + user, + json: {}, + }) + expect(res.status).toBe(202) + await h.drain() + expect( + (await json(await h.request(`${base}/files/uploads/${ticket.id}`, { user }))).status, + ).toBe('verified') + expect(h.bucket.objects.has(`repo/files/${sha(good)}`)).toBe(true) + expect(h.bucket.objects.has(`internal/uploads/${ticket.id}`)).toBe(false) + + // Bytes that don't match the declared hash are refused and removed. + const bad = await json( + await h.request(`${base}/files/uploads`, { + method: 'POST', + user, + json: { hash: sha('other'), size: 3 }, + }), + ) + await h.bucket.put(`internal/uploads/${bad.id}`, 'abc') + await h.request(`${base}/files/uploads/${bad.id}/complete`, { method: 'POST', user, json: {} }) + await h.drain() + const status = await json(await h.request(`${base}/files/uploads/${bad.id}`, { user })) + expect(status).toMatchObject({ status: 'failed' }) + expect(status.error).toMatch(/Hash mismatch/) + expect( + ( + await h.ports.db + .select() + .from(schema.files) + .where(eq(schema.files.hash, sha('other'))) + ).length, + ).toBe(0) + }) + + it('serves files by redirect only to those who may read them', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('docs') + await h.ports.db.update(schema.collections).set({ public: true }) + const base = '/api/collections/org/docs' + const pub = 'public pdf bytes' + const priv = 'private pdf bytes' + for (const b of [pub, priv]) { + expect( + (await h.request(`${base}/files/${sha(b)}`, { method: 'PUT', user, body: b })).status, + ).toBe(201) + } + const sid = ( + await json( + await h.request(`${base}/push`, { method: 'POST', user, json: { schemas: { Doc } } }), + ) + ).session_id + await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [ + { id: 'd1', type: 'Doc', data: { title: 'Public', pdf: { $file: `sha256:${sha(pub)}` } } }, + { + id: 'd2', + type: 'Doc', + data: { title: 'Private', pdf: { $file: `sha256:${sha(priv)}` } }, + private: true, + }, + ], + }) + expect((await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user })).status).toBe( + 201, + ) + + const anonPub = await h.request(`${base}/files/${sha(pub)}`) + expect(anonPub.status).toBe(302) + expect(anonPub.headers.get('location')).toContain(`repo/files/${sha(pub)}`) + expect((await h.request(`${base}/files/${sha(priv)}`)).status).toBe(404) + expect((await h.request(`${base}/files/${sha(priv)}`, { user })).status).toBe(302) + expect( + (await h.request(`${base}/files/${sha(pub)}`, { method: 'HEAD' })).headers.get( + 'content-length', + ), + ).toBe(String(pub.length)) + + // The files listing follows the same sets. + const anonList = await json(await h.request(`${base}/versions/latest/files`)) + expect(anonList.map((f: any) => f.hash)).toEqual([sha(pub)]) + const ownerList = await json(await h.request(`${base}/versions/latest/files`, { user })) + expect(ownerList.length).toBe(2) + void c + }) +}) diff --git a/packages/server/wrangler.jsonc b/packages/server/wrangler.jsonc index a6e04da..a096494 100644 --- a/packages/server/wrangler.jsonc +++ b/packages/server/wrangler.jsonc @@ -30,15 +30,15 @@ "R2_ENDPOINT": "https://b66a0000000000000000000000000000.r2.cloudflarestorage.com", "R2_BUCKET": "underlay-v2-staging", "OIDC_ISSUER_URL": "https://auth.knowledgefutures.org", - "OIDC_CLIENT_ID": "kf_underlay" + "OIDC_CLIENT_ID": "kf_underlay", }, "d1_databases": [ { "binding": "DB", "database_name": "underlay-v2-staging", "database_id": "00000000-0000-0000-0000-000000000000", - "migrations_dir": "drizzle" - } + "migrations_dir": "drizzle", + }, ], "queues": { "producers": [{ "binding": "JOBS", "queue": "underlay-v2-staging-jobs" }], @@ -47,11 +47,11 @@ "queue": "underlay-v2-staging-jobs", "max_batch_size": 10, "max_retries": 10, - "dead_letter_queue": "underlay-v2-staging-dead" - } - ] + "dead_letter_queue": "underlay-v2-staging-dead", + }, + ], }, - "triggers": { "crons": ["*/10 * * * *"] } + "triggers": { "crons": ["*/10 * * * *"] }, }, "next": { "name": "underlay-v2-next", @@ -62,15 +62,15 @@ "R2_ENDPOINT": "https://b66a0000000000000000000000000000.r2.cloudflarestorage.com", "R2_BUCKET": "underlay-v2-next", "OIDC_ISSUER_URL": "https://auth.knowledgefutures.org", - "OIDC_CLIENT_ID": "kf_underlay" + "OIDC_CLIENT_ID": "kf_underlay", }, "d1_databases": [ { "binding": "DB", "database_name": "underlay-v2-next", "database_id": "00000000-0000-0000-0000-000000000000", - "migrations_dir": "drizzle" - } + "migrations_dir": "drizzle", + }, ], "queues": { "producers": [{ "binding": "JOBS", "queue": "underlay-v2-next-jobs" }], @@ -79,11 +79,11 @@ "queue": "underlay-v2-next-jobs", "max_batch_size": 10, "max_retries": 10, - "dead_letter_queue": "underlay-v2-next-dead" - } - ] + "dead_letter_queue": "underlay-v2-next-dead", + }, + ], }, - "triggers": { "crons": ["*/10 * * * *"] } - } - } + "triggers": { "crons": ["*/10 * * * *"] }, + }, + }, } From bc4aeb4c109c9b868ee2a93202a4a7c83d6da473 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 17:15:00 -0400 Subject: [PATCH 011/178] v2 webhooks: v1 payload and signature, delivered and retried by jobs version.published creates one delivery per enabled webhook whose bump filter matches (idempotent per webhook and version) and queues a delivery job each; failures retry as delayed jobs (1 min doubling to 6 h, 5 attempts). Same body, headers and x-underlay-signature HMAC as v1. SSRF: v1's URL and private-address checks at registration; Node resolves and refuses private addresses before fetching (Workers have no private network). Management routes port v1's, for org owners and admins. A maintenance.sweep job expires idle push sessions and purges old deliveries (cron on Workers). --- packages/server/drizzle/0000_init.sql | 2 +- .../server/drizzle/meta/0000_snapshot.json | 5 +- .../server/drizzle/meta/0001_snapshot.json | 7 +- packages/server/drizzle/meta/_journal.json | 4 +- packages/server/src/api/access.ts | 9 +- packages/server/src/api/webhooks.ts | 192 ++++++++++++ packages/server/src/app.ts | 2 + packages/server/src/db/schema.ts | 4 +- packages/server/src/handlers.ts | 14 +- packages/server/src/node/guarded-fetch.ts | 20 ++ packages/server/src/node/main.ts | 2 + packages/server/src/ports.ts | 5 + packages/server/src/webhooks/webhooks.ts | 276 ++++++++++++++++++ packages/server/src/worker.ts | 1 + packages/server/test/db.test.ts | 1 + packages/server/test/harness.ts | 12 + packages/server/test/webhooks.test.ts | 129 ++++++++ 17 files changed, 669 insertions(+), 16 deletions(-) create mode 100644 packages/server/src/api/webhooks.ts create mode 100644 packages/server/src/node/guarded-fetch.ts create mode 100644 packages/server/src/webhooks/webhooks.ts create mode 100644 packages/server/test/webhooks.test.ts diff --git a/packages/server/drizzle/0000_init.sql b/packages/server/drizzle/0000_init.sql index c404185..2784ca3 100644 --- a/packages/server/drizzle/0000_init.sql +++ b/packages/server/drizzle/0000_init.sql @@ -74,7 +74,7 @@ CREATE TABLE `collection_webhooks` ( `id` text PRIMARY KEY NOT NULL, `collection_id` text NOT NULL, `url` text NOT NULL, - `bump_filter` text DEFAULT 'all' NOT NULL, + `bump_filter` text NOT NULL, `secret` text NOT NULL, `enabled` integer DEFAULT true NOT NULL, `created_by` text, diff --git a/packages/server/drizzle/meta/0000_snapshot.json b/packages/server/drizzle/meta/0000_snapshot.json index fad99ad..e2b132e 100644 --- a/packages/server/drizzle/meta/0000_snapshot.json +++ b/packages/server/drizzle/meta/0000_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "546a7fbc-8404-4b9e-a3d6-af1a1f02737a", + "id": "00e8c035-e6d3-45a4-8822-fdc5b49c2650", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { "account": { @@ -497,8 +497,7 @@ "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, - "default": "'all'" + "autoincrement": false }, "secret": { "name": "secret", diff --git a/packages/server/drizzle/meta/0001_snapshot.json b/packages/server/drizzle/meta/0001_snapshot.json index 61b93db..092732c 100644 --- a/packages/server/drizzle/meta/0001_snapshot.json +++ b/packages/server/drizzle/meta/0001_snapshot.json @@ -1,6 +1,6 @@ { - "id": "89c64381-abbc-4f02-8032-90d5d70523a9", - "prevId": "546a7fbc-8404-4b9e-a3d6-af1a1f02737a", + "id": "f0b0acb6-1748-4f0a-8c16-77349de423dc", + "prevId": "00e8c035-e6d3-45a4-8822-fdc5b49c2650", "version": "6", "dialect": "sqlite", "tables": { @@ -497,8 +497,7 @@ "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, - "default": "'all'" + "autoincrement": false }, "secret": { "name": "secret", diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index 0cc11d7..c85d2e2 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -5,14 +5,14 @@ { "idx": 0, "version": "6", - "when": 1791061421563, + "when": 1791062066301, "tag": "0000_init", "breakpoints": true }, { "idx": 1, "version": "6", - "when": 1791061422322, + "when": 1791062067031, "tag": "0001_platform_location", "breakpoints": true } diff --git a/packages/server/src/api/access.ts b/packages/server/src/api/access.ts index 4a2d909..ba1641d 100644 --- a/packages/server/src/api/access.ts +++ b/packages/server/src/api/access.ts @@ -27,6 +27,8 @@ export interface CollectionAccess { owner: typeof schema.organization.$inferSelect /** Members of the owning org read both sets. */ isMember: boolean + /** The member's role in the owning org ('owner' | 'admin' | 'member'), when a member. */ + role: string | null canRead: boolean canWrite: boolean /** Which sets this caller may read. */ @@ -48,14 +50,17 @@ export async function collectionAccess( if (!row || row.collection.deletedAt) return null let isMember = false + let role: string | null = null if (principal) { const keyCovers = principal.collectionIds === null || principal.collectionIds.includes(row.collection.id) if (keyCovers && principal.orgId) { isMember = principal.orgId === row.owner.id + // An org-owned key acts with the org's authority. + if (isMember) role = 'owner' } else if (keyCovers) { const [m] = await db - .select({ id: schema.member.id }) + .select({ id: schema.member.id, role: schema.member.role }) .from(schema.member) .where( and( @@ -65,6 +70,7 @@ export async function collectionAccess( ) .limit(1) isMember = !!m + role = m?.role ?? null } } const canRead = row.collection.public || isMember @@ -72,6 +78,7 @@ export async function collectionAccess( return { ...row, isMember, + role, canRead, canWrite, sets: isMember ? ['public', 'private'] : ['public'], diff --git a/packages/server/src/api/webhooks.ts b/packages/server/src/api/webhooks.ts new file mode 100644 index 0000000..e2b41a2 --- /dev/null +++ b/packages/server/src/api/webhooks.ts @@ -0,0 +1,192 @@ +/** + * Webhook management (v1 shapes), mounted at /api/collections. Org owners and + * admins only. + * + * GET /:owner/:slug/webhooks + * POST /:owner/:slug/webhooks {url, bumpFilter?, enabled?} → secret, once + * PATCH /:owner/:slug/webhooks/:id + * DELETE /:owner/:slug/webhooks/:id + * GET /:owner/:slug/webhooks/:id/deliveries?limit + * POST /:owner/:slug/webhooks/:id/deliveries/:deliveryId/retry + */ +import { and, desc, eq } from 'drizzle-orm' +import { type Context, Hono } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import { generateWebhookSecret, validateWebhookUrl } from '../webhooks/webhooks.js' +import { type CollectionAccess, jsonError, requireCollection } from './access.js' + +const BUMPS = ['major', 'minor', 'patch'] as const +type Bump = (typeof BUMPS)[number] +const bumpList = (v: unknown): Bump[] | null => + Array.isArray(v) && v.length > 0 && v.every((b) => BUMPS.includes(b)) ? (v as Bump[]) : null + +const publicFields = { + id: schema.collectionWebhooks.id, + url: schema.collectionWebhooks.url, + bumpFilter: schema.collectionWebhooks.bumpFilter, + enabled: schema.collectionWebhooks.enabled, + createdAt: schema.collectionWebhooks.createdAt, + lastDeliveryAt: schema.collectionWebhooks.lastDeliveryAt, +} + +async function requireAdmin(c: Context): Promise { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + if (access.role !== 'owner' && access.role !== 'admin') return jsonError(c, 403, 'Forbidden') + return access +} + +const allowInsecure = (c: Context) => + c.var.config.deployment === 'dev' || c.var.config.deployment === 'test' + +export function webhookRoutes() { + const app = new Hono() + + app.get('/:owner/:slug/webhooks', async (c) => { + const access = await requireAdmin(c) + if (access instanceof Response) return access + const webhooks = await c.var.ports.db + .select(publicFields) + .from(schema.collectionWebhooks) + .where(eq(schema.collectionWebhooks.collectionId, access.collection.id)) + .orderBy(desc(schema.collectionWebhooks.createdAt)) + return c.json({ webhooks }) + }) + + app.post('/:owner/:slug/webhooks', async (c) => { + const access = await requireAdmin(c) + if (access instanceof Response) return access + const body = (await c.req.json().catch(() => null)) as { + url?: unknown + bumpFilter?: unknown + enabled?: unknown + } | null + if (typeof body?.url !== 'string') return jsonError(c, 400, '"url" is required') + const check = validateWebhookUrl(body.url, allowInsecure(c)) + if (!check.ok) return jsonError(c, 422, check.reason) + const bumpFilter = body.bumpFilter === undefined ? [...BUMPS] : bumpList(body.bumpFilter) + if (!bumpFilter) + return jsonError(c, 400, '"bumpFilter" must be a non-empty list of major, minor, patch') + const secret = generateWebhookSecret() + const [created] = await c.var.ports.db + .insert(schema.collectionWebhooks) + .values({ + collectionId: access.collection.id, + url: check.url, + bumpFilter, + secret, + enabled: body.enabled !== false, + createdBy: c.var.principal?.userId ?? null, + }) + .returning(publicFields) + return c.json({ ...created, secret }, 201) + }) + + app.patch('/:owner/:slug/webhooks/:id', async (c) => { + const access = await requireAdmin(c) + if (access instanceof Response) return access + const body = (await c.req.json().catch(() => ({}))) as { + url?: unknown + bumpFilter?: unknown + enabled?: unknown + } + const set: Partial = {} + if (body.url !== undefined) { + if (typeof body.url !== 'string') return jsonError(c, 400, '"url" must be a string') + const check = validateWebhookUrl(body.url, allowInsecure(c)) + if (!check.ok) return jsonError(c, 422, check.reason) + set.url = check.url + } + if (body.bumpFilter !== undefined) { + const b = bumpList(body.bumpFilter) + if (!b) + return jsonError(c, 400, '"bumpFilter" must be a non-empty list of major, minor, patch') + set.bumpFilter = b + } + if (body.enabled !== undefined) set.enabled = body.enabled === true + if (Object.keys(set).length === 0) return c.json({ ok: true }) + const [updated] = await c.var.ports.db + .update(schema.collectionWebhooks) + .set({ ...set, updatedAt: new Date() }) + .where( + and( + eq(schema.collectionWebhooks.id, c.req.param('id')), + eq(schema.collectionWebhooks.collectionId, access.collection.id), + ), + ) + .returning(publicFields) + return updated ? c.json(updated) : jsonError(c, 404, 'Not found') + }) + + app.delete('/:owner/:slug/webhooks/:id', async (c) => { + const access = await requireAdmin(c) + if (access instanceof Response) return access + const deleted = await c.var.ports.db + .delete(schema.collectionWebhooks) + .where( + and( + eq(schema.collectionWebhooks.id, c.req.param('id')), + eq(schema.collectionWebhooks.collectionId, access.collection.id), + ), + ) + .returning({ id: schema.collectionWebhooks.id }) + return deleted.length ? c.json({ ok: true }) : jsonError(c, 404, 'Not found') + }) + + app.get('/:owner/:slug/webhooks/:id/deliveries', async (c) => { + const access = await requireAdmin(c) + if (access instanceof Response) return access + const limit = Math.min(200, Math.max(1, Number(c.req.query('limit') ?? 50) || 50)) + const deliveries = await c.var.ports.db + .select({ + id: schema.webhookDeliveries.id, + event: schema.webhookDeliveries.event, + semver: schema.webhookDeliveries.semver, + bumpType: schema.webhookDeliveries.bumpType, + status: schema.webhookDeliveries.status, + attempts: schema.webhookDeliveries.attempts, + responseCode: schema.webhookDeliveries.responseCode, + error: schema.webhookDeliveries.error, + durationMs: schema.webhookDeliveries.durationMs, + createdAt: schema.webhookDeliveries.createdAt, + deliveredAt: schema.webhookDeliveries.deliveredAt, + }) + .from(schema.webhookDeliveries) + .where( + and( + eq(schema.webhookDeliveries.webhookId, c.req.param('id')), + eq(schema.webhookDeliveries.collectionId, access.collection.id), + ), + ) + .orderBy(desc(schema.webhookDeliveries.createdAt)) + .limit(limit) + return c.json({ deliveries }) + }) + + app.post('/:owner/:slug/webhooks/:id/deliveries/:deliveryId/retry', async (c) => { + const access = await requireAdmin(c) + if (access instanceof Response) return access + const [d] = await c.var.ports.db + .select({ id: schema.webhookDeliveries.id }) + .from(schema.webhookDeliveries) + .where( + and( + eq(schema.webhookDeliveries.id, c.req.param('deliveryId')), + eq(schema.webhookDeliveries.webhookId, c.req.param('id')), + eq(schema.webhookDeliveries.collectionId, access.collection.id), + ), + ) + .limit(1) + if (!d) return jsonError(c, 404, 'Not found') + await c.var.ports.db + .update(schema.webhookDeliveries) + .set({ status: 'pending', attempts: 0 }) + .where(eq(schema.webhookDeliveries.id, d.id)) + await c.var.ports.jobs.enqueue({ type: 'webhooks.deliver', deliveryId: d.id }) + return c.json({ ok: true, status: 'pending' }) + }) + + return app +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index b6f3c44..f232267 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -10,6 +10,7 @@ import { collectionRoutes } from './api/collections.js' import { fileRoutes } from './api/files.js' import { pushRoutes } from './api/push.js' import { versionRoutes } from './api/versions.js' +import { webhookRoutes } from './api/webhooks.js' import type { Ports } from './ports.js' export interface AppConfig { @@ -75,6 +76,7 @@ export function createApp(setup: Setup) { app.route('/api/collections', fileRoutes()) app.route('/api/collections', pushRoutes()) app.route('/api/collections', versionRoutes()) + app.route('/api/collections', webhookRoutes()) app.route('/', collectionRoutes()) app.notFound((c) => c.json({ error: 'Not found', statusCode: 404 }, 404)) diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 5fc43f9..7d85d43 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -543,9 +543,9 @@ export const collectionWebhooks = sqliteTable( .notNull() .references(() => collections.id, { onDelete: 'cascade' }), url: text('url').notNull(), - bumpFilter: text('bump_filter', { enum: ['all', 'major', 'minor', 'patch'] }) + bumpFilter: json<('major' | 'minor' | 'patch')[]>('bump_filter') .notNull() - .default('all'), + .$defaultFn(() => ['major', 'minor', 'patch']), secret: text('secret').notNull(), enabled: bool('enabled').notNull().default(true), createdBy: text('created_by'), diff --git a/packages/server/src/handlers.ts b/packages/server/src/handlers.ts index d3153c5..e320df4 100644 --- a/packages/server/src/handlers.ts +++ b/packages/server/src/handlers.ts @@ -12,11 +12,19 @@ import { and, asc, eq, gt } from 'drizzle-orm' import * as schema from './db/schema.js' import './files/files.js' import { registerJob } from './jobs.js' +import { expireSessions } from './push/finalize.js' import { appendVersionLog } from './versions/commit.js' +import type { BumpType } from './versions/semver.js' +import { enqueueDeliveries, purgeOldDeliveries } from './webhooks/webhooks.js' -registerJob('version.published', async () => { - // Webhook deliveries, mirror sync and reference-log segments are added here - // as their phases land. +registerJob('version.published', async (job, ports) => { + await enqueueDeliveries(ports, String(job.versionId), job.bump as BumpType) + // Mirror sync (phase 11) and reference-log segments (phase 7) hang off here too. +}) + +registerJob('maintenance.sweep', async (_job, ports) => { + await expireSessions(ports) + await purgeOldDeliveries(ports) }) registerJob('repo.repairLog', async (job, ports) => { diff --git a/packages/server/src/node/guarded-fetch.ts b/packages/server/src/node/guarded-fetch.ts new file mode 100644 index 0000000..edd3f8f --- /dev/null +++ b/packages/server/src/node/guarded-fetch.ts @@ -0,0 +1,20 @@ +/** + * Node only: fetch for user-supplied URLs that refuses private addresses. + * + * Resolves the host first and refuses if any address is private. There is a + * window between this lookup and the connection (DNS rebinding); v1 closed it by + * pinning the address with an undici Agent, which is the next step here if Node + * deployments take untrusted webhook URLs. Workers have no private network. + */ +import { lookup } from 'node:dns/promises' + +import { ipKind, isPrivateIp } from '../webhooks/webhooks.js' + +export async function guardedFetch(url: string, init: RequestInit): Promise { + const host = new URL(url).hostname.replace(/^\[|\]$/g, '') + const addresses = ipKind(host) ? [{ address: host }] : await lookup(host, { all: true }) + if (addresses.some((a) => isPrivateIp(a.address))) { + throw new Error('Webhook host resolves to a private address') + } + return fetch(url, init) +} diff --git a/packages/server/src/node/main.ts b/packages/server/src/node/main.ts index 4129f74..c143db4 100644 --- a/packages/server/src/node/main.ts +++ b/packages/server/src/node/main.ts @@ -26,6 +26,7 @@ import { openNodeDb } from '../db/node.js' import { drainSqliteJobs, SqliteJobs } from '../jobs.js' import type { BlobStore, Ports } from '../ports.js' import { createStores } from '../stores.js' +import { guardedFetch } from './guarded-fetch.js' const env = process.env const port = Number(env.PORT ?? 4200) @@ -80,6 +81,7 @@ const ports: Ports = { kick() }, }, + outboundFetch: guardedFetch, waitUntil: (p) => { p.catch((err) => console.error('[waitUntil]', err)) }, diff --git a/packages/server/src/ports.ts b/packages/server/src/ports.ts index baecfd3..5b96147 100644 --- a/packages/server/src/ports.ts +++ b/packages/server/src/ports.ts @@ -62,6 +62,11 @@ export interface Ports { cache: Cache /** Signs version log entries (the deployment's Ed25519 key, imported once per isolate). */ signer(): Promise + /** + * fetch for user-supplied URLs (webhooks). Workers have no private network to + * reach; on Node this resolves the host and refuses private addresses. + */ + outboundFetch(url: string, init: RequestInit): Promise /** Run work after the response (Workers: ctx.waitUntil; Node: fire and forget with logging). */ waitUntil(p: Promise): void } diff --git a/packages/server/src/webhooks/webhooks.ts b/packages/server/src/webhooks/webhooks.ts new file mode 100644 index 0000000..54ceefb --- /dev/null +++ b/packages/server/src/webhooks/webhooks.ts @@ -0,0 +1,276 @@ +/** + * Webhooks: v1's payload, headers and signature, delivered by jobs. + * + * version.published job → one delivery row per enabled, matching webhook → + * one webhooks.deliver job per row; failures retry as delayed jobs with + * exponential backoff (1 min doubling, capped at 6 h), up to 5 attempts. + * + * Receivers verify `x-underlay-signature: sha256=`. + * + * SSRF: URLs are checked at registration (scheme, blocked hostnames, private IP + * literals). On Workers there is no private network to reach. On Node, the + * deployment's outbound fetch resolves and refuses private addresses + * (Ports.outboundFetch). + */ +import { createHmac, randomBytes } from 'node:crypto' + +import { and, eq, lt } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import { registerJob } from '../jobs.js' +import type { Ports } from '../ports.js' +import type { BumpType } from '../versions/semver.js' + +const DELIVERY_TIMEOUT_MS = 10_000 +export const MAX_ATTEMPTS = 5 +const BACKOFF_BASE_S = 60 +const BACKOFF_CAP_S = 6 * 60 * 60 +const RETENTION_MS = 30 * 24 * 60 * 60 * 1000 +const SIGNATURE_HEADER = 'x-underlay-signature' + +export const generateWebhookSecret = () => `ulwhsec_${randomBytes(24).toString('hex')}` +export const signPayload = (secret: string, body: string) => + `sha256=${createHmac('sha256', secret).update(body).digest('hex')}` + +// --- SSRF checks (ported from v1) --------------------------------------------------- + +const IPV4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/ + +function parseIPv6(ip: string): number[] | null { + let s = ip.toLowerCase() + const zone = s.indexOf('%') + if (zone !== -1) s = s.slice(0, zone) + const dotted = s.match(/(\d+)\.(\d+)\.(\d+)\.(\d+)$/) + if (dotted) { + const [a = 0, b = 0, c = 0, d = 0] = dotted.slice(1).map((n) => parseInt(n, 10)) + s = `${s.slice(0, -dotted[0].length)}${((a << 8) | b).toString(16)}:${((c << 8) | d).toString(16)}` + } + const halves = s.split('::') + if (halves.length > 2) return null + const head = halves[0] ? halves[0].split(':') : [] + const tail = halves[1] ? halves[1].split(':') : [] + const fill = 8 - head.length - tail.length + if (halves.length === 1 ? fill !== 0 : fill < 0) return null + const groups = [...head, ...Array(fill).fill('0'), ...tail].map((g) => parseInt(g, 16)) + return groups.length === 8 && groups.every((g) => Number.isInteger(g) && g >= 0 && g <= 0xffff) + ? groups + : null +} + +export function ipKind(s: string): 0 | 4 | 6 { + const m = IPV4.exec(s) + if (m) return m.slice(1).every((n) => Number(n) <= 255) ? 4 : 0 + return s.includes(':') && parseIPv6(s) ? 6 : 0 +} + +/** Private, loopback, link-local, CGNAT, benchmarking, multicast, reserved, NAT64 and embedded IPv4. */ +export function isPrivateIp(ip: string): boolean { + const kind = ipKind(ip) + if (kind === 4) { + const [a = 0, b = 0] = ip.split('.').map((n) => parseInt(n, 10)) + if (a === 10 || a === 127 || a === 0) return true + if (a === 169 && b === 254) return true + if (a === 172 && b >= 16 && b <= 31) return true + if (a === 192 && b === 168) return true + if (a === 100 && b >= 64 && b <= 127) return true + if (a === 198 && (b === 18 || b === 19)) return true + return a >= 224 + } + if (kind === 6) { + const g = parseIPv6(ip) + if (!g) return true + const [g0 = 0, g1 = 0, g2 = 0, g3 = 0, g4 = 0, g5 = 0, g6 = 0, g7 = 0] = g + if (g.slice(0, 7).every((x) => x === 0) && g7 <= 1) return true + if ((g0 & 0xffc0) === 0xfe80) return true + if ((g0 & 0xfe00) === 0xfc00) return true + if ((g0 & 0xff00) === 0xff00) return true + if (g0 === 0x64 && g1 === 0xff9b && !g2 && !g3 && !g4 && !g5) return true + if (!g0 && !g1 && !g2 && !g3 && !g4 && (g5 === 0xffff || g5 === 0)) { + return isPrivateIp(`${g6 >> 8}.${g6 & 0xff}.${g7 >> 8}.${g7 & 0xff}`) + } + return false + } + return false +} + +const bareHost = (h: string) => (h.startsWith('[') && h.endsWith(']') ? h.slice(1, -1) : h) + +export function validateWebhookUrl( + raw: string, + allowInsecure: boolean, +): { ok: true; url: string } | { ok: false; reason: string } { + let parsed: URL + try { + parsed = new URL(raw) + } catch { + return { ok: false, reason: 'Invalid URL' } + } + if (parsed.protocol !== 'https:' && !(allowInsecure && parsed.protocol === 'http:')) { + return { ok: false, reason: 'Webhook URL must use https' } + } + const host = bareHost(parsed.hostname).toLowerCase().replace(/\.$/, '') + if ( + host === 'localhost' || + host.endsWith('.localhost') || + host.endsWith('.local') || + host.endsWith('.internal') + ) { + return { ok: false, reason: 'Webhook URL host is not allowed' } + } + if (ipKind(host) && isPrivateIp(host)) + return { ok: false, reason: 'Webhook URL resolves to a private address' } + return { ok: true, url: parsed.toString() } +} + +// --- Enqueue and deliver ---------------------------------------------------------------- + +/** After a version is published: a delivery row and a job per matching webhook. */ +export async function enqueueDeliveries( + ports: Ports, + versionId: string, + bump: BumpType, +): Promise { + const { db } = ports + const [row] = await db + .select({ v: schema.versions, c: schema.collections, o: schema.organization }) + .from(schema.versions) + .innerJoin(schema.collections, eq(schema.collections.id, schema.versions.collectionId)) + .innerJoin(schema.organization, eq(schema.organization.id, schema.collections.organizationId)) + .where(eq(schema.versions.id, versionId)) + .limit(1) + if (!row) return 0 + const hooks = ( + await db + .select() + .from(schema.collectionWebhooks) + .where( + and( + eq(schema.collectionWebhooks.collectionId, row.c.id), + eq(schema.collectionWebhooks.enabled, true), + ), + ) + ).filter((h) => h.bumpFilter.includes(bump)) + if (hooks.length === 0) return 0 + // Idempotent per (webhook, version): a retried job doesn't double-deliver. + const existing = await db + .select({ webhookId: schema.webhookDeliveries.webhookId }) + .from(schema.webhookDeliveries) + .where(eq(schema.webhookDeliveries.versionId, versionId)) + const done = new Set(existing.map((e) => e.webhookId)) + const payload = { + event: 'version.created', + collection: { owner: row.o.slug, slug: row.c.slug }, + version: { + semver: row.v.semver, + hash: row.v.hash, + major: row.v.major, + minor: row.v.minor, + patch: row.v.patch, + recordCount: row.v.recordCount, + fileCount: row.v.fileCount, + }, + bumpType: bump, + } + const fresh = hooks.filter((h) => !done.has(h.id)) + if (fresh.length === 0) return 0 + const rows = await db + .insert(schema.webhookDeliveries) + .values( + fresh.map((h) => ({ + webhookId: h.id, + collectionId: row.c.id, + versionId, + semver: row.v.semver, + bumpType: bump, + payload, + })), + ) + .returning({ id: schema.webhookDeliveries.id }) + await ports.jobs.enqueueBatch(rows.map((r) => ({ type: 'webhooks.deliver', deliveryId: r.id }))) + return rows.length +} + +export async function deliver(ports: Ports, deliveryId: string): Promise { + const { db } = ports + const [row] = await db + .select({ d: schema.webhookDeliveries, h: schema.collectionWebhooks }) + .from(schema.webhookDeliveries) + .innerJoin( + schema.collectionWebhooks, + eq(schema.collectionWebhooks.id, schema.webhookDeliveries.webhookId), + ) + .where(eq(schema.webhookDeliveries.id, deliveryId)) + .limit(1) + if (!row || row.d.status === 'success') return + const attempt = row.d.attempts + 1 + const body = JSON.stringify({ + ...row.d.payload, + delivery: { id: row.d.id, timestamp: new Date().toISOString() }, + }) + + const record = async ( + ok: boolean, + responseCode: number | null, + error: string | null, + durationMs: number, + terminal = false, + ) => { + const exhausted = !ok && (terminal || attempt >= MAX_ATTEMPTS) + const delay = Math.min(BACKOFF_BASE_S * 2 ** (attempt - 1), BACKOFF_CAP_S) + await db + .update(schema.webhookDeliveries) + .set({ + status: ok ? 'success' : 'failed', + attempts: attempt, + responseCode, + error: error?.slice(0, 2000) ?? null, + durationMs, + nextAttemptAt: ok || exhausted ? null : new Date(Date.now() + delay * 1000), + deliveredAt: new Date(), + }) + .where(eq(schema.webhookDeliveries.id, deliveryId)) + if (ok) { + await db + .update(schema.collectionWebhooks) + .set({ lastDeliveryAt: new Date() }) + .where(eq(schema.collectionWebhooks.id, row.h.id)) + } else if (!exhausted) { + await ports.jobs.enqueue({ type: 'webhooks.deliver', deliveryId }, { delaySeconds: delay }) + } + } + + if (!row.h.enabled) return record(false, null, 'Webhook disabled', 0, true) + const started = Date.now() + try { + const res = await ports.outboundFetch(row.h.url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'user-agent': 'Underlay-Webhook/2.0', + 'x-underlay-event': row.d.event, + 'x-underlay-delivery': row.d.id, + [SIGNATURE_HEADER]: signPayload(row.h.secret, body), + }, + body, + signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS), + redirect: 'manual', + }) + void res.body?.cancel().catch(() => {}) + await record(res.ok, res.status, res.ok ? null : `HTTP ${res.status}`, Date.now() - started) + } catch (err) { + await record( + false, + null, + err instanceof Error ? err.message : String(err), + Date.now() - started, + ) + } +} + +export async function purgeOldDeliveries(ports: Ports): Promise { + await ports.db + .delete(schema.webhookDeliveries) + .where(lt(schema.webhookDeliveries.createdAt, new Date(Date.now() - RETENTION_MS))) +} + +registerJob('webhooks.deliver', async (job, ports) => deliver(ports, String(job.deliveryId))) diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index 4b21a19..9795e99 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -62,6 +62,7 @@ function makePorts(env: Env, ctx: ExecutionContext): Ports { signer: () => (signer ??= ed25519Signer(env.SIGNING_KEY)), jobs: new QueueJobs(env.JOBS as never), waitUntil: (p) => ctx.waitUntil(p), + outboundFetch: (url, init) => fetch(url, init), } } diff --git a/packages/server/test/db.test.ts b/packages/server/test/db.test.ts index 24c8675..8be21ca 100644 --- a/packages/server/test/db.test.ts +++ b/packages/server/test/db.test.ts @@ -69,6 +69,7 @@ describe('SQLite jobs', () => { internalPrefix: 'internal', }), cache: new MemoryCache(), + outboundFetch: fetch, signer: async () => { throw new Error('not used') }, diff --git a/packages/server/test/harness.ts b/packages/server/test/harness.ts index ac306c3..a0e262b 100644 --- a/packages/server/test/harness.ts +++ b/packages/server/test/harness.ts @@ -18,6 +18,17 @@ import { createStores } from '../src/stores.js' const dirs: string[] = [] +/** Outbound requests made by the app (webhooks), and the responder tests set. */ +export const outboundCalls: { url: string; init: RequestInit }[] = [] +export let outboundResponder: (url: string) => Response = () => new Response('ok') +export function respondOutbound(f: (url: string) => Response) { + outboundResponder = f +} +const outbound = async (url: string, init: RequestInit) => { + outboundCalls.push({ url, init }) + return outboundResponder(url) +} + export async function cleanup(): Promise { for (const d of dirs.splice(0)) await rm(d, { recursive: true, force: true }) } @@ -55,6 +66,7 @@ export async function harness(): Promise { signer: async () => signer, jobs: new SqliteJobs(db), waitUntil: (p) => void p.catch((err) => console.error(err)), + outboundFetch: (url, init) => outbound(url, init), } let orgMade = false const ensureOrg = async () => { diff --git a/packages/server/test/webhooks.test.ts b/packages/server/test/webhooks.test.ts new file mode 100644 index 0000000..fb475c8 --- /dev/null +++ b/packages/server/test/webhooks.test.ts @@ -0,0 +1,129 @@ +import { createHmac } from 'node:crypto' + +import { afterAll, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { isPrivateIp, validateWebhookUrl } from '../src/webhooks/webhooks.js' +import { cleanup, harness, outboundCalls, respondOutbound } from './harness.js' + +afterAll(cleanup) + +const Author = { type: 'object', properties: { name: { type: 'string' } } } + +async function json(res: Response) { + return (await res.json()) as any +} + +describe('webhook URL checks', () => { + it('refuses private and local targets', () => { + for (const u of [ + 'http://example.org', + 'https://localhost/x', + 'https://10.0.0.1/', + 'https://[::1]/', + 'https://169.254.169.254/', + 'https://svc.internal/', + ]) { + expect(validateWebhookUrl(u, false).ok).toBe(false) + } + expect(validateWebhookUrl('https://hooks.example.org/underlay', false).ok).toBe(true) + expect(isPrivateIp('::ffff:192.168.1.1')).toBe(true) + expect(isPrivateIp('8.8.8.8')).toBe(false) + }) +}) + +describe('webhooks', () => { + it('delivers signed version.created events after a push, and retries failures', async () => { + const h = await harness() + const user = await h.member() + await h.collection('hooked') + const base = '/api/collections/org/hooked' + let res = await h.request(`${base}/webhooks`, { + method: 'POST', + user, + json: { url: 'https://hooks.example.org/u', bumpFilter: ['major', 'minor'] }, + }) + expect(res.status).toBe(201) + const hook = await json(res) + expect(hook.secret).toMatch(/^ulwhsec_/) + const listed = await json(await h.request(`${base}/webhooks`, { user })) + expect(listed.webhooks[0]).not.toHaveProperty('secret') + + outboundCalls.length = 0 + let fail = true + respondOutbound(() => (fail ? new Response('nope', { status: 500 }) : new Response('ok'))) + + const sid = ( + await json( + await h.request(`${base}/push`, { method: 'POST', user, json: { schemas: { Author } } }), + ) + ).session_id + await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [{ id: 'a', type: 'Author', data: { name: 'A' } }], + }) + expect((await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user })).status).toBe( + 201, + ) + await h.drain() + expect(outboundCalls.length).toBe(1) + const call = outboundCalls[0]! + const body = String(call.init.body) + const headers = call.init.headers as Record + expect(headers['x-underlay-signature']).toBe( + `sha256=${createHmac('sha256', hook.secret).update(body).digest('hex')}`, + ) + expect(JSON.parse(body)).toMatchObject({ + event: 'version.created', + collection: { owner: 'org', slug: 'hooked' }, + version: { semver: 'v1.0.0' }, + bumpType: 'major', + }) + + // The failure was recorded and a retry queued with a delay. + let deliveries = ( + await json(await h.request(`${base}/webhooks/${hook.id}/deliveries`, { user })) + ).deliveries + expect(deliveries[0]).toMatchObject({ status: 'failed', attempts: 1, responseCode: 500 }) + fail = false + await h.ports.db.update(schema.jobs).set({ runAt: new Date(0) }) + await h.drain() + deliveries = (await json(await h.request(`${base}/webhooks/${hook.id}/deliveries`, { user }))) + .deliveries + expect(deliveries[0]).toMatchObject({ status: 'success', attempts: 2 }) + + // A patch bump doesn't match the filter. + const meta = ( + await json( + await h.request(`${base}/push`, { + method: 'POST', + user, + json: { base: 'v1.0.0', metadata_patch: { title: 'x' } }, + }), + ) + ).session_id + expect((await h.request(`${base}/push/${meta}/commit`, { method: 'POST', user })).status).toBe( + 201, + ) + outboundCalls.length = 0 + await h.drain() + expect(outboundCalls.length).toBe(0) + }) + + it('is for org owners and admins only', async () => { + const h = await harness() + await h.member('owner1') + await h.collection('hooked') + await h.ports.db.insert(schema.user).values({ id: 'm2', name: 'm', email: 'm2@example.org' }) + await h.ports.db + .insert(schema.member) + .values({ organizationId: 'org1', userId: 'm2', role: 'member' }) + expect((await h.request('/api/collections/org/hooked/webhooks', { user: 'm2' })).status).toBe( + 403, + ) + expect( + (await h.request('/api/collections/org/hooked/webhooks', { user: 'owner1' })).status, + ).toBe(200) + }) +}) From 2008e1810faeddd2f18693c975599e074b21c05f Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 17:16:12 -0400 Subject: [PATCH 012/178] v2 server: renderPage hook for SSR with an in-process API fetch --- packages/server/src/app.ts | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index f232267..202403b 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -3,7 +3,7 @@ * that builds the ports, config and authenticator for a request, and everything * below reads them from the context. */ -import { type Context, Hono } from 'hono' +import { type Context, type ExecutionContext, Hono } from 'hono' import type { Principal } from './api/access.js' import { collectionRoutes } from './api/collections.js' @@ -44,8 +44,19 @@ export type Setup = (c: Context) => { authenticate: Authenticate /** better-auth's own routes (/api/auth/*): sign-in, callbacks, sessions, keys, orgs. */ authHandler?: (req: Request) => Promise + /** + * Server-side rendering of UI pages (packages/web). `api` calls this app + * in-process: a Worker can't fetch its own zone (build doc finding 9), and on + * Node it saves a loopback round trip. + */ + renderPage?: RenderPage } +export type RenderPage = ( + req: Request, + api: (req: Request) => Promise, +) => Promise + export function createApp(setup: Setup) { const app = new Hono() @@ -79,6 +90,21 @@ export function createApp(setup: Setup) { app.route('/api/collections', webhookRoutes()) app.route('/', collectionRoutes()) + // Everything else is a UI page, when the deployment renders one. + app.get('*', async (c) => { + const { renderPage } = setup(c) + if (!renderPage || c.req.path.startsWith('/api/')) + return c.json({ error: 'Not found', statusCode: 404 }, 404) + // Hono throws reading executionCtx where there is none (Node). + let ctx: ExecutionContext | undefined + try { + ctx = c.executionCtx + } catch { + ctx = undefined + } + return renderPage(c.req.raw, async (req) => app.fetch(req, c.env, ctx)) + }) + app.notFound((c) => c.json({ error: 'Not found', statusCode: 404 }, 404)) app.onError((err, c) => { console.error('[app]', err) From 46f301fc6ce71ce5c221503ad5011ce1afa38e61 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 17:19:12 -0400 Subject: [PATCH 013/178] v2: validator swap (phase 2), rank/offset seeks, collection management Phase 2: @underlay/core validates with @cfworker/json-schema (no eval or new Function), adjusted to match v1's AJV + ajv-formats: formats ported from ajv-formats, keywords next to $ref applied, later-draft keywords ignored, meta-schema and $ref checks at compile time, AJV's messages, prototype-free data. The differential test (scripts/diff-validators.ts) over all public production data agrees on all 139 schemas, 330,300 records and 85,773 mutated records. checkSchema also refuses field-level privacy at any depth and non-boolean root private. Spec section 5.1 now fixes the dialect. Core: rankOf and entryAt (O(height) seeks by key and by position). Server: create, update, delete, transfer and fork collections, and metadata edits as patch versions reusing every set. A fork is a new root over the source's sets plus a forks row; owners keep the private set and its salt. --- docs/protocol-v2.md | 55 +- packages/core/scripts/diff-validators.ts | 855 +++++++++++++++++++++++ packages/core/src/index.ts | 20 +- packages/core/src/tree/read.ts | 59 ++ packages/core/src/validate.ts | 793 +++++++++++++++++++++ packages/core/test/tree.test.ts | 19 + packages/core/test/validate.test.ts | 435 ++++++++++++ packages/server/src/api/manage.ts | 253 +++++++ packages/server/src/app.ts | 2 + packages/server/src/versions/fork.ts | 139 ++++ packages/server/test/manage.test.ts | 176 +++++ 11 files changed, 2800 insertions(+), 6 deletions(-) create mode 100644 packages/core/scripts/diff-validators.ts create mode 100644 packages/core/src/validate.ts create mode 100644 packages/core/test/validate.test.ts create mode 100644 packages/server/src/api/manage.ts create mode 100644 packages/server/src/versions/fork.ts create mode 100644 packages/server/test/manage.test.ts diff --git a/docs/protocol-v2.md b/docs/protocol-v2.md index 287be2f..7a0ac5e 100644 --- a/docs/protocol-v2.md +++ b/docs/protocol-v2.md @@ -83,11 +83,56 @@ usual. A type's schema is a JSON Schema document. **Schema hash** = hash(JCS(schema)). -- A schema with `"private": true` at its root makes the type private (section 9). -- `"private": true` on a property (field-level privacy) is **rejected** in format 2. -- The validation dialect, and the exact behaviour required of a validator, is **to be specified** - in phase 2 of the build, after the differential test against production data. Until then, the - reference validator is the server's. +- A schema with `"private": true` at its root makes the type private (section 9). A root `private` + that isn't a boolean is rejected, so that `"private": "true"` can't publish a type by accident. +- `"private": true` on a property, at any depth (field-level privacy), is **rejected** in format 2. +- Limits: the schema's canonical form must be at most 256 KB, and `pattern` values and + `patternProperties` keys at most 256 characters each. + +### 5.1 Validation dialect + +**Which schemas are accepted.** A type schema must be a JSON object and a JSON Schema draft-07 +document. +- A root `$schema`, if present, must be `http://json-schema.org/draft-07/schema` (with or without a + trailing `#`); anything else is rejected. +- A schema is rejected unless all of these hold: + - it is valid against the draft-07 meta-schema; + - every `pattern` and `patternProperties` key compiles as an ECMAScript regular expression with + the `u` flag; + - every `$ref` resolves within the schema or to the draft-07 meta-schema. +- `$ref`s are resolved against the base URI `https://schema.underlay.invalid/` unless a `$id` sets + another. + +**How records are validated.** Draft-07, with these rules: +- Keywords alongside `$ref` are applied, as in draft 2019-09. +- Keywords draft-07 doesn't define are ignored. That includes later drafts' keywords + (`unevaluatedProperties`, `dependentRequired`, `prefixItems`, …) and draft-04's `id`. `$defs` + works as a container, and `$anchor` is honoured. +- `pattern` uses ECMAScript regular expressions with the `u` flag. +- `multipleOf` m accepts x when the floating-point remainder r = x mod m satisfies + |r| < 1.1920929e-7 or |m − r| < 1.1920929e-7. +- String lengths count Unicode code points. +- Object members are the parsed JSON's own keys. Names such as `__proto__` or `toString` carry no + special meaning. + +**Formats.** +- `format` constrains strings only, and only for these names: `date`, `time`, `date-time`, + `iso-time`, `iso-date-time`, `duration`, `uri`, `uri-reference`, `uri-template`, `url`, `email`, + `hostname`, `ipv4`, `ipv6`, `regex`, `uuid`, `json-pointer`, `json-pointer-uri-fragment`, + `relative-json-pointer`, `byte`. +- Each is defined as in ajv-formats 3.0 "full" mode (`packages/core/src/validate.ts`). In + particular: + - `date-time` and `time` require a time zone; + - `date-time` accepts `T`, `t` or whitespace as the separator; + - `email` requires a dot in the domain. +- Other format names are ignored. + +**What is normative.** Only the verdict. Error messages, and how many are reported, are not. + +The reference validator is `@cfworker/json-schema` with these rules applied +(`packages/core/src/validate.ts`). Over all public production data it agrees with format 1's AJV +configuration: 139 schemas, 330,300 records, and 85,773 mutated records +(`scripts/diff-validators.ts`). ## 6. Files diff --git a/packages/core/scripts/diff-validators.ts b/packages/core/scripts/diff-validators.ts new file mode 100644 index 0000000..6fa9939 --- /dev/null +++ b/packages/core/scripts/diff-validators.ts @@ -0,0 +1,855 @@ +/** + * Differential test: v1's validator (AJV, configured exactly as the v1 server) + * against the v2 validator (src/validate.ts), over every public schema and + * record on underlay.org. @hyperjump/json-schema (draft-07) runs alongside as a + * second interpreting validator. + * + * npx tsx packages/core/scripts/diff-validators.ts [--data ] [--offline] + * + * Three passes: + * 1. every distinct schema of every version is compiled by all three; + * 2. every record of each collection's latest version is validated; + * 3. production records only exercise the "valid" path (v1 refused the rest + * at push time), so a sample of each type's records is mutated against its + * schema (wrong types, missing required fields, extra fields, bad format + * and enum values) and validated again; and a corpus of edge-case strings + * is run through every format keyword. + * + * Downloads are read-only GETs to the public API, sent one at a time under the + * anonymous rate limit, and cached under --data so reruns are offline. + */ +import { + createReadStream, + createWriteStream, + existsSync, + mkdirSync, + readFileSync, + renameSync, + writeFileSync, +} from 'node:fs' +import { join } from 'node:path' +import { createInterface } from 'node:readline' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' + +import { Ajv } from 'ajv' +import addFormatsModule from 'ajv-formats' + +import { compileSchema, SchemaError } from '../src/validate.js' + +const API = 'https://www.underlay.org/api' +const args = process.argv.slice(2) +const option = (name: string, fallback: string) => { + const i = args.indexOf(name) + return i >= 0 && args[i + 1] ? args[i + 1]! : fallback +} +const DATA = option('--data', '/Users/travis/.claude-kf/jobs/71495a02/tmp/validator-data') +const OFFLINE = args.includes('--offline') +/** Validate only the mutation sample of production records (for iterating). */ +const QUICK = args.includes('--quick') +/** Records per type that pass 3 mutates. */ +const MUTATE_SAMPLE = 20 +const MUTATE_MAX_BYTES = 64 * 1024 +const MAX_LISTED = 40 + +// --- Download (cached) ---------------------------------------------------------- + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +let lastRequest = 0 + +async function get(url: string): Promise { + if (OFFLINE) throw new Error(`--offline and not cached: ${url}`) + // One request at a time, under the anonymous limit of 60 a minute: this is a + // production site. + for (let attempt = 0; ; attempt++) { + const wait = lastRequest + 1100 - Date.now() + if (wait > 0) await sleep(wait) + lastRequest = Date.now() + const res = await fetch(url, { + headers: { 'user-agent': 'underlay-validator-diff (read-only)' }, + }) + if (res.status === 429 && attempt < 5) { + await sleep(1000 * (Number(res.headers.get('retry-after')) || 60)) + continue + } + if (!res.ok) throw new Error(`GET ${url}: ${res.status}`) + return res + } +} + +async function cachedJson(file: string, url: string): Promise { + const path = join(DATA, file) + if (existsSync(path)) return JSON.parse(readFileSync(path, 'utf8')) as T + const body = await (await get(url)).text() + mkdirSync(join(path, '..'), { recursive: true }) + writeFileSync(path, body) + return JSON.parse(body) as T +} + +async function cachedFile(file: string, url: string): Promise { + const path = join(DATA, file) + if (existsSync(path)) return path + const res = await get(url) + mkdirSync(join(path, '..'), { recursive: true }) + await pipeline(Readable.fromWeb(res.body as never), createWriteStream(path + '.part')) + renameSync(path + '.part', path) + return path +} + +interface CollectionRow { + slug: string + ownerSlug: string + latestVersion: string | null +} + +async function listCollections(): Promise { + const out: CollectionRow[] = [] + const limit = 100 + for (let page = 0; ; page++) { + const body = await cachedJson<{ collections: CollectionRow[] }>( + `collections-p${page}.json`, + `${API}/collections?limit=${limit}&offset=${page * limit}`, + ) + out.push(...body.collections) + if (body.collections.length < limit) break + } + return out +} + +// --- The three validators --------------------------------------------------------- + +/** A compiled validator returning error strings (empty = valid), or the compile error. */ +type Compiled = ((data: unknown) => string[]) | string + +// v1, exactly: src/lib/core/validate.ts at the repo root. +const addFormats = addFormatsModule as unknown as (ajv: Ajv) => void +const ajv = new Ajv({ allErrors: true, strict: false }) +addFormats(ajv) +// AJV warns about (and ignores) unknown formats; keep the warnings out of the report. +const ajvWarnings = new Set() +;(ajv as unknown as { logger: unknown }).logger = { + log: () => {}, + warn: (msg: unknown) => ajvWarnings.add(String(msg)), + error: console.error, +} + +function ajvCompile(schema: unknown): Compiled { + try { + const v = ajv.compile(schema as object) + ajv.removeSchema(schema as object) + return (data) => + v(data) ? [] : (v.errors ?? []).map((e) => `${e.instancePath || '/'} ${e.message}`) + } catch (err) { + return (err as Error).message + } +} + +function v2Compile(schema: unknown): Compiled { + try { + return compileSchema(schema) + } catch (err) { + return err instanceof SchemaError ? err.message : `THROWN: ${(err as Error).message}` + } +} + +// @hyperjump is async to compile and keeps a global registry keyed by URI. +let hj: typeof import('@hyperjump/json-schema/draft-07') | null = null +let hjCounter = 0 +async function hjCompile(schema: unknown): Promise { + if (!hj) { + hj = await import('@hyperjump/json-schema/draft-07') + // Side effect only (registers the format handlers); it ships no types. + const formats = '@hyperjump/json-schema/formats' + await import(formats) + hj.setShouldValidateFormat(true) + } + const uri = `https://diff.invalid/schema/${hjCounter++}` + try { + hj.registerSchema( + structuredClone(schema) as never, + uri, + 'http://json-schema.org/draft-07/schema', + ) + const v = await hj.validate(uri) + return (data) => (v(data as never).valid ? [] : ['invalid']) + } catch (err) { + return (err as Error).message.slice(0, 300) + } +} + +interface Trio { + ajv: Compiled + v2: Compiled + hj: Compiled +} + +async function compileAll(schema: unknown): Promise { + return { ajv: ajvCompile(schema), v2: v2Compile(schema), hj: await hjCompile(schema) } +} + +// --- Comparison ------------------------------------------------------------------- + +const sameSet = (a: string[], b: string[]) => { + const x = [...a].sort() + const y = [...b].sort() + return x.length === y.length && x.every((v, i) => v === y[i]) +} + +class Tally { + checked = 0 + agreeValid = 0 + agreeInvalid = 0 + sameMessages = 0 + /** v2 reported some of AJV's messages: it short-circuits, AJV ran with allErrors. */ + subsetMessages = 0 + hjChecked = 0 + hjAgree = 0 + disagreements: string[] = [] + messageDiffs: string[] = [] + hjDisagreements: string[] = [] + /** Schemas both refused to compile, with both messages. */ + bothRejected: string[] = [] + + constructor(readonly name: string) {} + + compare(label: string, t: Trio, data: unknown): void { + if (typeof t.ajv === 'string' || typeof t.v2 === 'string') return // reported at compile + this.checked++ + const ae = t.ajv(data) + const be = t.v2(data) + const aValid = ae.length === 0 + const bValid = be.length === 0 + const show = (e: string[]) => (e.length ? JSON.stringify(e) : 'valid') + if (aValid !== bValid) { + this.disagreements.push(` ${label}\n ajv: ${show(ae)}\n v2: ${show(be)}`) + } else if (aValid) { + this.agreeValid++ + } else { + this.agreeInvalid++ + if (sameSet(ae, be)) this.sameMessages++ + else if (be.every((m) => ae.includes(m))) this.subsetMessages++ + else this.messageDiffs.push(` ${label}\n ajv: ${show(ae)}\n v2: ${show(be)}`) + } + if (typeof t.hj !== 'string') { + this.hjChecked++ + const hValid = t.hj(data).length === 0 + if (hValid === aValid) this.hjAgree++ + else + this.hjDisagreements.push( + ` ${label}: ajv ${show(ae)}, hyperjump ${hValid ? 'valid' : 'invalid'}`, + ) + } + } + + print(): void { + const list = (lines: string[]) => { + for (const line of lines.slice(0, MAX_LISTED)) console.log(line) + if (lines.length > MAX_LISTED) console.log(` … and ${lines.length - MAX_LISTED} more`) + } + console.log(`\n${this.name}: ${this.checked} checked`) + console.log(` agree valid: ${this.agreeValid}`) + console.log( + ` agree invalid: ${this.agreeInvalid} (identical messages: ${this.sameMessages}, ` + + `v2's a subset of AJV's: ${this.subsetMessages})`, + ) + console.log(` disagree: ${this.disagreements.length}`) + list(this.disagreements) + if (this.messageDiffs.length) { + console.log(` verdicts agree, messages differ: ${this.messageDiffs.length}`) + list(this.messageDiffs) + } + if (this.bothRejected.length) { + console.log(` schemas both refused to compile: ${this.bothRejected.length}`) + list(this.bothRejected) + } + console.log(` hyperjump agrees with ajv on ${this.hjAgree}/${this.hjChecked}`) + list(this.hjDisagreements) + } +} + +// --- Mutations (pass 3) ----------------------------------------------------------- + +const FORMAT_CORPUS = [ + '', + 'x', + '2020-01-01', + '2020-02-29', + '2021-02-29', + '2020-13-01', + '20200101', + '12:00:00', + '12:00:00Z', + '12:00:00+01:00', + '23:59:60Z', + '2020-01-01T00:00:00', + '2020-01-01T00:00:00Z', + '2020-01-01T00:00:00.123Z', + '2020-01-01t00:00:00z', + '2020-01-01 00:00:00Z', + '2020-01-01T00:00:00+0100', + '2020-01-01T00:00:00+01:00', + '2020-01-01T00:00:00+25:00', + '2020-01-01T00:00:00+01:99', + '2020-12-31T23:59:60Z', + '2020-12-31T22:59:60-01:00', + '2020-12-31T12:59:60Z', + '2020-02-30T00:00:00Z', + 'a@b', + 'a@b.co', + 'a.b+c@example.org', + '"a b"@example.org', + 'a..b@example.org', + '.a@example.org', + 'a@-example.org', + 'a@exa_mple.org', + `${'a'.repeat(65)}@example.org`, + `a@${'b'.repeat(64)}.org`, + 'user@[127.0.0.1]', + 'http://example.com', + 'https://example.com/a b', + 'https://example.com/a%20b?q=1#f', + 'https://例子.测试/', + 'HTTP://EXAMPLE.COM/%zz', + 'example.com', + '/relative/path', + '#frag', + 'urn:isbn:0451450523', + 'mailto:a@example.org', + 'doi:10.1000/182', + 'http://[::1]/', + 'ftp://ftp.example.com/file', + '550e8400-e29b-41d4-a716-446655440000', + 'urn:uuid:550e8400-e29b-41d4-a716-446655440000', + '550E8400-E29B-41D4-A716-446655440000', + '550e8400e29b41d4a716446655440000', + '127.0.0.1', + '001.002.003.004', + '256.0.0.1', + '::1', + 'example', + 'ex_ample.com', + '-example.com', + 'P1D', + 'P1.5D', + 'PT', + '/a/b~0c', + '/a~2', + '0/a', + '^[a-z]+$', + '(', + '\\p{L}', + 'a\\Z', + 'QUJD', + 'QUJ', +] +const ALL_FORMATS = [ + 'date', + 'time', + 'date-time', + 'iso-time', + 'iso-date-time', + 'duration', + 'uri', + 'uri-reference', + 'uri-template', + 'url', + 'email', + 'hostname', + 'ipv4', + 'ipv6', + 'regex', + 'uuid', + 'json-pointer', + 'json-pointer-uri-fragment', + 'relative-json-pointer', + 'byte', + 'int32', + 'int64', + 'float', + 'double', + 'password', + 'binary', +] + +const isObject = (v: unknown): v is Record => + v !== null && typeof v === 'object' && !Array.isArray(v) + +function wrongType(v: unknown): unknown { + if (typeof v === 'string') return 12345 + if (typeof v === 'number') return Number.isInteger(v) ? 1.5 : 'x' + if (Array.isArray(v)) return {} + if (isObject(v)) return [] + return 'x' +} + +/** Labelled variants of `data` that probe the keywords its schema uses. */ +function* mutations(schema: unknown, data: unknown): Generator<[string, unknown]> { + if (!isObject(data)) { + yield ['wrong type', wrongType(data)] + return + } + yield ['extra field', { ...data, __extra: 1 }] + yield ['not an object', [data]] + if (!isObject(schema)) return + for (const k of Array.isArray(schema.required) ? (schema.required as string[]) : []) { + if (!(k in data)) continue + const copy = { ...data } + delete copy[k] + yield [`drop required ${k}`, copy] + } + const props = isObject(schema.properties) ? schema.properties : {} + for (const [k, sub] of Object.entries(props)) { + if (!(k in data)) continue + yield [`${k}: wrong type`, { ...data, [k]: wrongType(data[k]) }] + yield [`${k}: null`, { ...data, [k]: null }] + if (!isObject(sub)) continue + if (typeof sub.format === 'string' || typeof sub.pattern === 'string') { + for (const s of FORMAT_CORPUS) yield [`${k}: ${JSON.stringify(s)}`, { ...data, [k]: s }] + } + if (Array.isArray(sub.enum)) yield [`${k}: not in enum`, { ...data, [k]: '__not_in_enum__' }] + const v = data[k] + if (Array.isArray(v) && v.length > 0) { + yield [`${k}[0]: wrong type`, { ...data, [k]: [wrongType(v[0]), ...v.slice(1)] }] + if (isObject(v[0])) { + for (const ik of Object.keys(v[0])) { + const item = { ...v[0], [ik]: wrongType(v[0][ik]) } + yield [`${k}[0].${ik}: wrong type`, { ...data, [k]: [item, ...v.slice(1)] }] + } + } + } + if (isObject(v)) { + for (const ik of Object.keys(v)) { + yield [`${k}.${ik}: wrong type`, { ...data, [k]: { ...v, [ik]: wrongType(v[ik]) } }] + } + yield [`${k}: extra field`, { ...data, [k]: { ...v, __extra: 1 } }] + } + } +} + +// --- Survey ----------------------------------------------------------------------- + +const LATER_KEYWORDS = new Set([ + '$recursiveRef', + '$recursiveAnchor', + '$anchor', + '$defs', + '$vocabulary', + '$dynamicRef', + '$dynamicAnchor', + 'unevaluatedProperties', + 'unevaluatedItems', + 'dependentRequired', + 'dependentSchemas', + 'prefixItems', + 'minContains', + 'maxContains', +]) +const DATA_KEYWORDS = new Set(['enum', 'const', 'default', 'examples', 'required']) +const MAP_KEYWORDS = new Set([ + 'properties', + 'patternProperties', + 'definitions', + '$defs', + 'dependencies', +]) + +const count = (m: Map, k: string) => m.set(k, (m.get(k) ?? 0) + 1) + +interface Survey { + $schema: Map + formats: Map + keywords: Map + refs: Map + refWithSiblings: number + privateFields: number +} + +function surveySchema(node: unknown, s: Survey, root = true): void { + if (Array.isArray(node)) { + for (const item of node) surveySchema(item, s, false) + return + } + if (!isObject(node)) return + if (root) count(s.$schema, typeof node.$schema === 'string' ? node.$schema : '(none)') + if (typeof node.format === 'string') count(s.formats, node.format) + if (typeof node.$ref === 'string') { + const r = node.$ref + count( + s.refs, + r.startsWith('#/definitions/') + ? '#/definitions/…' + : r.startsWith('#/$defs/') + ? '#/$defs/…' + : r, + ) + if (Object.keys(node).some((k) => !['$ref', 'description', 'title'].includes(k))) { + s.refWithSiblings++ + } + } + for (const [k, v] of Object.entries(node)) { + count(s.keywords, k) + if (DATA_KEYWORDS.has(k)) continue + if (MAP_KEYWORDS.has(k) && isObject(v)) { + for (const sub of Object.values(v)) { + if (k === 'properties' && isObject(sub) && sub.private === true) s.privateFields++ + surveySchema(sub, s, false) + } + } else { + surveySchema(v, s, false) + } + } +} + +const sortedCounts = (m: Map) => + [...m.entries()].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)) + +// --- Main ------------------------------------------------------------------------- + +async function main() { + mkdirSync(DATA, { recursive: true }) + const collections = await listCollections() + console.log(`collections listed: ${collections.length}`) + + // Every distinct schema (by content) across every version, and where it was seen. + const schemas = new Map() + const latest: { key: string; semver: string; schemas: Record }[] = [] + let versionCount = 0 + for (const c of collections) { + const key = `${c.ownerSlug}/${c.slug}` + if (!c.latestVersion) { + console.log(` ${key}: no versions, skipped`) + continue + } + const base = `${API}/collections/${key}` + const versions = await cachedJson<{ semver: string }[]>( + `${key}/versions.json`, + `${base}/versions`, + ) + for (const v of versions) { + versionCount++ + const detail = await cachedJson<{ schemas?: Record }>( + `${key}/versions/${v.semver}.json`, + `${base}/versions/${v.semver}`, + ) + for (const [slug, schema] of Object.entries(detail.schemas ?? {})) { + const id = JSON.stringify(schema) + const entry = schemas.get(id) ?? { schema, seenIn: [] } + entry.seenIn.push(`${key}@${v.semver}:${slug}`) + schemas.set(id, entry) + } + if (v.semver === c.latestVersion) { + latest.push({ key, semver: v.semver, schemas: detail.schemas ?? {} }) + } + } + } + + const survey: Survey = { + $schema: new Map(), + formats: new Map(), + keywords: new Map(), + refs: new Map(), + refWithSiblings: 0, + privateFields: 0, + } + for (const { schema } of schemas.values()) surveySchema(schema, survey) + + const started = Date.now() + const progress = (what: string) => + process.stderr.write(`[${((Date.now() - started) / 1000).toFixed(1)}s] ${what}\n`) + progress('pass 1: compile') + // Pass 1: compile. + const compileReport: string[] = [] + let compileAgree = 0 + for (const { schema, seenIn } of schemas.values()) { + const t = await compileAll(schema) + const ok = [t.ajv, t.v2, t.hj].map((c) => typeof c !== 'string') + if (ok[0] === ok[1] && ok[0] === ok[2]) compileAgree++ + if (ok.includes(false)) { + const show = (c: Compiled) => (typeof c === 'string' ? c : 'ok') + compileReport.push( + ` ${seenIn[0]}${seenIn.length > 1 ? ` (+${seenIn.length - 1} more)` : ''}\n` + + ` ajv: ${show(t.ajv)}\n v2: ${show(t.v2)}\n hyperjump: ${show(t.hj)}`, + ) + } + } + + // Passes 2 and 3: records of each latest version, and mutations of a sample. + const production = new Tally('production records') + const mutated = new Tally('mutated records') + for (const { key, semver, schemas: typeSchemas } of latest) { + progress(`pass 2/3: ${key}@${semver}`) + const byType = new Map() + for (const [slug, schema] of Object.entries(typeSchemas)) + byType.set(slug, await compileAll(schema)) + const sampled = new Map() + const file = await cachedFile( + `${key}/records-${semver}.ndjson`, + `${API}/collections/${key}/versions/${semver}/records.ndjson`, + ) + const lines = createInterface({ input: createReadStream(file), crlfDelay: Infinity }) + for await (const line of lines) { + if (!line) continue + const rec = JSON.parse(line) as { id: string; type: string; data: unknown } + const t = byType.get(rec.type) + if (!t) throw new Error(`${key}@${semver}: no schema for type ${rec.type}`) + const label = `${key}@${semver} ${rec.type} ${JSON.stringify(rec.id)}` + const n = sampled.get(rec.type) ?? 0 + if (QUICK && n >= MUTATE_SAMPLE) continue + production.compare(label, t, rec.data) + // Large records are skipped: AJV takes seconds per call on some of them. + if (n < MUTATE_SAMPLE && line.length < MUTATE_MAX_BYTES) { + sampled.set(rec.type, n + 1) + for (const [what, data] of mutations(typeSchemas[rec.type], rec.data)) { + mutated.compare(`${label} [${what}]`, t, data) + } + } + } + } + + progress('pass 3b: format corpus') + // Pass 3b: every format keyword over the corpus, on strings and on numbers. + const formatTally = new Tally('format corpus') + for (const f of ALL_FORMATS) { + const t = await compileAll({ format: f }) + for (const s of [...FORMAT_CORPUS, 2 ** 31, 1.5, -1]) { + formatTally.compare(`format ${f}: ${JSON.stringify(s)}`, t, s) + } + } + + progress('pass 3c: keyword corpus') + // Pass 3c: every draft-07 keyword, and a few schema shapes, over small instances. + const keywordTally = new Tally('keyword corpus') + for (const [schema, instances] of KEYWORD_CORPUS) { + const t = await compileAll(schema) + const show = (c: Compiled) => (typeof c === 'string' ? c : 'ok') + const failed = [t.ajv, t.v2].map((c) => typeof c === 'string') + if (failed[0] !== failed[1]) { + keywordTally.disagreements.push( + ` compile ${JSON.stringify(schema)}\n ajv: ${show(t.ajv)}\n v2: ${show(t.v2)}`, + ) + } else if (failed[0]) { + keywordTally.bothRejected.push( + ` ${JSON.stringify(schema)}\n ajv: ${show(t.ajv)}\n v2: ${show(t.v2)}`, + ) + } + if (failed[0] || failed[1]) continue + for (const data of instances) + keywordTally.compare(`${JSON.stringify(schema)} ← ${JSON.stringify(data)}`, t, data) + } + + // --- Report --------------------------------------------------------------------- + console.log(`versions: ${versionCount} (latest: ${latest.length})`) + console.log(`distinct schemas: ${schemas.size}`) + console.log(`\n$schema values (distinct schemas):`) + for (const [k, n] of sortedCounts(survey.$schema)) console.log(` ${n}\t${k}`) + console.log(`formats (occurrences):`) + for (const [k, n] of sortedCounts(survey.formats)) console.log(` ${n}\t${k}`) + console.log(`$ref targets (occurrences):`) + for (const [k, n] of sortedCounts(survey.refs)) console.log(` ${n}\t${k}`) + console.log(`$ref with sibling keywords: ${survey.refWithSiblings}`) + console.log(`field-level "private": true: ${survey.privateFields}`) + const later = sortedCounts(survey.keywords).filter(([k]) => LATER_KEYWORDS.has(k)) + console.log( + `2019-09+ keywords: ${later.length ? later.map(([k, n]) => `${k}=${n}`).join(', ') : 'none'}`, + ) + console.log(`all keys in schema positions (occurrences):`) + console.log( + ' ' + + sortedCounts(survey.keywords) + .map(([k, n]) => `${k}=${n}`) + .join(', '), + ) + if (ajvWarnings.size) console.log(`AJV warnings: ${[...ajvWarnings].join(' | ')}`) + + console.log(`\nschema compile: ${compileAgree}/${schemas.size} agree (ajv, v2, hyperjump)`) + for (const line of compileReport) console.log(line) + + production.print() + mutated.print() + formatTally.print() + keywordTally.print() +} + +/** [schema, instances]: each instance is validated by all three. */ +const KEYWORD_CORPUS: [unknown, unknown[]][] = [ + [true, [1, null]], + [false, [1]], + [{}, [1, 'a', null, [], {}]], + [{ type: 'integer' }, [1, 1.0, 1.5, '1', 2 ** 53, -0]], + [{ type: ['string', 'null'] }, ['a', null, 1]], + [{ type: 'number' }, [1, 'x']], + [{ enum: [1, 'a', null, [1], { a: 1 }] }, [1, 'a', null, [1], { a: 1 }, 2, [2], { a: 2 }, true]], + [{ const: { a: [1, 2] } }, [{ a: [1, 2] }, { a: [2, 1] }]], + [{ const: 1 }, [1, 1.0, '1']], + [{ multipleOf: 0.01 }, [0.07, 0.1, 1.23, 19.99]], + [{ multipleOf: 3 }, [9, 10, 4.5]], + [{ minimum: 1, maximum: 3 }, [0, 1, 3, 4, 'x']], + [{ exclusiveMinimum: 1, exclusiveMaximum: 3 }, [1, 2, 3]], + [{ minLength: 2, maxLength: 3 }, ['a', 'ab', 'abcd', '😀', '😀😀', 'é']], + [{ pattern: '^a' }, ['abc', 'bac', 1]], + [{ pattern: '\\p{Lu}' }, ['A', 'a']], + [{ pattern: '^.$' }, ['😀']], + [{ items: { type: 'string' } }, [['a'], ['a', 1, 2], 'x']], + [ + { items: [{ type: 'string' }, { type: 'number' }] }, + [ + ['a', 1], + [1, 'a'], + ['a', 1, null], + ], + ], + [{ items: [{ type: 'string' }], additionalItems: false }, [['a'], ['a', 1], ['a', 1, 2]]], + [ + { items: [{ type: 'string' }], additionalItems: { type: 'number' } }, + [ + ['a', 1], + ['a', 'b', 'c'], + ], + ], + [{ additionalItems: false }, [[1, 2]]], + [{ minItems: 1, maxItems: 2 }, [[], [1], [1, 2, 3]]], + [ + { uniqueItems: true }, + [ + [1, 2], + [1, 1], + [{ a: 1 }, { a: 1 }], + [1, 2, 1, 2], + [[1], [1]], + ], + ], + [{ contains: { type: 'string' } }, [[1, 'a'], [1, 2], []]], + [{ minProperties: 1, maxProperties: 2 }, [{}, { a: 1 }, { a: 1, b: 2, c: 3 }]], + [{ required: ['a', 'b'] }, [{ a: 1, b: 2 }, { a: 1 }, {}, []]], + [ + { properties: { a: { type: 'string' }, 'x/y~z': { type: 'number' } } }, + [{ a: 'x' }, { a: 1 }, { 'x/y~z': 'q' }], + ], + [ + { properties: { 'a b': { type: 'string' }, 'é%': { type: 'string' } } }, + [{ 'a b': 1, 'é%': 2 }], + ], + [{ patternProperties: { '^x-': { type: 'string' } } }, [{ 'x-a': 'ok', 'x-b': 1, y: 1 }]], + [{ properties: { a: {} }, additionalProperties: false }, [{ a: 1 }, { a: 1, b: 2, c: 3 }]], + [ + { properties: { a: {} }, patternProperties: { '^p': {} }, additionalProperties: false }, + [{ a: 1, p1: 2 }, { q: 1 }], + ], + [{ additionalProperties: { type: 'string' } }, [{ a: 'x', b: 1 }]], + [{ properties: { a: false } }, [{ a: 1 }, {}]], + [{ dependencies: { a: ['b', 'c'] } }, [{ a: 1 }, { a: 1, b: 1, c: 1 }, { b: 1 }]], + [{ dependencies: { a: { required: ['b'] } } }, [{ a: 1 }, { a: 1, b: 1 }]], + [{ propertyNames: { maxLength: 2 } }, [{ ab: 1 }, { abc: 1 }]], + [{ propertyNames: { pattern: '^[a-z]+$' } }, [{ A: 1, b: 2 }]], + [ + { if: { type: 'string' }, then: { minLength: 2 }, else: { type: 'number' } }, + ['a', 'ab', 1, null], + ], + [ + { if: { properties: { k: { const: 'x' } } }, then: { required: ['v'] } }, + [{ k: 'x' }, { k: 'y' }], + ], + [{ allOf: [{ type: 'string' }, { minLength: 2 }] }, ['a', 'ab', 1]], + [{ anyOf: [{ type: 'string' }, { type: 'number' }] }, ['a', 1, null]], + [{ oneOf: [{ type: 'integer' }, { type: 'number' }] }, [1, 1.5, 'x']], + [{ not: { type: 'string' } }, [1, 'a']], + [{ format: 'email' }, ['a@b.co', 'a@b', 5]], + [{ format: 'unknown-format' }, ['x']], + [{ type: 'object', 'x-ref-type': 'T', version: '1', private: true }, [{}, 1]], + [ + { definitions: { s: { type: 'string' } }, properties: { a: { $ref: '#/definitions/s' } } }, + [{ a: 'x' }, { a: 1 }], + ], + [{ $defs: { s: { type: 'string' } }, properties: { a: { $ref: '#/$defs/s' } } }, [{ a: 1 }]], + [ + { + definitions: { s: { type: 'string' } }, + properties: { a: { $ref: '#/definitions/s', maxLength: 1 } }, + }, + [{ a: 'xy' }], + ], + [ + { $ref: '#/definitions/d', definitions: { d: { type: 'object', required: ['t'] } } }, + [{ t: 1 }, {}], + ], + [{ properties: { a: { type: 'string' } }, items: { $ref: '#/properties/a' } }, [['x', 1]]], + [{ definitions: { 'a b': { type: 'string' } }, $ref: '#/definitions/a%20b' }, ['x', 1]], + [{ definitions: { 'a/b': { type: 'string' } }, $ref: '#/definitions/a~1b' }, ['x', 1]], + [ + { + $id: 'https://example.org/s', + definitions: { a: { type: 'string' } }, + $ref: 'https://example.org/s#/definitions/a', + }, + ['x', 1], + ], + [ + { + $id: 'https://example.org/root.json', + definitions: { a: { $id: 'a.json', type: 'string' } }, + $ref: 'a.json', + }, + ['x', 1], + ], + [{ definitions: { a: { $anchor: 'foo', type: 'string' } }, $ref: '#foo' }, ['x', 1]], + [ + { + definitions: { + n: { + type: 'object', + properties: { c: { type: 'array', items: { $ref: '#/definitions/n' } } }, + }, + }, + $ref: '#/definitions/n', + }, + [{ c: [{ c: [] }] }, { c: [{ c: [1] }] }], + ], + [{ id: 'other', definitions: { a: { type: 'string' } }, $ref: '#/definitions/a' }, ['x', 1]], + [{ unevaluatedProperties: false, properties: { a: {} } }, [{ b: 1 }]], + [{ dependentRequired: { a: ['b'] } }, [{ a: 1 }]], + [{ dependentSchemas: { a: { required: ['b'] } } }, [{ a: 1 }]], + [{ prefixItems: [{ type: 'string' }] }, [[1]]], + [{ contains: { type: 'string' }, minContains: 2, maxContains: 2 }, [['a'], ['a', 'b', 'c']]], + [{ $schema: 'http://json-schema.org/draft-07/schema' }, [1]], + [{ $schema: 'http://json-schema.org/draft-07/schema#' }, [1]], + [{ $schema: 'https://json-schema.org/draft-07/schema#' }, [1]], + [{ $schema: 'http://json-schema.org/draft-04/schema#' }, [1]], + [{ $schema: 'https://json-schema.org/draft/2020-12/schema' }, [1]], + [ + { properties: { a: { $schema: 'http://json-schema.org/draft-04/schema#', type: 'string' } } }, + [{ a: 1 }], + ], + [{ type: 'text' }, [1]], + [{ required: 'a' }, [1]], + [{ pattern: '(' }, [1]], + [{ pattern: '\\a' }, ['a']], + [{ patternProperties: { '(': {} } }, [1]], + [{ $ref: '#/definitions/missing' }, [1]], + [{ $ref: 'https://remote.invalid/schema.json' }, [1]], + [{ $ref: 'http://json-schema.org/draft-07/schema#' }, [{ type: 'string' }, { type: 3 }]], + [{ enum: [] }, [1]], + [{ minLength: -1 }, ['a']], + [{ multipleOf: 0 }, [1]], + [ + { properties: { a: { default: 'x', examples: ['y'], readOnly: true, $comment: 'c' } } }, + [{ a: 1 }], + ], + // Names that exist on Object.prototype are ordinary property names. + [{ required: ['toString'] }, [{}, { toString: 1 }]], + [{ properties: { constructor: { type: 'string' } } }, [{}, { constructor: 1 }]], + [{ dependencies: { toString: ['b'] } }, [{}]], + [{ format: 'hasOwnProperty' }, ['x']], + [ + JSON.parse('{"properties":{"__proto__":{"type":"string"}}}'), + [JSON.parse('{"__proto__":1}'), {}], + ], + // Every error, or the first per loop (schemas with branching keywords). + [{ properties: { a: { type: 'string' }, b: { type: 'string' } } }, [{ a: 1, b: 2 }]], + [ + { properties: { a: { type: 'string' }, b: { type: 'string' } }, not: { type: 'null' } }, + [{ a: 1, b: 2 }], + ], + [{ items: { type: 'string' }, anyOf: [{}] }, [[1, 2]]], +] + +await main() diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index a8da0c9..55ee428 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -21,6 +21,16 @@ export { scanJson, } from './input-rules.js' export { fileRefs } from './file-refs.js' +export { + checkSchema, + checkSchemaBounds, + compileSchema, + type ExtraFieldWarning, + findExtraFields, + SchemaError, + type SchemaValidator, + stripToSchema, +} from './validate.js' export * from './root.js' export * from './tree/node.js' export { @@ -46,5 +56,13 @@ export { type MergeResult, type MergeStats, } from './tree/merge.js' -export { type DiffEntry, diffTrees, getEntry, iterate, type IterateOptions } from './tree/read.js' +export { + type DiffEntry, + diffTrees, + entryAt, + getEntry, + iterate, + type IterateOptions, + rankOf, +} from './tree/read.js' export { verifyTree, type VerifyResult } from './tree/verify.js' diff --git a/packages/core/src/tree/read.ts b/packages/core/src/tree/read.ts index 0b727fb..e09e3bc 100644 --- a/packages/core/src/tree/read.ts +++ b/packages/core/src/tree/read.ts @@ -26,6 +26,65 @@ export async function getEntry( } } +/** + * The number of entries with keys less than `key` (its rank). One node read per + * level, using the counts in interior entries. + */ +export async function rankOf( + source: NodeSource, + root: string | null, + key: string, +): Promise { + if (root === null) return 0 + let rank = 0 + let hash = root + for (;;) { + const node = await source.node(hash) + if (node.kind === 'leaf') { + for (const e of node.entries) { + if (compareUtf8(source.spec.key(e), key) >= 0) break + rank++ + } + return rank + } + let next: string | null = null + for (const c of node.children) { + if (compareUtf8(key, c.lastKey) <= 0) { + next = c.hash + break + } + rank += c.count + } + if (next === null) return rank + hash = next + } +} + +/** The entry at a position (0-based), or null past the end. One node read per level. */ +export async function entryAt( + source: NodeSource, + root: string | null, + index: number, +): Promise { + if (root === null || index < 0) return null + let hash = root + let i = index + for (;;) { + const node = await source.node(hash) + if (node.kind === 'leaf') return node.entries[i] ?? null + let next: string | null = null + for (const c of node.children) { + if (i < c.count) { + next = c.hash + break + } + i -= c.count + } + if (next === null) return null + hash = next + } +} + export interface IterateOptions { /** Start after this key (keyset pagination). Exclusive with `offset`. */ after?: string diff --git a/packages/core/src/validate.ts b/packages/core/src/validate.ts new file mode 100644 index 0000000..082d54f --- /dev/null +++ b/packages/core/src/validate.ts @@ -0,0 +1,793 @@ +/** + * Record validation against type schemas (protocol v2, section 5). + * + * v1 validated with AJV, which compiles every schema to JavaScript with + * `new Function`. Cloudflare Workers forbid that, so v2 uses + * @cfworker/json-schema, which interprets the schema and never evaluates code. + * The dialect is v1's, and scripts/diff-validators.ts checks the two agree on + * every production schema and record: + * + * - JSON Schema draft-07. A root `$schema`, if present, must name draft-07. + * - Keywords next to `$ref` are applied, as AJV does (draft-07 says to ignore them). + * - Keywords draft-07 doesn't define are ignored. That includes later drafts' + * keywords, which @cfworker/json-schema would otherwise enforce in any draft. + * - `format` constrains strings only, with ajv-formats' definitions (see + * AJV_FORMATS below). Unknown formats are ignored. + * - A schema must itself be valid against the draft-07 meta-schema, every + * `pattern` must be a valid Unicode (`u` flag) regex, and every `$ref` must + * resolve inside the schema; all three are checked when it is compiled. + */ +import { + deepCompareStrict, + dereference, + format as formats, + validate as interpret, + type OutputUnit, + type Schema, +} from '@cfworker/json-schema' + +import { MAX_SCHEMA_BYTES } from './constants.js' +import { hashSchema } from './hash.js' +import { checkTypeSlug } from './input-rules.js' +import { jcs } from './jcs.js' +import { utf8ByteLength } from './utf8.js' + +/** Longest `pattern` (or `patternProperties` key) accepted: long patterns are the main ReDoS vector. */ +const MAX_PATTERN_LENGTH = 256 +const MAX_CACHED_VALIDATORS = 500 + +/** A schema that can't be compiled: not valid draft-07, or an unresolvable `$ref`. */ +export class SchemaError extends Error { + constructor(message: string) { + super(message) + this.name = 'SchemaError' + } +} + +/** Validate one record's `data`. Returns `" "` strings; empty means valid. */ +export type SchemaValidator = (data: unknown) => string[] + +type Lookup = Record + +// Fixed rather than the library default, which is the page URL in a browser: +// relative `$id`s and `$ref`s must resolve the same everywhere. +const BASE_URI = new URL('https://schema.underlay.invalid/') + +const META_ID = 'http://json-schema.org/draft-07/schema' +const DRAFT_07 = new Set([ + 'http://json-schema.org/draft-07/schema', + 'http://json-schema.org/draft-07/schema#', +]) + +// Keywords whose values are maps from a name to a subschema: the names are data, +// not keywords, so a property called "prefixItems" must survive `toDialect`. +const MAP_KEYWORDS = new Set([ + 'properties', + 'patternProperties', + 'definitions', + '$defs', + 'dependencies', +]) +// Keywords whose values are instance data, never schemas. +const DATA_KEYWORDS = new Set(['enum', 'const', 'default', 'examples', 'required', 'type']) +// Keywords @cfworker/json-schema implements but draft-07 (and so AJV) ignores. +// `id` is draft-04's `$id`; the library honours it, AJV's draft-07 does not. +const NOT_DRAFT_07 = [ + 'id', + '$recursiveRef', + '$recursiveAnchor', + 'unevaluatedProperties', + 'unevaluatedItems', + 'dependentRequired', + 'dependentSchemas', + 'prefixItems', + 'minContains', + 'maxContains', +] + +const isObject = (v: unknown): v is Record => + v !== null && typeof v === 'object' && !Array.isArray(v) + +// --- Compile ------------------------------------------------------------------ + +// Schemas are content-addressed, so compiled validators are keyed by schema +// hash: every push parses its schemas afresh, and identity would never hit. +const validatorCache = new Map() + +/** + * Compile a type schema, reusing the validator for identical schema content. + * Throws SchemaError if the schema is not valid draft-07 or a `$ref` doesn't + * resolve. Run `checkSchema` first: this does not bound size or patterns. + */ +export function compileSchema(schema: unknown): SchemaValidator { + let key: string + let validator: SchemaValidator + try { + key = hashSchema(schema) + const cached = validatorCache.get(key) + if (cached) return cached + validator = build(schema) + } catch (err) { + // Every compile step recurses over the schema; a deeply nested one + // overflows the stack, which is a bad schema, not a server error. + if (err instanceof SchemaError) throw err + throw new SchemaError(`schema could not be compiled: ${(err as Error).message}`) + } + if (validatorCache.size >= MAX_CACHED_VALIDATORS) { + const oldest = validatorCache.keys().next().value + if (oldest !== undefined) validatorCache.delete(oldest) + } + validatorCache.set(key, validator) + return validator +} + +function build(schema: unknown): SchemaValidator { + // Boolean schemas are fine below the root, but a type's schema is an object: + // v1 could not compile `true` or `false` there either. + if (!isObject(schema)) throw new SchemaError('schema must be an object') + if (schema.$schema !== undefined && !DRAFT_07.has(schema.$schema as string)) { + throw new SchemaError( + `unsupported $schema ${JSON.stringify(schema.$schema)}: schemas must be JSON Schema draft-07`, + ) + } + // A bare copy: the library tests membership with `in`, and annotates the + // schema objects it is given. + const body = bare(schema) as Schema + const meta = interpret(body, META_SCHEMA, '7', META_LOOKUP, true) + if (!meta.valid) { + throw new SchemaError( + `schema is invalid: ${messages(meta.errors, META_SCHEMA, META_LOOKUP, body).join(', ')}`, + ) + } + // The meta-schema's `regex` format accepts what `new RegExp(p)` accepts, but + // patterns run with the `u` flag (as in AJV), which is stricter. + const badPattern = findBadPattern(body) + if (badPattern) throw new SchemaError(badPattern) + + toDialect(body) + // Seeded with the meta-schema, which AJV also resolves `$ref`s to. + const lookup: Lookup = Object.assign(Object.create(null) as Lookup, META_LOOKUP) + try { + dereference(body, lookup, BASE_URI) + } catch (err) { + throw new SchemaError((err as Error).message) + } + // The library only finds a dangling `$ref` when a record reaches it; AJV + // refused the schema up front, and so does v2. + for (const sub of Object.values(lookup)) { + if (typeof sub === 'object' && typeof sub.$ref === 'string') { + if (lookup[sub.__absolute_ref__ ?? sub.$ref] === undefined) { + throw new SchemaError(`can't resolve reference ${sub.$ref}`) + } + } + } + + // Short-circuiting stops a `properties` or `items` loop at its first failure. + // It never changes the verdict, only how many errors come back, so it is off + // where it is safe: AJV (v1) reported every error. It is unsafe in schemas + // with keywords whose failing branches are explored and thrown away: each + // such branch re-validates its whole subtree, which is exponential in depth + // on recursive schemas. A 567 KB production record under a recursive `oneOf` + // schema took 1.5 ms short-circuited and did not finish otherwise (AJV with + // allErrors, as v1 ran it, took 7 s). A `$ref` to the meta-schema brings + // its `anyOf`s in. + const refsMeta = Object.entries(lookup).some( + ([uri, sub]) => + !uri.startsWith(META_ID) && + typeof sub === 'object' && + !!sub.__absolute_ref__?.startsWith(META_ID), + ) + const shortCircuit = refsMeta || hasBranching(body) + + return (data) => { + try { + const instance = bare(data) + // '2019-09' only changes `$ref` handling here (siblings are applied); the + // later-draft keywords it would add were removed by `toDialect`. + const result = interpret(instance, body, '2019-09', lookup, shortCircuit) + return result.valid ? [] : messages(result.errors, body, lookup, instance) + } catch (err) { + // A schema that recurses without consuming data, e.g. `{"$ref":"#"}`, + // overflows the stack. Report it against the record, don't crash the push. + return [`/ schema could not be applied: ${(err as Error).message}`] + } + } +} + +/** + * A deep copy whose objects have no prototype. The library tests membership + * with `in`, so on ordinary objects `required: ["toString"]` would pass for `{}` + * and `properties: {"constructor": …}` would be checked against `Object`. + */ +function bare(value: unknown): unknown { + if (Array.isArray(value)) return value.map(bare) + if (value === null || typeof value !== 'object') return value + const out = Object.create(null) as Record + for (const [k, v] of Object.entries(value)) out[k] = bare(v) + return out +} + +const BRANCHING = new Set(['anyOf', 'oneOf', 'not', 'if', 'contains']) + +/** Whether a schema uses a keyword that validates and discards a failing branch. */ +function hasBranching(node: unknown): boolean { + if (Array.isArray(node)) return node.some(hasBranching) + if (!isObject(node)) return false + for (const [k, v] of Object.entries(node)) { + if (DATA_KEYWORDS.has(k)) continue + if (BRANCHING.has(k)) return true + if ( + MAP_KEYWORDS.has(k) && isObject(v) ? Object.values(v).some(hasBranching) : hasBranching(v) + ) { + return true + } + } + return false +} + +/** Remove, in place, the keywords @cfworker/json-schema honours but draft-07 doesn't. */ +function toDialect(node: unknown): void { + if (Array.isArray(node)) { + for (const item of node) toDialect(item) + return + } + if (!isObject(node)) return + for (const k of NOT_DRAFT_07) delete node[k] + for (const [k, v] of Object.entries(node)) { + if (DATA_KEYWORDS.has(k)) continue + if (MAP_KEYWORDS.has(k) && isObject(v)) { + for (const sub of Object.values(v)) toDialect(sub) + } else { + toDialect(v) + } + } +} + +function findBadPattern(node: unknown): string | null { + if (Array.isArray(node)) { + for (const item of node) { + const found = findBadPattern(item) + if (found) return found + } + return null + } + if (!isObject(node)) return null + const patterns: unknown[] = [node.pattern] + if (isObject(node.patternProperties)) patterns.push(...Object.keys(node.patternProperties)) + for (const p of patterns) { + if (typeof p !== 'string') continue + try { + new RegExp(p, 'u') + } catch (err) { + return `pattern ${JSON.stringify(p)} is invalid: ${(err as Error).message}` + } + } + for (const [k, v] of Object.entries(node)) { + if (DATA_KEYWORDS.has(k)) continue + const found = + MAP_KEYWORDS.has(k) && isObject(v) ? findBadPattern(Object.values(v)) : findBadPattern(v) + if (found) return found + } + return null +} + +// --- Formats ------------------------------------------------------------------ +// +// v1 used ajv-formats in "full" mode. @cfworker/json-schema's formats agree with +// it on date, uri, uri-reference, uri-template, url, uuid, hostname, ipv6 and the +// JSON pointer formats, but differ on the ones below, so these are ajv-formats' +// definitions (MIT, https://github.com/ajv-validator/ajv-formats), installed in +// the library's format table. ajv-formats' number formats (int32, int64, float, +// double) are not carried over: the library applies `format` to strings only. +// No production schema uses them. + +const isLeapYear = (y: number) => y % 4 === 0 && (y % 100 !== 0 || y % 400 === 0) +const DATE = /^(\d\d\d\d)-(\d\d)-(\d\d)$/ +const DAYS = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] +function date(str: string): boolean { + const m = DATE.exec(str) + if (!m) return false + const year = +m[1]! + const month = +m[2]! + const day = +m[3]! + return ( + month >= 1 && + month <= 12 && + day >= 1 && + day <= (month === 2 && isLeapYear(year) ? 29 : DAYS[month]!) + ) +} + +const TIME = /^(\d\d):(\d\d):(\d\d(?:\.\d+)?)(z|([+-])(\d\d)(?::?(\d\d))?)?$/i +function time(strictTimeZone: boolean) { + return (str: string): boolean => { + const m = TIME.exec(str) + if (!m) return false + const hr = +m[1]! + const min = +m[2]! + const sec = +m[3]! + const tz = m[4] + const tzSign = m[5] === '-' ? -1 : 1 + const tzH = +(m[6] || 0) + const tzM = +(m[7] || 0) + if (tzH > 23 || tzM > 59 || (strictTimeZone && !tz)) return false + if (hr <= 23 && min <= 59 && sec < 60) return true + // A leap second is valid only at 23:59:60 UTC. + const utcMin = min - tzM * tzSign + const utcHr = hr - tzH * tzSign - (utcMin < 0 ? 1 : 0) + return (utcHr === 23 || utcHr === -1) && (utcMin === 59 || utcMin === -1) && sec < 61 + } +} + +const DATE_TIME_SEPARATOR = /t|\s/i +function dateTime(strictTimeZone: boolean) { + const t = time(strictTimeZone) + return (str: string): boolean => { + const parts = str.split(DATE_TIME_SEPARATOR) + return parts.length === 2 && date(parts[0]!) && t(parts[1]!) + } +} + +const Z_ANCHOR = /[^\\]\\Z/ +function regex(str: string): boolean { + if (Z_ANCHOR.test(str)) return false + try { + new RegExp(str) + return true + } catch { + return false + } +} + +const matches = (re: RegExp) => (str: string) => re.test(str) + +const AJV_FORMATS: Record boolean> = { + date, + time: time(true), + 'date-time': dateTime(true), + 'iso-time': time(false), + 'iso-date-time': dateTime(false), + duration: matches(/^P(?!$)((\d+Y)?(\d+M)?(\d+D)?(T(?=\d)(\d+H)?(\d+M)?(\d+S)?)?|(\d+W)?)$/), + email: matches( + /^[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i, + ), + ipv4: matches( + /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/, + ), + regex, + // `m` as in ajv-formats, quirk included: a string passes if any one line does. + byte: matches(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/m), +} + +// The library reads formats from this shared table; there is no per-validator +// option. Nothing else in the Worker uses the library, so replacing entries is +// safe. Without a prototype, `format: "hasOwnProperty"` is an unknown format +// (ignored) rather than a method called on every string. +Object.assign(formats, AJV_FORMATS) +Object.setPrototypeOf(formats, null) + +// --- Messages ----------------------------------------------------------------- +// +// AJV's wording, which v1 clients have seen: `${instancePath || '/'} ${message}`. +// The library reports wrapper errors ("A subschema had errors.") above each +// leaf; AJV reports only the leaves, so wrappers are dropped. + +function messages( + errors: OutputUnit[], + root: Schema | boolean, + lookup: Lookup, + instance: unknown, +): string[] { + const out: string[] = [] + for (let i = 0; i < errors.length; i++) { + const e = errors[i]! + const message = describe(e, root, lookup, instance) + if (message !== null) out.push(`${instancePath(e.instanceLocation)} ${message}`) + // `additionalProperties: false` is reported by AJV once, on the object. The + // library follows its wrapper with a "False boolean schema" leaf for the + // property, whose keywordLocation is (wrongly) the instance location, so it + // can only be recognised by position. + const additional = e.keyword === 'additionalProperties' || e.keyword === 'additionalItems' + if (additional && message !== null && errors[i + 1]?.keyword === 'false') i++ + // A property name's own errors follow its wrapper, located at the property; + // AJV locates them at the object. + if (e.keyword === 'propertyNames') { + const name = /^Property name "(.*)" does/.exec(e.error)?.[1] ?? '' + const at = `${e.instanceLocation}/${encodeURI(escapeSegment(name))}` + while (errors[i + 1]?.instanceLocation === at) { + const leaf = errors[++i]! + const leafMessage = describe(leaf, root, lookup, instance) + if (leafMessage !== null) out.push(`${instancePath(e.instanceLocation)} ${leafMessage}`) + } + } + } + return out +} + +function describe( + e: OutputUnit, + root: Schema | boolean, + lookup: Lookup, + instance: unknown, +): string | null { + const value = () => schemaAt(e.keywordLocation, root, lookup) + const quoted = (re: RegExp) => re.exec(e.error)?.slice(1) ?? [] + switch (e.keyword) { + case '$ref': + case 'properties': + case 'patternProperties': + case 'items': + case 'allOf': + return null + case 'false': + return 'boolean schema is false' + case 'additionalProperties': + return value() === false ? 'must NOT have additional properties' : null + case 'additionalItems': { + if (value() !== false) return null + const items = schemaAt(e.keywordLocation.replace(/additionalItems$/, 'items'), root, lookup) + return `must NOT have more than ${Array.isArray(items) ? items.length : 0} items` + } + case 'dependencies': { + const [key] = quoted(/^Instance has "(.*)" but does not have ".*"\.$/) + if (key === undefined) return null // the schema form: its leaves carry the errors + const deps = (value() as Record | undefined)?.[key] + const list = Array.isArray(deps) ? deps : [] + return `must have ${list.length === 1 ? 'property' : 'properties'} ${list.join(', ')} when property ${key} is present` + } + case 'type': { + const t = value() + return `must be ${Array.isArray(t) ? t.join(',') : String(t)}` + } + case 'required': + return `must have required property '${quoted(/property "(.*)"\.$/)[0]}'` + case 'const': + return 'must be equal to constant' + case 'enum': + return 'must be equal to one of the allowed values' + case 'not': + return 'must NOT be valid' + case 'anyOf': + return 'must match a schema in anyOf' + case 'oneOf': + return 'must match exactly one schema in oneOf' + case 'if': + return e.error.includes('"then"') ? 'must match "then" schema' : 'must match "else" schema' + case 'propertyNames': + return 'property name must be valid' + case 'contains': + return 'must contain at least 1 valid item(s)' + case 'uniqueItems': { + // AJV names the last duplicate pair; the library, the first. + const items = valueAt(e.instanceLocation, instance) + if (Array.isArray(items)) { + for (let i = items.length - 1; i > 0; i--) { + for (let j = i - 1; j >= 0; j--) { + if (deepCompareStrict(items[i], items[j])) { + return `must NOT have duplicate items (items ## ${j} and ${i} are identical)` + } + } + } + } + return 'must NOT have duplicate items' + } + case 'minimum': + return `must be >= ${String(value())}` + case 'maximum': + return `must be <= ${String(value())}` + case 'exclusiveMinimum': + return `must be > ${String(value())}` + case 'exclusiveMaximum': + return `must be < ${String(value())}` + case 'multipleOf': + return `must be multiple of ${String(value())}` + case 'minLength': + return `must NOT have fewer than ${String(value())} characters` + case 'maxLength': + return `must NOT have more than ${String(value())} characters` + case 'minItems': + return `must NOT have fewer than ${String(value())} items` + case 'maxItems': + return `must NOT have more than ${String(value())} items` + case 'minProperties': + return `must NOT have fewer than ${String(value())} properties` + case 'maxProperties': + return `must NOT have more than ${String(value())} properties` + case 'pattern': + return `must match pattern "${String(value())}"` + case 'format': + return `must match format "${String(value())}"` + default: + return e.error + } +} + +// The library writes locations as `#` plus JSON Pointer segments passed through +// encodeURI; AJV's instancePath is the plain JSON Pointer. +function instancePath(location: string): string { + let path = location.slice(1) + try { + path = decodeURI(path) + } catch { + // keep it encoded + } + return path || '/' +} + +const unescapeSegment = (s: string) => { + try { + s = decodeURI(s) + } catch { + // keep it encoded + } + return s.replace(/~1/g, '/').replace(/~0/g, '~') +} + +/** The instance value at an instance location. */ +function valueAt(location: string, root: unknown): unknown { + let node = root + for (const raw of location.split('/').slice(1)) { + if (node === null || typeof node !== 'object') return undefined + node = (node as Record)[unescapeSegment(raw)] + } + return node +} + +/** The schema value at a keyword location, following `$ref` segments through the lookup. */ +function schemaAt(location: string, root: Schema | boolean, lookup: Lookup): unknown { + let node: unknown = root + let inMap = false + for (const raw of location.split('/').slice(1)) { + if (node === null || typeof node !== 'object') return undefined + const seg = unescapeSegment(raw) + const obj = node as Schema + if (!inMap && seg === '$ref' && typeof obj.$ref === 'string') { + node = lookup[obj.__absolute_ref__ ?? obj.$ref] + continue + } + node = (node as Record)[seg] + inMap = !inMap && MAP_KEYWORDS.has(seg) + } + return node +} + +// --- Bounds and format-2 rules ------------------------------------------------ + +/** + * Check every schema in a push: see `checkSchema`. Returns the first error + * message, or null if all are acceptable. + */ +export function checkSchemaBounds(schemas: Record): string | null { + for (const [slug, body] of Object.entries(schemas)) { + const error = checkSchema(slug, body) + if (error) return error + } + return null +} + +/** + * Bound a caller-supplied schema before it is compiled and run server-side, and + * apply the format-2 schema rules: + * + * - the type slug passes `checkTypeSlug`; + * - the canonical (JCS) schema is at most MAX_SCHEMA_BYTES; + * - no `pattern` or `patternProperties` key is longer than 256 characters; + * - a root `private`, if present, is a boolean; + * - no property is marked `"private": true`. Format 1 stripped such fields from + * public views; format 2 has no field-level privacy, and accepting the marker + * would publish a field its author meant to hide. + * + * Returns an error message, or null if the schema is acceptable. + */ +export function checkSchema(slug: string, body: unknown): string | null { + const slugError = checkTypeSlug(slug) + if (slugError) return `Invalid type slug ${JSON.stringify(slug)}: ${slugError}` + let canonical: string + try { + canonical = jcs(body) + } catch (err) { + return `Schema "${slug}" is not JSON: ${(err as Error).message}` + } + if (utf8ByteLength(canonical) > MAX_SCHEMA_BYTES) { + return `Schema "${slug}" exceeds maximum size of ${MAX_SCHEMA_BYTES} bytes` + } + const longPattern = findLongPattern(body) + if (longPattern) { + return `Schema "${slug}" has a ${longPattern} longer than ${MAX_PATTERN_LENGTH} characters` + } + if (isObject(body) && body.private !== undefined && typeof body.private !== 'boolean') { + return `Schema "${slug}": "private" must be a boolean` + } + const privateField = findPrivateField(body, '') + if (privateField !== null) { + return ( + `Schema "${slug}" marks property ${privateField} as private: field-level privacy ` + + 'is not supported; make the whole type private instead' + ) + } + return null +} + +function findLongPattern(node: unknown): string | null { + if (Array.isArray(node)) { + for (const item of node) { + const found = findLongPattern(item) + if (found) return found + } + return null + } + if (!isObject(node)) return null + for (const [key, value] of Object.entries(node)) { + if (key === 'pattern' && typeof value === 'string' && value.length > MAX_PATTERN_LENGTH) { + return '"pattern"' + } + // Keys of patternProperties are regexes too; v1 didn't bound them. + if (key === 'patternProperties' && isObject(value)) { + if (Object.keys(value).some((p) => p.length > MAX_PATTERN_LENGTH)) { + return '"patternProperties" pattern' + } + } + const found = findLongPattern(value) + if (found) return found + } + return null +} + +/** + * The JSON Pointer of the first property schema with `"private": true`, at any + * depth: nested markers never did anything in format 1 either, and are refused + * for the same reason. + */ +function findPrivateField(node: unknown, path: string): string | null { + if (Array.isArray(node)) { + for (let i = 0; i < node.length; i++) { + const found = findPrivateField(node[i], `${path}/${i}`) + if (found !== null) return found + } + return null + } + if (!isObject(node)) return null + for (const [k, v] of Object.entries(node)) { + if (DATA_KEYWORDS.has(k)) continue + const at = `${path}/${escapeSegment(k)}` + if (k === 'properties' && isObject(v)) { + for (const [name, sub] of Object.entries(v)) { + const subPath = `${at}/${escapeSegment(name)}` + if (isObject(sub) && sub.private === true) return JSON.stringify(subPath) + const found = findPrivateField(sub, subPath) + if (found !== null) return found + } + continue + } + const found = findPrivateField(v, at) + if (found !== null) return found + } + return null +} + +const escapeSegment = (s: string) => s.replace(/~/g, '~0').replace(/\//g, '~1') + +// --- Extra fields ------------------------------------------------------------- + +export interface ExtraFieldWarning { + recordId: string + type: string + fields: string[] +} + +/** Records with top-level fields their type's schema doesn't list in `properties`. */ +export function findExtraFields( + records: { recordId: string; type: string; data: unknown }[], + schemas: Record }>, +): ExtraFieldWarning[] { + const warnings: ExtraFieldWarning[] = [] + for (const rec of records) { + const props = schemas[rec.type]?.properties + if (!props || typeof rec.data !== 'object' || rec.data === null) continue + const extra = Object.keys(rec.data).filter((k) => !Object.hasOwn(props, k)) + if (extra.length > 0) warnings.push({ recordId: rec.recordId, type: rec.type, fields: extra }) + } + return warnings +} + +/** `data` without the top-level fields that `schemaProperties` doesn't list. */ +export function stripToSchema( + data: Record, + schemaProperties: Record, +): Record { + const result: Record = {} + for (const key of Object.keys(data)) { + if (Object.hasOwn(schemaProperties, key)) result[key] = data[key] + } + return result +} + +// --- Draft-07 meta-schema ----------------------------------------------------- +// +// http://json-schema.org/draft-07/schema, verbatim. AJV checks every schema +// against it before compiling; without the check a schema such as +// `{"type": "text"}` would compile and then reject every record. + +const META_SCHEMA: Schema = { + $schema: 'http://json-schema.org/draft-07/schema#', + $id: 'http://json-schema.org/draft-07/schema#', + title: 'Core schema meta-schema', + definitions: { + schemaArray: { type: 'array', minItems: 1, items: { $ref: '#' } }, + nonNegativeInteger: { type: 'integer', minimum: 0 }, + nonNegativeIntegerDefault0: { + allOf: [{ $ref: '#/definitions/nonNegativeInteger' }, { default: 0 }], + }, + simpleTypes: { enum: ['array', 'boolean', 'integer', 'null', 'number', 'object', 'string'] }, + stringArray: { type: 'array', items: { type: 'string' }, uniqueItems: true, default: [] }, + }, + type: ['object', 'boolean'], + properties: { + $id: { type: 'string', format: 'uri-reference' }, + $schema: { type: 'string', format: 'uri' }, + $ref: { type: 'string', format: 'uri-reference' }, + $comment: { type: 'string' }, + title: { type: 'string' }, + description: { type: 'string' }, + default: true, + readOnly: { type: 'boolean', default: false }, + examples: { type: 'array', items: true }, + multipleOf: { type: 'number', exclusiveMinimum: 0 }, + maximum: { type: 'number' }, + exclusiveMaximum: { type: 'number' }, + minimum: { type: 'number' }, + exclusiveMinimum: { type: 'number' }, + maxLength: { $ref: '#/definitions/nonNegativeInteger' }, + minLength: { $ref: '#/definitions/nonNegativeIntegerDefault0' }, + pattern: { type: 'string', format: 'regex' }, + additionalItems: { $ref: '#' }, + items: { anyOf: [{ $ref: '#' }, { $ref: '#/definitions/schemaArray' }], default: true }, + maxItems: { $ref: '#/definitions/nonNegativeInteger' }, + minItems: { $ref: '#/definitions/nonNegativeIntegerDefault0' }, + uniqueItems: { type: 'boolean', default: false }, + contains: { $ref: '#' }, + maxProperties: { $ref: '#/definitions/nonNegativeInteger' }, + minProperties: { $ref: '#/definitions/nonNegativeIntegerDefault0' }, + required: { $ref: '#/definitions/stringArray' }, + additionalProperties: { $ref: '#' }, + definitions: { type: 'object', additionalProperties: { $ref: '#' }, default: {} }, + properties: { type: 'object', additionalProperties: { $ref: '#' }, default: {} }, + patternProperties: { + type: 'object', + additionalProperties: { $ref: '#' }, + propertyNames: { format: 'regex' }, + default: {}, + }, + dependencies: { + type: 'object', + additionalProperties: { anyOf: [{ $ref: '#' }, { $ref: '#/definitions/stringArray' }] }, + }, + propertyNames: { $ref: '#' }, + const: true, + enum: { type: 'array', items: true, minItems: 1, uniqueItems: true }, + type: { + anyOf: [ + { $ref: '#/definitions/simpleTypes' }, + { + type: 'array', + items: { $ref: '#/definitions/simpleTypes' }, + minItems: 1, + uniqueItems: true, + }, + ], + }, + format: { type: 'string' }, + contentMediaType: { type: 'string' }, + contentEncoding: { type: 'string' }, + if: { $ref: '#' }, + then: { $ref: '#' }, + else: { $ref: '#' }, + allOf: { $ref: '#/definitions/schemaArray' }, + anyOf: { $ref: '#/definitions/schemaArray' }, + oneOf: { $ref: '#/definitions/schemaArray' }, + not: { $ref: '#' }, + }, + default: true, +} + +const META_LOOKUP: Lookup = dereference(META_SCHEMA, Object.create(null) as Lookup, BASE_URI) diff --git a/packages/core/test/tree.test.ts b/packages/core/test/tree.test.ts index c50fd8c..275923f 100644 --- a/packages/core/test/tree.test.ts +++ b/packages/core/test/tree.test.ts @@ -6,12 +6,14 @@ import { type Change, compareUtf8, diffTrees, + entryAt, getEntry, iterate, MapSource, MemorySink, mergeTree, protocolChunking, + rankOf, type RecordEntry, recordTree, sha256Hex, @@ -306,3 +308,20 @@ describe('read', () => { ) }) }) + +describe('rank and offset seeks', () => { + it('rankOf and entryAt agree with the sorted entries (property)', async () => { + await fc.assert( + fc.asyncProperty(keySet(300), keyArb, fc.nat(), async (keys, probe, at) => { + const { sink, source } = store() + const es = sorted(keys.map((k) => entry(k))) + const root = buildTree(recordTree, sink, es, { chunking: tiny })?.hash ?? null + const want = es.filter((e) => compareUtf8(e.key, probe) < 0).length + expect(await rankOf(source, root, probe)).toBe(want) + const i = es.length === 0 ? 0 : at % (es.length + 1) + expect((await entryAt(source, root, i))?.key ?? null).toBe(es[i]?.key ?? null) + }), + { numRuns: 200 * RUNS }, + ) + }) +}) diff --git a/packages/core/test/validate.test.ts b/packages/core/test/validate.test.ts new file mode 100644 index 0000000..2552028 --- /dev/null +++ b/packages/core/test/validate.test.ts @@ -0,0 +1,435 @@ +import { describe, expect, it } from 'vitest' + +import { + checkSchema, + checkSchemaBounds, + compileSchema, + findExtraFields, + MAX_SCHEMA_BYTES, + SchemaError, + stripToSchema, +} from '../src/index.js' + +const errors = (schema: unknown, data: unknown) => compileSchema(schema)(data) +const valid = (schema: unknown, data: unknown) => errors(schema, data).length === 0 +const compileError = (schema: unknown): string | null => { + try { + compileSchema(schema) + return null + } catch (err) { + expect(err).toBeInstanceOf(SchemaError) + return (err as Error).message + } +} + +describe('compileSchema', () => { + it('reuses the validator for identical schema content', () => { + const a = compileSchema({ type: 'object', properties: { x: { type: 'string' } } }) + const b = compileSchema({ properties: { x: { type: 'string' } }, type: 'object' }) + expect(b).toBe(a) + }) + + it('does not modify the schema it is given', () => { + const schema = { + definitions: { a: { type: 'string' } }, + properties: { x: { $ref: '#/definitions/a' } }, + } + const before = JSON.stringify(schema) + compileSchema(schema)({ x: 1 }) + expect(JSON.stringify(schema)).toBe(before) + expect(Object.getOwnPropertyNames(schema.properties.x)).toEqual(['$ref']) + }) + + it('reports errors as " ", in AJV wording', () => { + const schema = { + type: 'object', + required: ['id', 'name'], + properties: { + id: { type: 'integer' }, + tags: { type: 'array', items: { type: 'string' }, maxItems: 2 }, + nested: { type: 'object', properties: { 'a/b': { type: 'string' } } }, + }, + } + expect(errors(schema, { id: 1, name: 'x' })).toEqual([]) + expect(errors(schema, { id: 1.5, tags: ['a', 2, 'c'], nested: { 'a/b': 3 } }).sort()).toEqual( + [ + "/ must have required property 'name'", + '/id must be integer', + '/tags must NOT have more than 2 items', + '/tags/1 must be string', + '/nested/a~1b must be string', + ].sort(), + ) + expect(errors({ type: ['string', 'null'] }, 1)).toEqual(['/ must be string,null']) + }) + + it('checks required and additionalProperties', () => { + const schema = { required: ['a'], properties: { a: {} }, additionalProperties: false } + expect(valid(schema, { a: 1 })).toBe(true) + expect(errors(schema, {})).toEqual(["/ must have required property 'a'"]) + expect(errors(schema, { a: 1, b: 2 })).toEqual(['/ must NOT have additional properties']) + // A subschema for additional properties applies to each of them. + expect(errors({ additionalProperties: { type: 'number' } }, { a: 1, b: 'x' })).toEqual([ + '/b must be number', + ]) + }) + + it('checks enum, const, numbers and strings', () => { + expect(valid({ enum: ['a', 1, null, { k: [1] }] }, { k: [1] })).toBe(true) + expect(errors({ enum: ['a', 1] }, 'b')).toEqual([ + '/ must be equal to one of the allowed values', + ]) + expect(errors({ const: 'a' }, 'b')).toEqual(['/ must be equal to constant']) + expect(errors({ minimum: 2, maximum: 5 }, 1)).toEqual(['/ must be >= 2']) + expect(errors({ exclusiveMaximum: 5 }, 5)).toEqual(['/ must be < 5']) + expect(errors({ minLength: 2 }, 'a')).toEqual(['/ must NOT have fewer than 2 characters']) + expect(errors({ pattern: '^[a-z]+$' }, 'A')).toEqual(['/ must match pattern "^[a-z]+$"']) + // Patterns are Unicode regexes, as in AJV: \p{…} works. + expect(valid({ pattern: '^\\p{L}+$' }, 'été')).toBe(true) + }) + + it('checks the formats used in practice', () => { + const cases: [string, string, boolean][] = [ + ['date', '2024-02-29', true], + ['date', '2023-02-29', false], + ['date', '2024-1-01', false], + ['date-time', '2024-01-01T12:00:00Z', true], + ['date-time', '2024-01-01T12:00:00.5+01:00', true], + ['date-time', '2024-01-01 12:00:00Z', true], + ['date-time', '2024-01-01T12:00:00', false], // no time zone + ['date-time', '2024-01-01', false], + ['email', 'a.b+c@example.org', true], + ['email', 'a@localhost', false], // AJV requires a dot in the domain + ['email', 'not an email', false], + ['uri', 'https://example.org/a?b=c#d', true], + ['uri', 'urn:isbn:0451450523', true], + ['uri', '/relative/path', false], + ['uri', 'example.org', false], + ['uuid', '550e8400-e29b-41d4-a716-446655440000', true], + ['uuid', 'urn:uuid:550e8400-e29b-41d4-a716-446655440000', true], + ['uuid', '550e8400e29b41d4a716446655440000', false], + ] + for (const [format, value, ok] of cases) { + expect([format, value, valid({ type: 'string', format }, value)]).toEqual([format, value, ok]) + } + expect(errors({ format: 'date' }, 'x')).toEqual(['/ must match format "date"']) + // Formats constrain strings only; other types pass. + expect(valid({ format: 'date' }, 5)).toBe(true) + }) + + it('ignores unknown formats', () => { + expect(valid({ type: 'string', format: 'not-a-format' }, 'anything')).toBe(true) + }) + + it('resolves $ref to #/definitions and #/$defs, and applies keywords next to $ref', () => { + const schema = { + definitions: { name: { type: 'string', minLength: 1 } }, + $defs: { count: { type: 'integer', minimum: 0 } }, + properties: { + name: { $ref: '#/definitions/name' }, + count: { $ref: '#/$defs/count' }, + short: { $ref: '#/definitions/name', maxLength: 3 }, + }, + } + expect(valid(schema, { name: 'a', count: 0, short: 'abc' })).toBe(true) + expect(errors(schema, { name: '', count: -1, short: 'abcd' }).sort()).toEqual( + [ + '/count must be >= 0', + '/name must NOT have fewer than 1 characters', + '/short must NOT have more than 3 characters', + ].sort(), + ) + }) + + it('handles a root $ref with sibling definitions (a production schema shape)', () => { + const schema = { + $schema: 'http://json-schema.org/draft-07/schema#', + $id: 'https://example.org/schema.json', + $ref: '#/definitions/Doc', + definitions: { + Doc: { type: 'object', required: ['type'], properties: { type: { enum: ['Doc'] } } }, + }, + } + expect(valid(schema, { type: 'Doc' })).toBe(true) + expect(errors(schema, { type: 'x' })).toEqual([ + '/type must be equal to one of the allowed values', + ]) + }) + + it('validates recursive schemas', () => { + const schema = { + definitions: { + node: { + type: 'object', + properties: { children: { type: 'array', items: { $ref: '#/definitions/node' } } }, + }, + }, + $ref: '#/definitions/node', + } + expect(valid(schema, { children: [{ children: [] }] })).toBe(true) + expect(errors(schema, { children: [{ children: [5] }] })).toEqual([ + '/children/0/children/0 must be object', + ]) + }) + + it('ignores unknown keywords, like AJV with strict: false', () => { + const schema = { + type: 'object', + 'x-ref-type': 'Community', + version: '1.0', + properties: { a: { type: 'string', 'x-ui': { widget: 'text' }, private: false } }, + } + expect(valid(schema, { a: 'x' })).toBe(true) + }) + + it('ignores keywords from later drafts, as draft-07 does', () => { + expect(valid({ properties: { a: {} }, unevaluatedProperties: false }, { b: 1 })).toBe(true) + expect(valid({ dependentRequired: { a: ['b'] } }, { a: 1 })).toBe(true) + expect(valid({ prefixItems: [{ type: 'string' }] }, [1])).toBe(true) + expect(valid({ contains: { type: 'string' }, minContains: 2 }, ['a'])).toBe(true) + // ...but a property that happens to be named like one is still a property. + expect(errors({ properties: { prefixItems: { type: 'string' } } }, { prefixItems: 1 })).toEqual( + ['/prefixItems must be string'], + ) + // draft-07's own `dependencies` applies. + expect(errors({ dependencies: { a: ['b'] } }, { a: 1 })).toEqual([ + '/ must have property b when property a is present', + ]) + }) + + it('accepts boolean subschemas, but not at the root', () => { + expect(valid({ properties: { a: true } }, { a: 1 })).toBe(true) + expect(errors({ properties: { a: false } }, { a: 1 })).toEqual(['/a boolean schema is false']) + expect(compileError(true)).toBe('schema must be an object') + }) + + it('treats inherited JavaScript names as ordinary property names', () => { + expect(errors({ required: ['toString'] }, {})).toEqual([ + "/ must have required property 'toString'", + ]) + expect(valid({ properties: { constructor: { type: 'string' } } }, {})).toBe(true) + expect(errors({ properties: { constructor: { type: 'string' } } }, { constructor: 1 })).toEqual( + ['/constructor must be string'], + ) + expect(errors({ dependencies: { toString: ['b'] } }, {})).toEqual([]) + expect(valid({ type: 'string', format: 'hasOwnProperty' }, 'x')).toBe(true) + // JSON.parse makes "__proto__" an ordinary key; an object literal would not. + const schema = JSON.parse( + '{"required":["__proto__"],"properties":{"__proto__":{"required":["b"]}}}', + ) + expect(errors(schema, JSON.parse('{"__proto__": {"a": 1}}'))).toEqual([ + "/__proto__ must have required property 'b'", + ]) + expect(errors(schema, {})).toEqual(["/ must have required property '__proto__'"]) + }) + + it('applies multipleOf with a tolerance, unlike AJV', () => { + // 0.07 / 0.01 is 7.000000000000001 in binary floating point; AJV (v1) + // rejected it. The tolerance is the library's: |remainder| < 1.1920929e-7. + expect(valid({ multipleOf: 0.01 }, 0.07)).toBe(true) + expect(errors({ multipleOf: 0.01 }, 0.075)).toEqual(['/ must be multiple of 0.01']) + }) + + it('resolves $ref to the draft-07 meta-schema, as AJV does', () => { + const schema = { $ref: 'http://json-schema.org/draft-07/schema#' } + expect(valid(schema, { type: 'string' })).toBe(true) + expect(valid(schema, { type: 3 })).toBe(false) + }) + + it('reports AJV-style messages for dependencies, uniqueItems and propertyNames', () => { + expect(errors({ dependencies: { a: ['b', 'c'] } }, { a: 1, c: 1 })).toEqual([ + '/ must have properties b, c when property a is present', + ]) + expect(errors({ uniqueItems: true }, [1, 2, 1, 2])).toEqual([ + '/ must NOT have duplicate items (items ## 1 and 3 are identical)', + ]) + expect(errors({ propertyNames: { maxLength: 2 } }, { abc: 1 }).sort()).toEqual([ + '/ must NOT have more than 2 characters', + '/ property name must be valid', + ]) + }) + + it('refuses a schema nested too deeply to compile', () => { + let deep: unknown = {} + for (let i = 0; i < 20_000; i++) deep = { items: deep } + expect(compileError(deep)).toMatch(/^schema could not be compiled|^schema is invalid/) + }) + + it('rejects schemas that are not valid draft-07', () => { + expect(compileError('string')).toBe('schema must be an object') + expect(compileError({ type: 'text' })).toMatch(/^schema is invalid: /) + expect(compileError({ required: 'a' })).toMatch(/^schema is invalid: /) + expect(compileError({ pattern: '(' })).toMatch(/^schema is invalid: /) + expect(compileError({ $ref: '#/definitions/missing' })).toBe( + "can't resolve reference #/definitions/missing", + ) + }) + + it('accepts draft-07 $schema and refuses other drafts', () => { + expect(compileError({ $schema: 'http://json-schema.org/draft-07/schema#' })).toBe(null) + expect(compileError({ $schema: 'http://json-schema.org/draft-07/schema' })).toBe(null) + expect(compileError({ $schema: 'https://json-schema.org/draft/2020-12/schema' })).toMatch( + /^unsupported \$schema/, + ) + expect(compileError({ $schema: 'http://json-schema.org/draft-04/schema#' })).toMatch( + /^unsupported \$schema/, + ) + }) + + it('reports every error, except in schemas with branching keywords', () => { + const data = { a: 1, b: 2 } + const props = { a: { type: 'string' }, b: { type: 'string' } } + expect(errors({ properties: props }, data)).toEqual(['/a must be string', '/b must be string']) + // With anyOf/oneOf/not/if/contains anywhere, a properties or items loop + // stops at its first failure: reporting everything is exponential there. + expect(errors({ properties: props, not: { type: 'null' } }, data)).toEqual([ + '/a must be string', + ]) + expect(valid({ properties: props, not: { type: 'null' } }, { a: 'x', b: 'y' })).toBe(true) + }) + + it('stays fast on a deep record under a recursive oneOf schema', () => { + // The shape that made exhaustive checking exponential (a production schema). + const schema = { + $ref: '#/definitions/node', + definitions: { + node: { + oneOf: [ + { + type: 'object', + required: ['type'], + properties: { + type: { const: 'a' }, + children: { type: 'array', items: { $ref: '#/definitions/node' } }, + }, + }, + { + type: 'object', + required: ['type'], + properties: { + type: { const: 'b' }, + children: { type: 'array', items: { $ref: '#/definitions/node' } }, + }, + }, + { + type: 'object', + required: ['type'], + properties: { + type: { const: 'c' }, + children: { type: 'array', items: { $ref: '#/definitions/node' } }, + }, + }, + ], + }, + }, + } + let data: unknown = { type: 'c' } + for (let i = 0; i < 40; i++) + data = { type: i % 2 ? 'a' : 'b', children: [data, { type: 'c', x: 1 }] } + const started = performance.now() + expect(valid(schema, data)).toBe(true) + expect(valid(schema, { type: 'a', children: [{ type: 'z' }] })).toBe(false) + expect(performance.now() - started).toBeLessThan(1000) + }) + + it('reports a schema that recurses without consuming data instead of throwing', () => { + const v = compileSchema({ $ref: '#' }) + expect(v(1)[0]).toMatch(/^\/ schema could not be applied/) + }) +}) + +describe('checkSchema', () => { + it('accepts ordinary schemas, and a private type', () => { + expect( + checkSchema('Person', { type: 'object', properties: { name: { type: 'string' } } }), + ).toBe(null) + expect(checkSchema('Secret', { private: true, type: 'object' })).toBe(null) + expect(checkSchema('Open', { private: false })).toBe(null) + }) + + it('rejects field-level privacy, at any depth', () => { + expect( + checkSchema('Person', { properties: { ssn: { type: 'string', private: true } } }), + ).toMatch(/marks property "\/properties\/ssn" as private/) + expect( + checkSchema('Person', { + definitions: { a: { properties: { b: { properties: { c: { private: true } } } } } }, + }), + ).toMatch(/"\/definitions\/a\/properties\/b\/properties\/c"/) + // A property named "private", or private in a data position, is not the marker. + expect(checkSchema('T', { properties: { private: { type: 'boolean' } } })).toBe(null) + expect( + checkSchema('T', { + properties: { a: { default: { properties: { b: { private: true } } } } }, + }), + ).toBe(null) + }) + + it('requires a root "private" to be a boolean', () => { + expect(checkSchema('T', { private: 'true' })).toMatch(/"private" must be a boolean/) + }) + + it('bounds size on the canonical form', () => { + const big = { description: 'x'.repeat(MAX_SCHEMA_BYTES) } + expect(checkSchema('T', big)).toMatch(/exceeds maximum size/) + // Whitespace isn't counted: the limit applies to the JCS bytes. + const fits = { description: 'x'.repeat(MAX_SCHEMA_BYTES - 20) } + expect(checkSchema('T', fits)).toBe(null) + // Multi-byte characters count as their UTF-8 length. + const wide = { description: 'é'.repeat(MAX_SCHEMA_BYTES / 2) } + expect(checkSchema('T', wide)).toMatch(/exceeds maximum size/) + }) + + it('bounds pattern length, including patternProperties keys', () => { + const long = 'a'.repeat(257) + expect(checkSchema('T', { properties: { a: { pattern: 'a'.repeat(256) } } })).toBe(null) + expect(checkSchema('T', { properties: { a: { pattern: long } } })).toMatch( + /"pattern" longer than 256/, + ) + expect(checkSchema('T', { patternProperties: { [long]: {} } })).toMatch( + /"patternProperties" pattern longer than 256/, + ) + }) + + it('checks type slugs', () => { + expect(checkSchema('a/b', {})).toMatch(/Invalid type slug "a\/b"/) + expect(checkSchema('.hidden', {})).toMatch(/Invalid type slug/) + }) + + it('checkSchemaBounds returns the first error in a set', () => { + expect(checkSchemaBounds({ A: {}, B: { type: 'object' } })).toBe(null) + expect(checkSchemaBounds({ A: {}, B: { properties: { x: { private: true } } } })).toMatch( + /^Schema "B"/, + ) + }) +}) + +describe('extra fields', () => { + const schemas = { T: { properties: { a: {}, b: {} } }, Open: {} } + + it('findExtraFields lists top-level fields not in properties', () => { + expect( + findExtraFields( + [ + { recordId: '1', type: 'T', data: { a: 1, c: 2, d: 3 } }, + { recordId: '2', type: 'T', data: { a: 1 } }, + { recordId: '3', type: 'Open', data: { z: 1 } }, + { recordId: '4', type: 'T', data: 'scalar' }, + { recordId: '5', type: 'T', data: { toString: 1 } }, + ], + schemas, + ), + ).toEqual([ + { recordId: '1', type: 'T', fields: ['c', 'd'] }, + { recordId: '5', type: 'T', fields: ['toString'] }, + ]) + }) + + it('stripToSchema keeps only listed top-level fields', () => { + expect(stripToSchema({ a: 1, c: 2, b: { x: 1 } }, { a: {}, b: {} })).toEqual({ + a: 1, + b: { x: 1 }, + }) + expect(stripToSchema({ constructor: 1 }, {})).toEqual({}) + }) +}) diff --git a/packages/server/src/api/manage.ts b/packages/server/src/api/manage.ts new file mode 100644 index 0000000..ef1c7d6 --- /dev/null +++ b/packages/server/src/api/manage.ts @@ -0,0 +1,253 @@ +/** + * Creating and changing collections (v1 shapes). + * + * POST /api/accounts/:owner/collections {slug, name?, public?} + * PATCH /api/collections/:owner/:slug {name?, slug?, public?} + * DELETE /api/collections/:owner/:slug + * POST /api/collections/:owner/:slug/transfer {targetOrgSlug} + * POST /api/collections/:owner/:slug/fork {targetOrg, slug?} + * POST /api/collections/:owner/:slug/metadata {...metadata patch} + * + * Collection settings are mutable rows; nothing here rewrites version data. + * A metadata edit is a new version that reuses every set: one root object. + */ +import { and, eq } from 'drizzle-orm' +import { type Context, Hono } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import { validateSlug } from '../lib/slug.js' +import { headBase } from '../push/delta.js' +import { commitVersion } from '../versions/commit.js' +import { createCollectionRows, forkCollection } from '../versions/fork.js' +import { jsonError, requireCollection } from './access.js' + +async function membership(c: Context, orgSlug: string) { + const p = c.var.principal + const [org] = await c.var.ports.db + .select() + .from(schema.organization) + .where(eq(schema.organization.slug, orgSlug)) + .limit(1) + if (!org) return { org: null, role: null } + if (!p || p.collectionIds || p.scope === 'read') return { org, role: null } + if (p.orgId) return { org, role: p.orgId === org.id ? 'owner' : null } + const [m] = await c.var.ports.db + .select({ role: schema.member.role }) + .from(schema.member) + .where(and(eq(schema.member.organizationId, org.id), eq(schema.member.userId, p.userId))) + .limit(1) + return { org, role: m?.role ?? null } +} + +const isAdmin = (role: string | null) => role === 'owner' || role === 'admin' + +export function manageRoutes() { + const app = new Hono() + + app.post('/api/accounts/:owner/collections', async (c) => { + if (!c.var.principal) return jsonError(c, 401, 'Authentication required') + const { org, role } = await membership(c, c.req.param('owner')) + if (!org) return jsonError(c, 404, 'Org not found') + if (!role) return jsonError(c, 403, 'Forbidden') + const body = (await c.req.json().catch(() => null)) as { + slug?: unknown + name?: unknown + public?: unknown + } | null + const slugError = validateSlug(body?.slug) + if (slugError) return jsonError(c, 422, slugError) + const slug = body!.slug as string + const [taken] = await c.var.ports.db + .select({ id: schema.collections.id }) + .from(schema.collections) + .where(and(eq(schema.collections.organizationId, org.id), eq(schema.collections.slug, slug))) + .limit(1) + if (taken) return jsonError(c, 409, 'Collection already exists') + const name = + typeof body?.name === 'string' && body.name.trim() ? body.name.trim().slice(0, 200) : slug + const col = await createCollectionRows(c.var.ports, { + organizationId: org.id, + slug, + name, + public: body?.public === true, + }) + return c.json({ id: col.id, owner: org.slug, slug, name }, 201) + }) + + app.patch('/api/collections/:owner/:slug', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const body = (await c.req.json().catch(() => ({}))) as { + name?: unknown + slug?: unknown + public?: unknown + } + const set: Partial = {} + if (typeof body.name === 'string' && body.name.trim()) set.name = body.name.trim().slice(0, 200) + if (body.public !== undefined) { + // Visibility is an admin decision (v1 rule): it changes who can read every version. + if (!isAdmin(access.role)) + return jsonError(c, 403, 'Only org owners and admins can change visibility') + set.public = body.public === true + } + if (body.slug !== undefined && body.slug !== access.collection.slug) { + const err = validateSlug(body.slug) + if (err) return jsonError(c, 422, err) + const [taken] = await c.var.ports.db + .select({ id: schema.collections.id }) + .from(schema.collections) + .where( + and( + eq(schema.collections.organizationId, access.owner.id), + eq(schema.collections.slug, body.slug as string), + ), + ) + .limit(1) + if (taken) return jsonError(c, 409, 'Collection already exists') + set.slug = body.slug as string + } + if (Object.keys(set).length > 0) { + await c.var.ports.db + .update(schema.collections) + .set({ ...set, updatedAt: new Date() }) + .where(eq(schema.collections.id, access.collection.id)) + } + return c.json({ ok: true, slug: set.slug ?? access.collection.slug }) + }) + + app.delete('/api/collections/:owner/:slug', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + if (!isAdmin(access.role)) return jsonError(c, 403, 'Forbidden') + // Rows go (cascading to versions, sessions, placements, webhooks). Repository + // objects stay until garbage collection exists (edge-redesign.md, Storage layout). + await c.var.ports.db + .delete(schema.collections) + .where(eq(schema.collections.id, access.collection.id)) + return c.json({ ok: true }) + }) + + app.post('/api/collections/:owner/:slug/transfer', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + if (!isAdmin(access.role)) return jsonError(c, 403, 'Forbidden') + const body = (await c.req.json().catch(() => ({}))) as { targetOrgSlug?: unknown } + if (typeof body.targetOrgSlug !== 'string') + return jsonError(c, 400, '"targetOrgSlug" is required') + const target = await membership(c, body.targetOrgSlug) + if (!target.org) return jsonError(c, 404, 'Target org not found') + if (!isAdmin(target.role)) + return jsonError(c, 403, 'You must be an owner or admin of the target org') + const [taken] = await c.var.ports.db + .select({ id: schema.collections.id }) + .from(schema.collections) + .where( + and( + eq(schema.collections.organizationId, target.org.id), + eq(schema.collections.slug, access.collection.slug), + ), + ) + .limit(1) + if (taken) return jsonError(c, 409, 'The target org already has a collection with this slug') + await c.var.ports.db + .update(schema.collections) + .set({ organizationId: target.org.id, updatedAt: new Date() }) + .where(eq(schema.collections.id, access.collection.id)) + return c.json({ ok: true, newOwner: target.org.slug }) + }) + + app.post('/api/collections/:owner/:slug/fork', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + if (!c.var.principal) return jsonError(c, 401, 'Authentication required') + const body = (await c.req.json().catch(() => ({}))) as { targetOrg?: unknown; slug?: unknown } + if (typeof body.targetOrg !== 'string') return jsonError(c, 400, '"targetOrg" is required') + const target = await membership(c, body.targetOrg) + if (!target.org) return jsonError(c, 404, 'Target org not found') + if (!target.role) return jsonError(c, 403, 'Forbidden') + const slug = typeof body.slug === 'string' ? body.slug : access.collection.slug + const err = validateSlug(slug) + if (err) return jsonError(c, 422, err) + const head = access.collection.headVersionId + if (!head) return jsonError(c, 422, 'Nothing to fork: the collection has no versions') + const [version] = await c.var.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, head)) + const [taken] = await c.var.ports.db + .select({ id: schema.collections.id }) + .from(schema.collections) + .where( + and( + eq(schema.collections.organizationId, target.org.id), + eq(schema.collections.slug, slug), + ), + ) + .limit(1) + if (taken) return jsonError(c, 409, 'Collection already exists') + const forked = await forkCollection( + c.var.ports, + { collection: access.collection, version: version! }, + { organizationId: target.org.id, slug, name: access.collection.name, public: false }, + access.isMember, + ) + return c.json( + { + id: forked.collection.id, + owner: target.org.slug, + slug, + name: forked.collection.name, + forkedFrom: { + owner: access.owner.slug, + slug: access.collection.slug, + version: version!.semver, + }, + version: { semver: forked.version.semver, recordCount: forked.version.recordCount }, + }, + 201, + ) + }) + + app.post('/api/collections/:owner/:slug/metadata', async (c) => { + const access = await requireCollection(c, 'write') + if (access instanceof Response) return access + const patch = (await c.req.json().catch(() => null)) as Record | null + if (!patch || typeof patch !== 'object' || Array.isArray(patch)) + return jsonError(c, 400, 'Body must be an object') + const ports = c.var.ports + const base = await headBase(ports, access.collection.id) + if (!base) return jsonError(c, 422, 'No versions exist yet') + const repo = await ports.stores.forCollection(access.collection.id) + const root = await repo.root(base.hash) + const priv = root.private ? await repo.privateSet(root.private) : null + const metadata = { ...(root.metadata ?? {}), ...patch } + if (JSON.stringify(metadata) === JSON.stringify(root.metadata)) + return c.json({ semver: base.semver, unchanged: true }) + const hashes = new Map() + for (const [slug, t] of Object.entries(priv?.types ?? {})) hashes.set(slug, t.schema) + for (const [slug, t] of Object.entries(root.public.types)) hashes.set(slug, t.schema) + const types = await Promise.all( + [...hashes].map(async ([slug, h]) => ({ + slug, + schema: await repo.schema(h), + schemaHash: h, + public: null, + private: null, + })), + ) + const r = await commitVersion(ports, { + collectionId: access.collection.id, + base, + types, + metadata, + message: 'Metadata update', + pushedBy: c.var.principal?.userId ?? null, + }) + if (r.status === 'conflict') return jsonError(c, 409, 'Version conflict') + if (r.status !== 'committed') return c.json({ semver: base.semver, unchanged: true }) + return c.json({ semver: r.version.semver, hash: r.version.hash, status: 'completed' }, 201) + }) + + return app +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 202403b..c236ac4 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -8,6 +8,7 @@ import { type Context, type ExecutionContext, Hono } from 'hono' import type { Principal } from './api/access.js' import { collectionRoutes } from './api/collections.js' import { fileRoutes } from './api/files.js' +import { manageRoutes } from './api/manage.js' import { pushRoutes } from './api/push.js' import { versionRoutes } from './api/versions.js' import { webhookRoutes } from './api/webhooks.js' @@ -88,6 +89,7 @@ export function createApp(setup: Setup) { app.route('/api/collections', pushRoutes()) app.route('/api/collections', versionRoutes()) app.route('/api/collections', webhookRoutes()) + app.route('/', manageRoutes()) app.route('/', collectionRoutes()) // Everything else is a UI page, when the deployment renders one. diff --git a/packages/server/src/versions/fork.ts b/packages/server/src/versions/fork.ts new file mode 100644 index 0000000..fc0347d --- /dev/null +++ b/packages/server/src/versions/fork.ts @@ -0,0 +1,139 @@ +/** + * Fork (edge-redesign.md, Commit): the fork's first version is a new root that + * reuses the source version's sets, plus a `forks` row. No data is copied. + * + * A fork by an owner keeps the private set; it inherits the source collection's + * salt so the private commitment (and so the version hash) stays the same and + * later pushes of the same content still dedupe. A fork by anyone else takes + * the public set only. Both collections must share a primary location, since a + * repository never references another location. + */ +import { makeRoot, newSalt } from '@underlay/core' +import { eq } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Ports } from '../ports.js' +import { appendVersionLog } from './commit.js' +import { publishVersion, type SchemaUsageChange } from './publish.js' + +export async function createCollectionRows( + ports: Ports, + c: { organizationId: string; slug: string; name: string; public: boolean; privateSalt?: string }, +): Promise { + const id = crypto.randomUUID() + await ports.db.batch([ + ports.db.insert(schema.collections).values({ + id, + organizationId: c.organizationId, + slug: c.slug, + name: c.name, + public: c.public, + privateSalt: c.privateSalt ?? newSalt(), + }), + ports.db.insert(schema.placements).values({ + collectionId: id, + locationId: schema.PLATFORM_LOCATION_ID, + role: 'primary', + sets: 'public+private', + }), + ]) + const [row] = await ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, id)) + return row! +} + +export async function forkCollection( + ports: Ports, + source: { + collection: typeof schema.collections.$inferSelect + version: typeof schema.versions.$inferSelect + }, + target: { organizationId: string; slug: string; name: string; public: boolean }, + includePrivate: boolean, +): Promise<{ + collection: typeof schema.collections.$inferSelect + version: typeof schema.versions.$inferSelect +}> { + const repo = await ports.stores.forCollection(source.collection.id) + const root = await repo.root(source.version.hash) + const priv = includePrivate && root.private ? await repo.privateSet(root.private) : null + + const collection = await createCollectionRows(ports, { + ...target, + ...(priv ? { privateSalt: source.collection.privateSalt } : {}), + }) + const targetRepo = await ports.stores.forCollection(collection.id) + if (targetRepo !== repo) throw new Error('Forks across storage locations are not supported yet') + + const newRoot = makeRoot(root.metadata, root.public, priv) + const hash = await repo.putRoot(newRoot) + + const v = source.version + const keepPrivate = !!priv + const usage: SchemaUsageChange[] = [ + ...Object.entries(root.public.types).map(([slug, t]) => ({ + set: 'public' as const, + typeSlug: slug, + schemaHash: t.schema, + wasOpen: false, + })), + ...Object.entries(priv?.types ?? {}).map(([slug, t]) => ({ + set: 'private' as const, + typeSlug: slug, + schemaHash: t.schema, + wasOpen: false, + })), + ] + const versionId = crypto.randomUUID() + const published = await publishVersion(ports.db, { + version: { + id: versionId, + collectionId: collection.id, + seq: 1, + semver: 'v1.0.0', + major: 1, + minor: 0, + patch: 0, + hash, + baseSemver: null, + message: `Forked from ${source.collection.slug} ${v.semver}`, + pushedBy: null, + appId: null, + actorId: null, + recordCount: keepPrivate ? v.recordCount : v.publicRecordCount, + publicRecordCount: v.publicRecordCount, + fileCount: keepPrivate ? v.fileCount : v.publicFileCount, + totalBytes: keepPrivate ? v.totalBytes : v.publicTotalBytes, + publicFileCount: v.publicFileCount, + publicTotalBytes: v.publicTotalBytes, + typeCounts: keepPrivate ? v.typeCounts : v.publicTypeCounts, + publicTypeCounts: v.publicTypeCounts, + hasPrivate: newRoot.private !== null, + publicRefsRoot: v.publicRefsRoot, + privateRefsRoot: keepPrivate ? v.privateRefsRoot : null, + changes: { added: keepPrivate ? v.recordCount : v.publicRecordCount, removed: 0, updated: 0 }, + }, + baseVersionId: null, + collectionUpdate: { + publicFilesRoot: root.public.files.root, + summary: source.collection.summary, + }, + schemaHashes: [], + usage, + }) + if (!published.ok) throw new Error('Fork publish lost a race on a brand-new collection') + await ports.db.insert(schema.forks).values({ + childCollectionId: collection.id, + parentCollectionId: source.collection.id, + parentSeq: v.seq, + }) + const [version] = await ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, versionId)) + await appendVersionLog(ports, repo, collection.id, version!) + await ports.jobs.enqueue({ type: 'version.published', versionId, bump: 'major' }) + return { collection, version: version! } +} diff --git a/packages/server/test/manage.test.ts b/packages/server/test/manage.test.ts new file mode 100644 index 0000000..0c73923 --- /dev/null +++ b/packages/server/test/manage.test.ts @@ -0,0 +1,176 @@ +import { readHead, verifyLog } from '@underlay/repo' +import { eq } from 'drizzle-orm' +import { afterAll, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { cleanup, harness } from './harness.js' + +afterAll(cleanup) + +const Author = { type: 'object', properties: { name: { type: 'string' } } } + +async function json(res: Response) { + return (await res.json()) as any +} + +describe('collection management', () => { + it('creates, updates, edits metadata, forks, transfers and deletes', async () => { + const h = await harness() + const user = await h.member() + let res = await h.request('/api/accounts/org/collections', { + method: 'POST', + user, + json: { slug: 'new-one', name: 'New One', public: true }, + }) + expect(res.status).toBe(201) + expect( + ( + await h.request('/api/accounts/org/collections', { + method: 'POST', + user, + json: { slug: 'new-one' }, + }) + ).status, + ).toBe(409) + expect( + ( + await h.request('/api/accounts/org/collections', { + method: 'POST', + user, + json: { slug: 'API' }, + }) + ).status, + ).toBe(422) + expect( + (await h.request('/api/accounts/org/collections', { method: 'POST', json: { slug: 'x' } })) + .status, + ).toBe(401) + + const base = '/api/collections/org/new-one' + const sid = ( + await json( + await h.request(`${base}/push`, { + method: 'POST', + user, + json: { schemas: { Author }, metadata: { title: 'T' } }, + }), + ) + ).session_id + await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [ + { id: 'a', type: 'Author', data: { name: 'A' } }, + { id: 'b', type: 'Author', data: { name: 'B' }, private: true }, + ], + }) + expect((await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user })).status).toBe( + 201, + ) + + // Metadata edit: a patch version reusing every set. + res = await h.request(`${base}/metadata`, { + method: 'POST', + user, + json: { description: 'Edited' }, + }) + expect(res.status).toBe(201) + expect((await json(res)).semver).toBe('v1.0.1') + const detail = await json(await h.request(base)) + expect(detail.latestVersion.metadata).toEqual({ title: 'T', description: 'Edited' }) + expect(detail.description).toBe('Edited') + + // Fork into another org by a member there: public set only, no data copied. + await h.ports.db.insert(schema.user).values({ id: 'u2', name: 'u2', email: 'u2@example.org' }) + await h.ports.db.insert(schema.organization).values({ id: 'org2', name: 'Two', slug: 'two' }) + await h.ports.db + .insert(schema.member) + .values({ organizationId: 'org2', userId: 'u2', role: 'owner' }) + const putsBefore = h.bucket.puts + res = await h.request(`${base}/fork`, { + method: 'POST', + user: 'u2', + json: { targetOrg: 'two' }, + }) + expect(res.status).toBe(201) + const fork = await json(res) + expect(fork).toMatchObject({ + owner: 'two', + slug: 'new-one', + forkedFrom: { version: 'v1.0.1' }, + version: { recordCount: 1 }, + }) + expect(h.bucket.puts - putsBefore).toBeLessThanOrEqual(4) // root, log entry, head.json + const [child] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, fork.id)) + const repo = await h.ports.stores.forCollection(child!.id) + const [srcHead] = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.semver, 'v1.0.1')) + const forkVersion = ( + await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.collectionId, child!.id)) + )[0]! + const srcRoot = await repo.root(srcHead!.hash) + const forkRoot = await repo.root(forkVersion.hash) + expect(forkRoot.public).toEqual(srcRoot.public) + expect(forkRoot.private).toBe(null) + expect((await readHead(repo, child!.id))?.seq).toBe(1) + await expect(verifyLog(repo, child!.id, [h.signer.publicKey])).resolves.toBeTruthy() + // The fork can push on top of its first version. + const fsid = ( + await json( + await h.request('/api/collections/two/new-one/push', { + method: 'POST', + user: 'u2', + json: { base: 'v1.0.0' }, + }), + ) + ).session_id + await h.request(`/api/collections/two/new-one/push/${fsid}/records`, { + method: 'POST', + user: 'u2', + ndjson: [{ id: 'c', type: 'Author', data: { name: 'C' } }], + }) + expect( + ( + await h.request(`/api/collections/two/new-one/push/${fsid}/commit`, { + method: 'POST', + user: 'u2', + }) + ).status, + ).toBe(201) + + // Rename and visibility; transfer needs admin in both orgs. + res = await h.request(base, { method: 'PATCH', user, json: { slug: 'renamed', public: false } }) + expect(await json(res)).toEqual({ ok: true, slug: 'renamed' }) + expect((await h.request('/api/collections/org/renamed')).status).toBe(404) + expect( + ( + await h.request('/api/collections/org/renamed/transfer', { + method: 'POST', + user, + json: { targetOrgSlug: 'two' }, + }) + ).status, + ).toBe(403) + await h.ports.db + .insert(schema.member) + .values({ organizationId: 'org2', userId: user, role: 'admin' }) + res = await h.request('/api/collections/org/renamed/transfer', { + method: 'POST', + user, + json: { targetOrgSlug: 'two' }, + }) + expect(await json(res)).toEqual({ ok: true, newOwner: 'two' }) + expect( + (await h.request('/api/collections/two/renamed', { method: 'DELETE', user })).status, + ).toBe(200) + expect((await h.request('/api/collections/two/renamed', { user })).status).toBe(404) + }) +}) From a2d05c0ca0bd06cb9331fae866c3d2aa23bab6a9 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 17:20:30 -0400 Subject: [PATCH 014/178] v2 schemas: collection schemas, global schema search and detail, labels Schema ids are hashes. Visibility comes from schema_usage (public set of a public collection, or a collection of the caller's orgs), so a private type's schema stays hidden. q matches labels and type slugs; bodies come from the repository. --- packages/server/src/api/schemas.ts | 278 +++++++++++++++++++++++++++ packages/server/src/app.ts | 2 + packages/server/test/schemas.test.ts | 71 +++++++ 3 files changed, 351 insertions(+) create mode 100644 packages/server/src/api/schemas.ts create mode 100644 packages/server/test/schemas.test.ts diff --git a/packages/server/src/api/schemas.ts b/packages/server/src/api/schemas.ts new file mode 100644 index 0000000..d5c275d --- /dev/null +++ b/packages/server/src/api/schemas.ts @@ -0,0 +1,278 @@ +/** + * Schema reads and labels (v1 shapes, with the schema hash as the id). + * + * GET /api/collections/:owner/:slug/schemas?version&raw + * GET /api/schemas?label|slug|schema_hash|q&limit&offset + * GET /api/schemas/:id id = schema hash + * POST /api/schemas/:id/labels {label} + * DELETE /api/schemas/:id/labels/:label admin keys only + * + * Visibility comes from schema_usage: a schema is visible when it is used in + * the public set of a public collection, or by a collection in one of the + * caller's orgs. `q` matches labels and type slugs; schema bodies live in the + * repository, not SQLite, so there is no full-text search over them. + */ +import { and, desc, eq, inArray, isNull, or, sql } from 'drizzle-orm' +import { type Context, Hono } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import { findVersion, loadView } from '../versions/view.js' +import { jsonError, requireCollection } from './access.js' + +const MAX_LABEL_LENGTH = 100 + +async function callerOrgIds(c: Context): Promise { + const p = c.var.principal + if (!p || p.collectionIds) return [] + if (p.orgId) return [p.orgId] + const rows = await c.var.ports.db + .select({ id: schema.member.organizationId }) + .from(schema.member) + .where(eq(schema.member.userId, p.userId)) + return rows.map((r) => r.id) +} + +/** A SQL condition: this schema hash is visible to the caller. */ +function visibleSchema(hashCol: unknown, orgIds: string[]) { + return sql`EXISTS ( + SELECT 1 FROM ${schema.schemaUsage} u + JOIN ${schema.collections} c ON c.id = u.collection_id + WHERE u.schema_hash = ${hashCol} + AND ((c.public = 1 AND u."set" = 'public') + ${ + orgIds.length + ? sql`OR c.organization_id IN (${sql.join( + orgIds.map((id) => sql`${id}`), + sql`, `, + )})` + : sql`` + }) + )` +} + +async function labelsFor(c: Context, hashes: string[]) { + if (hashes.length === 0) return new Map() + const rows = await c.var.ports.db + .select() + .from(schema.schemaLabels) + .where(inArray(schema.schemaLabels.schemaHash, hashes)) + const out = new Map() + for (const r of rows) { + if (!out.has(r.schemaHash)) out.set(r.schemaHash, []) + out.get(r.schemaHash)!.push({ label: r.label, createdAt: r.createdAt }) + } + return out +} + +/** Schema bodies come from a repository that holds them: any collection using the schema. */ +async function schemaBody( + c: Context, + hash: string, +): Promise | null> { + const [u] = await c.var.ports.db + .select({ collectionId: schema.schemaUsage.collectionId }) + .from(schema.schemaUsage) + .where(eq(schema.schemaUsage.schemaHash, hash)) + .limit(1) + if (!u) return null + const repo = await c.var.ports.stores.forCollection(u.collectionId) + return repo.schema(hash) +} + +export function schemaRoutes() { + const app = new Hono() + + app.get('/api/collections/:owner/:slug/schemas', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const v = await findVersion( + c.var.ports.db, + access.collection.id, + c.req.query('version') ?? 'latest', + access.collection.headVersionId, + ) + if (!v) return jsonError(c, 404, 'No versions found') + const repo = await c.var.ports.stores.forCollection(access.collection.id) + const view = await loadView(repo, v, access.isMember) + const raw = c.req.query('raw') === 'true' + const labels = raw + ? new Map() + : await labelsFor( + c, + view.types.map((t) => t.schemaHash), + ) + const schemas = await Promise.all( + view.types.map(async (t) => { + const body = await repo.schema(t.schemaHash) + const l = labels.get(t.schemaHash) + return { + slug: t.slug, + schemaId: t.schemaHash, + schemaHash: t.schemaHash, + schema: l?.length + ? { ...body, 'x-underlay-labels': l.map((x: { label: string }) => x.label) } + : body, + } + }), + ) + return c.json({ version: v.semver, semver: v.semver, schemas }) + }) + + app.get('/api/schemas', async (c) => { + const orgIds = await callerOrgIds(c) + const limit = Math.min(100, Math.max(1, Number(c.req.query('limit') ?? 50) || 50)) + const offset = Math.max(0, Number(c.req.query('offset') ?? 0) || 0) + const { db } = c.var.ports + const one = c.req.query('schema_hash') + if (one) { + const [row] = await db + .select() + .from(schema.schemas) + .where(and(eq(schema.schemas.hash, one), visibleSchema(schema.schemas.hash, orgIds))) + if (!row) return jsonError(c, 404, 'Schema not found') + const [usage] = await db + .select({ n: sql`count(DISTINCT collection_id)` }) + .from(schema.schemaUsage) + .where(eq(schema.schemaUsage.schemaHash, one)) + const labels = (await labelsFor(c, [one])).get(one) ?? [] + return c.json({ + id: one, + schemaHash: one, + schema: await schemaBody(c, one), + createdAt: row.createdAt, + labels: labels.map((l) => l.label), + usageCount: usage?.n ?? 0, + }) + } + const label = c.req.query('label') + const slug = c.req.query('slug') + const q = c.req.query('q') + const conds = [visibleSchema(schema.schemas.hash, orgIds)] + if (label) + conds.push( + sql`EXISTS (SELECT 1 FROM ${schema.schemaLabels} l WHERE l.schema_hash = ${schema.schemas.hash} AND l.label LIKE ${`%${label}%`})`, + ) + if (slug) + conds.push( + sql`EXISTS (SELECT 1 FROM ${schema.schemaUsage} u WHERE u.schema_hash = ${schema.schemas.hash} AND u.type_slug = ${slug})`, + ) + if (q) { + conds.push( + or( + sql`EXISTS (SELECT 1 FROM ${schema.schemaLabels} l WHERE l.schema_hash = ${schema.schemas.hash} AND l.label LIKE ${`%${q}%`})`, + sql`EXISTS (SELECT 1 FROM ${schema.schemaUsage} u WHERE u.schema_hash = ${schema.schemas.hash} AND u.type_slug LIKE ${`%${q}%`})`, + )!, + ) + } + const rows = await db + .select() + .from(schema.schemas) + .where(and(...conds)) + .orderBy(desc(schema.schemas.createdAt)) + .limit(limit) + .offset(offset) + const labels = await labelsFor( + c, + rows.map((r) => r.hash), + ) + const out = await Promise.all( + rows.map(async (r) => ({ + id: r.hash, + schemaHash: r.hash, + schema: await schemaBody(c, r.hash), + createdAt: r.createdAt, + labels: (labels.get(r.hash) ?? []).map((l) => l.label), + })), + ) + return c.json(out) + }) + + app.get('/api/schemas/:id', async (c) => { + const orgIds = await callerOrgIds(c) + const hash = c.req.param('id') + const { db } = c.var.ports + const [row] = await db + .select() + .from(schema.schemas) + .where(and(eq(schema.schemas.hash, hash), visibleSchema(schema.schemas.hash, orgIds))) + if (!row) return jsonError(c, 404, 'Schema not found') + const usage = await db + .select({ + slug: schema.schemaUsage.typeSlug, + owner: schema.organization.slug, + collection: schema.collections.slug, + headSemver: schema.versions.semver, + }) + .from(schema.schemaUsage) + .innerJoin(schema.collections, eq(schema.collections.id, schema.schemaUsage.collectionId)) + .innerJoin(schema.organization, eq(schema.organization.id, schema.collections.organizationId)) + .leftJoin(schema.versions, eq(schema.versions.id, schema.collections.headVersionId)) + .where( + and( + eq(schema.schemaUsage.schemaHash, hash), + eq(schema.collections.public, true), + eq(schema.schemaUsage.set, 'public'), + isNull(schema.schemaUsage.toSeq), + ), + ) + .limit(50) + return c.json({ + id: hash, + schemaHash: hash, + schema: await schemaBody(c, hash), + createdAt: row.createdAt, + labels: (await labelsFor(c, [hash])).get(hash) ?? [], + usage: usage.map((u) => ({ + slug: u.slug, + semver: u.headSemver, + collection: `${u.owner}/${u.collection}`, + })), + }) + }) + + app.post('/api/schemas/:id/labels', async (c) => { + const p = c.var.principal + if (!p) return jsonError(c, 401, 'Authentication required') + if (p.scope === 'read') return jsonError(c, 403, 'Write access required') + const hash = c.req.param('id') + const body = (await c.req.json().catch(() => ({}))) as { label?: unknown } + const label = typeof body.label === 'string' ? body.label.trim() : '' + if (!label) return jsonError(c, 400, 'Label is required') + if (label.length > MAX_LABEL_LENGTH) + return jsonError(c, 400, `Label must be at most ${MAX_LABEL_LENGTH} characters`) + const [row] = await c.var.ports.db + .select() + .from(schema.schemas) + .where( + and( + eq(schema.schemas.hash, hash), + visibleSchema(schema.schemas.hash, await callerOrgIds(c)), + ), + ) + if (!row) return jsonError(c, 404, 'Schema not found') + const inserted = await c.var.ports.db + .insert(schema.schemaLabels) + .values({ schemaHash: hash, label }) + .onConflictDoNothing() + .returning() + return inserted.length + ? c.json({ status: 'created', schemaId: hash, label }, 201) + : c.json({ status: 'exists', schemaId: hash, label }) + }) + + app.delete('/api/schemas/:id/labels/:label', async (c) => { + if (c.var.principal?.scope !== 'admin') return jsonError(c, 403, 'Admin access required') + await c.var.ports.db + .delete(schema.schemaLabels) + .where( + and( + eq(schema.schemaLabels.schemaHash, c.req.param('id')), + eq(schema.schemaLabels.label, c.req.param('label')), + ), + ) + return c.json({ ok: true }) + }) + + return app +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index c236ac4..fd411a9 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -10,6 +10,7 @@ import { collectionRoutes } from './api/collections.js' import { fileRoutes } from './api/files.js' import { manageRoutes } from './api/manage.js' import { pushRoutes } from './api/push.js' +import { schemaRoutes } from './api/schemas.js' import { versionRoutes } from './api/versions.js' import { webhookRoutes } from './api/webhooks.js' import type { Ports } from './ports.js' @@ -89,6 +90,7 @@ export function createApp(setup: Setup) { app.route('/api/collections', pushRoutes()) app.route('/api/collections', versionRoutes()) app.route('/api/collections', webhookRoutes()) + app.route('/', schemaRoutes()) app.route('/', manageRoutes()) app.route('/', collectionRoutes()) diff --git a/packages/server/test/schemas.test.ts b/packages/server/test/schemas.test.ts new file mode 100644 index 0000000..23c1b65 --- /dev/null +++ b/packages/server/test/schemas.test.ts @@ -0,0 +1,71 @@ +import { hashSchema } from '@underlay/core' +import { afterAll, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { cleanup, harness } from './harness.js' + +afterAll(cleanup) + +const Author = { type: 'object', properties: { name: { type: 'string' } } } +const Secret = { type: 'object', private: true, properties: { note: { type: 'string' } } } + +async function json(res: Response) { + return (await res.json()) as any +} + +describe('schemas', () => { + it('lists collection schemas and global schemas by visibility, with labels', async () => { + const h = await harness() + const user = await h.member() + await h.collection('lib') + await h.ports.db.update(schema.collections).set({ public: true }) + const base = '/api/collections/org/lib' + const sid = ( + await json( + await h.request(`${base}/push`, { + method: 'POST', + user, + json: { schemas: { Author, Secret } }, + }), + ) + ).session_id + await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [ + { id: 'a', type: 'Author', data: { name: 'A' } }, + { id: 's', type: 'Secret', data: { note: 'n' } }, + ], + }) + expect((await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user })).status).toBe( + 201, + ) + + const anon = await json(await h.request(`${base}/schemas`)) + expect(anon.schemas.map((s: any) => s.slug)).toEqual(['Author']) + expect(anon.schemas[0]).toMatchObject({ schemaHash: hashSchema(Author), schema: Author }) + const owner = await json(await h.request(`${base}/schemas`, { user })) + expect(owner.schemas.map((s: any) => s.slug)).toEqual(['Author', 'Secret']) + + // Global: the private type's schema is visible to members only. + expect((await h.request(`/api/schemas/${hashSchema(Secret)}`)).status).toBe(404) + expect((await h.request(`/api/schemas/${hashSchema(Secret)}`, { user })).status).toBe(200) + const detail = await json(await h.request(`/api/schemas/${hashSchema(Author)}`)) + expect(detail.usage).toEqual([{ slug: 'Author', semver: 'v1.0.0', collection: 'org/lib' }]) + + expect( + ( + await h.request(`/api/schemas/${hashSchema(Author)}/labels`, { + method: 'POST', + user, + json: { label: 'person' }, + }) + ).status, + ).toBe(201) + const byLabel = await json(await h.request('/api/schemas?label=pers')) + expect(byLabel.map((s: any) => s.labels)).toEqual([['person']]) + const labelled = await json(await h.request(`${base}/schemas`)) + expect(labelled.schemas[0].schema['x-underlay-labels']).toEqual(['person']) + expect((await json(await h.request('/api/schemas?q=Auth'))).length).toBe(1) + }) +}) From ea3c842409c160449f828452674c4b7134ac2b52 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 17:26:44 -0400 Subject: [PATCH 015/178] v2 reference log (phase 7): provenance, records by hash, files by hash After each publish a job diffs the version against the previous one (O(changes)) into events [hash, kind, collection, set, seq, op, type, id], sorts them by hash (spilling to sorted runs past 100k) and writes an immutable tier-0 run of segments: gzip blocks of ~1,024 events, an index with a Bloom filter, at most 256k events per segment. Size- tiered compaction merges FAN_IN runs into the next tier as hash-range parts run as jobs and swaps them in atomically. Segment rows, counters (collections.ref_events/ref_bytes: canonical event bytes) and the indexed flag go in one batch guarded on the version still being unindexed, so retries don't double count. Forks write no events; queries extend a parent's intervals into its forks until the fork removes the record. Routes: GET /api/records/:hash/provenance, POST /api/records/batch, GET /api/collections/files/:hash, all filtered by the caller's access before anything is returned. Format 1 hashes resolve via legacy_hashes. --- packages/server/drizzle/0000_init.sql | 33 ++ .../server/drizzle/meta/0000_snapshot.json | 198 +++++++- .../server/drizzle/meta/0001_snapshot.json | 200 +++++++- packages/server/drizzle/meta/_journal.json | 4 +- packages/server/src/api/records.ts | 178 +++++++ packages/server/src/app.ts | 3 + packages/server/src/db/schema.ts | 57 +++ packages/server/src/handlers.ts | 2 + packages/server/src/refs/log.ts | 437 ++++++++++++++++++ packages/server/src/refs/segments.ts | 240 ++++++++++ packages/server/src/versions/publish.ts | 1 + packages/server/test/refs.test.ts | 159 +++++++ 12 files changed, 1507 insertions(+), 5 deletions(-) create mode 100644 packages/server/src/api/records.ts create mode 100644 packages/server/src/refs/log.ts create mode 100644 packages/server/src/refs/segments.ts create mode 100644 packages/server/test/refs.test.ts diff --git a/packages/server/drizzle/0000_init.sql b/packages/server/drizzle/0000_init.sql index 2784ca3..86f221e 100644 --- a/packages/server/drizzle/0000_init.sql +++ b/packages/server/drizzle/0000_init.sql @@ -289,6 +289,38 @@ CREATE TABLE `push_sessions` ( --> statement-breakpoint CREATE INDEX `push_sessions_collection_idx` ON `push_sessions` (`collection_id`);--> statement-breakpoint CREATE INDEX `push_sessions_expires_idx` ON `push_sessions` (`status`,`expires_at`);--> statement-breakpoint +CREATE TABLE `ref_compaction_parts` ( + `compaction_id` text NOT NULL, + `part` integer NOT NULL, + PRIMARY KEY(`compaction_id`, `part`) +); +--> statement-breakpoint +CREATE TABLE `ref_compactions` ( + `id` text PRIMARY KEY NOT NULL, + `tier` integer NOT NULL, + `input_runs` text NOT NULL, + `output_run` text NOT NULL, + `parts` integer NOT NULL, + `prefix_length` integer NOT NULL, + `status` text DEFAULT 'running' NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL +); +--> statement-breakpoint +CREATE TABLE `ref_segments` ( + `id` text PRIMARY KEY NOT NULL, + `run_id` text NOT NULL, + `tier` integer NOT NULL, + `first_hash` text NOT NULL, + `last_hash` text NOT NULL, + `count` integer NOT NULL, + `bytes` integer NOT NULL, + `state` text DEFAULT 'live' NOT NULL, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL +); +--> statement-breakpoint +CREATE INDEX `ref_segments_range_idx` ON `ref_segments` (`state`,`first_hash`,`last_hash`);--> statement-breakpoint +CREATE INDEX `ref_segments_run_idx` ON `ref_segments` (`run_id`);--> statement-breakpoint +CREATE INDEX `ref_segments_tier_idx` ON `ref_segments` (`tier`);--> statement-breakpoint CREATE TABLE `schema_labels` ( `schema_hash` text NOT NULL, `label` text NOT NULL, @@ -396,6 +428,7 @@ CREATE TABLE `versions` ( `has_private` integer DEFAULT false NOT NULL, `public_refs_root` text, `private_refs_root` text, + `refs_indexed` integer DEFAULT false NOT NULL, `changes` text, `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, FOREIGN KEY (`collection_id`) REFERENCES `collections`(`id`) ON UPDATE no action ON DELETE cascade diff --git a/packages/server/drizzle/meta/0000_snapshot.json b/packages/server/drizzle/meta/0000_snapshot.json index e2b132e..7a2a2ab 100644 --- a/packages/server/drizzle/meta/0000_snapshot.json +++ b/packages/server/drizzle/meta/0000_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "00e8c035-e6d3-45a4-8822-fdc5b49c2650", + "id": "77ee0bb4-7f78-4dae-8821-66450c4fbb6a", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { "account": { @@ -1868,6 +1868,194 @@ "uniqueConstraints": {}, "checkConstraints": {} }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, "schema_labels": { "name": "schema_labels", "columns": { @@ -2546,6 +2734,14 @@ "notNull": false, "autoincrement": false }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, "changes": { "name": "changes", "type": "text", diff --git a/packages/server/drizzle/meta/0001_snapshot.json b/packages/server/drizzle/meta/0001_snapshot.json index 092732c..14f5543 100644 --- a/packages/server/drizzle/meta/0001_snapshot.json +++ b/packages/server/drizzle/meta/0001_snapshot.json @@ -1,6 +1,6 @@ { - "id": "f0b0acb6-1748-4f0a-8c16-77349de423dc", - "prevId": "00e8c035-e6d3-45a4-8822-fdc5b49c2650", + "id": "088590c4-e66d-4011-891b-84b0aea7034c", + "prevId": "77ee0bb4-7f78-4dae-8821-66450c4fbb6a", "version": "6", "dialect": "sqlite", "tables": { @@ -1868,6 +1868,194 @@ "uniqueConstraints": {}, "checkConstraints": {} }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, "schema_labels": { "name": "schema_labels", "columns": { @@ -2546,6 +2734,14 @@ "notNull": false, "autoincrement": false }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, "changes": { "name": "changes", "type": "text", diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index c85d2e2..79d5259 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -5,14 +5,14 @@ { "idx": 0, "version": "6", - "when": 1791062066301, + "when": 1791062646832, "tag": "0000_init", "breakpoints": true }, { "idx": 1, "version": "6", - "when": 1791062067031, + "when": 1791062647591, "tag": "0001_platform_location", "breakpoints": true } diff --git a/packages/server/src/api/records.ts b/packages/server/src/api/records.ts new file mode 100644 index 0000000..5d12d70 --- /dev/null +++ b/packages/server/src/api/records.ts @@ -0,0 +1,178 @@ +/** + * Records and files by hash, through the reference log (v1 shapes). + * + * GET /api/records/:hash/provenance + * POST /api/records/batch {hashes} → NDJSON {id,type,data,hash} + * GET /api/collections/files/:hash 302, from any collection the caller may read + * + * Results are filtered by the caller's access before anything is returned, + * counts included (edge-redesign.md, Security notes): a presence counts only if + * it is in a public collection's public set, or in a collection of the caller's + * orgs. Format 1 record hashes resolve through legacy_hashes. + */ +import { and, asc, eq, gte, inArray, lt } from 'drizzle-orm' +import { type Context, Hono } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import { presignDownload } from '../files/files.js' +import { type Presence, presenceOf } from '../refs/log.js' +import { getRecord, loadView } from '../versions/view.js' +import { jsonError } from './access.js' + +async function memberOrgs(c: Context): Promise> { + const p = c.var.principal + if (!p || p.collectionIds) return new Set() + if (p.orgId) return new Set([p.orgId]) + const rows = await c.var.ports.db + .select({ id: schema.member.organizationId }) + .from(schema.member) + .where(eq(schema.member.userId, p.userId)) + return new Set(rows.map((r) => r.id)) +} + +/** Presences the caller may see, with their collections. */ +async function visiblePresence(c: Context, hash: string) { + const { db } = c.var.ports + const [alias] = await db + .select() + .from(schema.legacyHashes) + .where(eq(schema.legacyHashes.legacyHash, hash)) + const target = alias?.hash ?? hash + const presence = await presenceOf(c.var.ports, target) + if (presence.length === 0) + return { + target, + items: [] as { + p: Presence + c: typeof schema.collections.$inferSelect + owner: typeof schema.organization.$inferSelect + member: boolean + }[], + } + const ids = [...new Set(presence.map((p) => p.collectionId))] + const cols = await db + .select({ c: schema.collections, owner: schema.organization }) + .from(schema.collections) + .innerJoin(schema.organization, eq(schema.organization.id, schema.collections.organizationId)) + .where(inArray(schema.collections.id, ids)) + const orgs = await memberOrgs(c) + const items = presence.flatMap((p) => { + const col = cols.find((x) => x.c.id === p.collectionId) + if (!col) return [] + const member = orgs.has(col.c.organizationId) + const visible = member || (col.c.public && p.set === 'public') + return visible ? [{ p, c: col.c, owner: col.owner, member }] : [] + }) + return { target, items } +} + +/** The record body at a version where it was present. */ +async function bodyAt( + c: Context, + item: { p: Presence; c: typeof schema.collections.$inferSelect; member: boolean }, +) { + const seq = item.p.to === null ? null : item.p.to - 1 + const [v] = + seq === null + ? await c.var.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, item.c.headVersionId ?? '')) + : await c.var.ports.db + .select() + .from(schema.versions) + .where(and(eq(schema.versions.collectionId, item.c.id), eq(schema.versions.seq, seq))) + if (!v) return null + const repo = await c.var.ports.stores.forCollection(item.c.id) + const view = await loadView(repo, v, item.member) + const t = view.types.find((x) => x.slug === item.p.type) + return t ? getRecord(view, t, item.p.id) : null +} + +export function recordRoutes() { + const app = new Hono() + + app.get('/api/records/:hash/provenance', async (c) => { + const hash = c.req.param('hash').replace(/^sha256:/, '') + const { target, items } = await visiblePresence(c, hash) + const records = items.filter((i) => i.p.kind === 'r') + if (records.length === 0) return jsonError(c, 404, 'Record not found') + const rec = await bodyAt(c, records[0]!) + if (!rec) return jsonError(c, 404, 'Record not found') + const parsed = JSON.parse(rec.body!) as { id: string; type: string; data: unknown } + const { db } = c.var.ports + const references: { + owner: string + collection: string + collectionName: string + semver: string + versionCreatedAt: Date + }[] = [] + for (const i of records) { + const versions = await db + .select({ semver: schema.versions.semver, createdAt: schema.versions.createdAt }) + .from(schema.versions) + .where( + and( + eq(schema.versions.collectionId, i.c.id), + gte(schema.versions.seq, i.p.from), + i.p.to === null ? undefined : lt(schema.versions.seq, i.p.to), + ), + ) + .orderBy(asc(schema.versions.seq)) + .limit(100) + for (const v of versions) { + references.push({ + owner: i.owner.slug, + collection: i.c.slug, + collectionName: i.c.name, + semver: v.semver, + versionCreatedAt: v.createdAt, + }) + } + } + references.sort((a, b) => a.versionCreatedAt.getTime() - b.versionCreatedAt.getTime()) + return c.json({ + hash, + recordHash: target, + recordId: parsed.id, + type: parsed.type, + data: parsed.data, + size: rec.size, + firstSeen: references[0]?.versionCreatedAt ?? null, + createdAt: references[0]?.versionCreatedAt ?? null, + references, + }) + }) + + app.post('/api/records/batch', async (c) => { + const body = (await c.req.json().catch(() => null)) as { hashes?: unknown } | null + const hashes = Array.isArray(body?.hashes) + ? body.hashes.filter((h): h is string => typeof h === 'string') + : null + if (!hashes || hashes.length === 0 || hashes.length > 10_000) + return jsonError(c, 400, '"hashes" must be 1–10,000 record hashes') + const lines: string[] = [] + for (const h of hashes) { + const { items } = await visiblePresence(c, h.replace(/^sha256:/, '')) + const first = items.find((i) => i.p.kind === 'r') + if (!first) continue + const rec = await bodyAt(c, first) + if (rec) lines.push(`${rec.body!.slice(0, -1)},"hash":"${rec.hash}"}`) + } + return new Response(lines.length ? lines.join('\n') + '\n' : '', { + headers: { 'content-type': 'application/x-ndjson' }, + }) + }) + + app.get('/api/collections/files/:hash', async (c) => { + const hash = c.req.param('hash').replace(/^sha256:/, '') + const { items } = await visiblePresence(c, hash) + if (!items.some((i) => i.p.kind === 'f')) return jsonError(c, 404, 'File not found') + const url = await presignDownload(c.var.ports, hash) + return url ? c.redirect(url, 302) : jsonError(c, 404, 'File not found') + }) + + return app +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index fd411a9..38aa1a2 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -10,6 +10,7 @@ import { collectionRoutes } from './api/collections.js' import { fileRoutes } from './api/files.js' import { manageRoutes } from './api/manage.js' import { pushRoutes } from './api/push.js' +import { recordRoutes } from './api/records.js' import { schemaRoutes } from './api/schemas.js' import { versionRoutes } from './api/versions.js' import { webhookRoutes } from './api/webhooks.js' @@ -86,6 +87,8 @@ export function createApp(setup: Setup) { }), ) + // Before the :owner/:slug routes: /api/collections/files/:hash would match them. + app.route('/', recordRoutes()) app.route('/api/collections', fileRoutes()) app.route('/api/collections', pushRoutes()) app.route('/api/collections', versionRoutes()) diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 7d85d43..65c997f 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -267,6 +267,8 @@ export const versions = sqliteTable( */ publicRefsRoot: text('public_refs_root'), privateRefsRoot: text('private_refs_root'), + /** The reference log has this version's events (written by the refs.index job). */ + refsIndexed: bool('refs_indexed').notNull().default(false), /** Change counts against the previous version (drive semver and webhooks). */ changes: json<{ added: number; removed: number; updated: number }>('changes'), createdAt: createdAt(), @@ -513,6 +515,61 @@ export const pushRuns = sqliteTable( (t) => [primaryKey({ columns: [t.sessionId, t.seq] })], ) +// --- Reference log (provenance; edge-redesign.md "Provenance: the reference log") -------- + +/** + * The manifest of the reference log: one row per immutable segment in the + * platform's internal area. A segment holds events sorted by hash; segments of + * one run cover disjoint hash ranges. Size-tiered compaction merges runs of a + * tier into one run of the next, so a query reads O(log n) runs. + */ +export const refSegments = sqliteTable( + 'ref_segments', + { + id: id(), + runId: text('run_id').notNull(), + tier: integer('tier').notNull(), + firstHash: text('first_hash').notNull(), + lastHash: text('last_hash').notNull(), + count: integer('count').notNull(), + bytes: integer('bytes').notNull(), + /** pending: written by an unfinished compaction; live: queried; retired: replaced. */ + state: text('state', { enum: ['pending', 'live', 'retired'] }) + .notNull() + .default('live'), + createdAt: createdAt(), + }, + (t) => [ + index('ref_segments_range_idx').on(t.state, t.firstHash, t.lastHash), + index('ref_segments_run_idx').on(t.runId), + index('ref_segments_tier_idx').on(t.tier), + ], +) + +/** A compaction merging runs of one tier, split into hash-range parts run as jobs. */ +export const refCompactions = sqliteTable('ref_compactions', { + id: id(), + tier: integer('tier').notNull(), + inputRuns: json('input_runs').notNull(), + outputRun: text('output_run').notNull(), + parts: integer('parts').notNull(), + /** Hex prefix length of each part's hash range. */ + prefixLength: integer('prefix_length').notNull(), + status: text('status', { enum: ['running', 'done'] }) + .notNull() + .default('running'), + createdAt: createdAt(), +}) + +export const refCompactionParts = sqliteTable( + 'ref_compaction_parts', + { + compactionId: text('compaction_id').notNull(), + part: integer('part').notNull(), + }, + (t) => [primaryKey({ columns: [t.compactionId, t.part] })], +) + // --- Jobs (Node only; Cloudflare uses Queues) ---------------------------------------- export const jobs = sqliteTable( diff --git a/packages/server/src/handlers.ts b/packages/server/src/handlers.ts index e320df4..35cda0e 100644 --- a/packages/server/src/handlers.ts +++ b/packages/server/src/handlers.ts @@ -11,6 +11,7 @@ import { and, asc, eq, gt } from 'drizzle-orm' import * as schema from './db/schema.js' import './files/files.js' +import './refs/log.js' import { registerJob } from './jobs.js' import { expireSessions } from './push/finalize.js' import { appendVersionLog } from './versions/commit.js' @@ -19,6 +20,7 @@ import { enqueueDeliveries, purgeOldDeliveries } from './webhooks/webhooks.js' registerJob('version.published', async (job, ports) => { await enqueueDeliveries(ports, String(job.versionId), job.bump as BumpType) + await ports.jobs.enqueue({ type: 'refs.index', versionId: String(job.versionId) }) // Mirror sync (phase 11) and reference-log segments (phase 7) hang off here too. }) diff --git a/packages/server/src/refs/log.ts b/packages/server/src/refs/log.ts new file mode 100644 index 0000000..6484a7a --- /dev/null +++ b/packages/server/src/refs/log.ts @@ -0,0 +1,437 @@ +/** + * The reference log: which collections and versions contain a record or file + * hash, in which set (edge-redesign.md, "Provenance: the reference log"). + * + * Index changes, not snapshots. A version's events are the diff between it and + * the previous version (+ added, - removed, both for an update or a set move), + * so writing costs O(changes). They're produced by a job after publish — the + * plan accepts seconds of lag — which is also exactly the rebuild procedure. + * A fork's first version writes no events; queries extend a parent's intervals + * into its forks. + * + * Storage is a size-tiered LSM of segments (segments.ts): each version adds a + * tier-0 run; when a tier has FAN_IN runs they merge into one run of the next + * tier, as hash-range parts run as jobs. Queries read every live run whose + * segments cover the hash: O(log n) runs, each a cached index check and usually + * no block read. + */ +import { compareUtf8, diffTrees, fileTree, recordTree } from '@underlay/core' +import { RepoSource } from '@underlay/repo' +import { and, asc, eq, gte, inArray, lte, sql } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import { registerJob } from '../jobs.js' +import type { Ports } from '../ports.js' +import { mergeRuns, type RunIndex, writeRun } from '../push/runs.js' +import { loadView } from '../versions/view.js' +import { + compareEvents, + deleteSegment, + eventBytes, + lookupSegment, + readSegment, + type RefEvent, + SegmentWriter, + type WrittenSegment, +} from './segments.js' + +export const FAN_IN = 8 +const SPILL_EVENTS = 100_000 +/** Events per compaction part, roughly: sizes the hash-range split. */ +const PART_EVENTS = 4_000_000 + +// --- Generating a version's events --------------------------------------------------- + +async function* versionEvents( + ports: Ports, + version: typeof schema.versions.$inferSelect, +): AsyncGenerator { + const repo = await ports.stores.forCollection(version.collectionId) + const [prev] = + version.seq > 1 + ? await ports.db + .select() + .from(schema.versions) + .where( + and( + eq(schema.versions.collectionId, version.collectionId), + eq(schema.versions.seq, version.seq - 1), + ), + ) + : [] + const now = await loadView(repo, version, true) + const before = prev ? await loadView(repo, prev, true) : null + const records = new RepoSource(recordTree, repo) + const files = new RepoSource(fileTree, repo) + const c = version.collectionId + const seq = version.seq + for (const set of ['public', 'private'] as const) { + const slugs = new Set([ + ...now.types.map((t) => t.slug), + ...(before?.types.map((t) => t.slug) ?? []), + ]) + for (const slug of [...slugs].sort(compareUtf8)) { + const a = before?.types.find((t) => t.slug === slug)?.[set]?.root ?? null + const b = now.types.find((t) => t.slug === slug)?.[set]?.root ?? null + for await (const d of diffTrees(records, a, b)) { + if (d.before) yield [d.before.hash, 'r', c, set, seq, '-', slug, d.key] + if (d.after) yield [d.after.hash, 'r', c, set, seq, '+', slug, d.key] + } + } + const fa = (set === 'public' ? before?.public : before?.private)?.files.root ?? null + const fb = (set === 'public' ? now.public : now.private)?.files.root ?? null + for await (const d of diffTrees(files, fa, fb)) { + if (d.before) yield [d.key, 'f', c, set, seq, '-', '', ''] + if (d.after) yield [d.key, 'f', c, set, seq, '+', '', ''] + } + } +} + +/** Sort events by hash, spilling to sorted runs past SPILL_EVENTS (bounded memory). */ +async function* sortedEvents( + ports: Ports, + scratch: string, + events: AsyncIterable, +): AsyncGenerator { + let buf: RefEvent[] = [] + const runs: RunIndex[] = [] + const spill = async () => { + const entries = buf.map((e) => ({ + t: e[0], + k: JSON.stringify(e.slice(1)), + b: JSON.stringify(e), + })) + runs.push(await writeRun(ports.stores.internal, scratch, runs.length + 1, entries)) + buf = [] + } + for await (const e of events) { + buf.push(e) + if (buf.length >= SPILL_EVENTS) await spill() + } + if (runs.length === 0) { + yield* buf.sort(compareEvents) + return + } + if (buf.length) await spill() + // Run keys are unique per event, so the merge keeps every event. + for await (const r of mergeRuns(ports.stores.internal, scratch, runs)) + yield JSON.parse(r.b!) as RefEvent +} + +const segmentRows = ( + segs: WrittenSegment[], + runId: string, + tier: number, + state: 'live' | 'pending', +) => + segs.map((s) => ({ + id: s.id, + runId, + tier, + firstHash: s.firstHash, + lastHash: s.lastHash, + count: s.count, + bytes: s.bytes, + state, + })) + +/** + * Index one version: write its events as a tier-0 run, then record the run, the + * flag and the collection's billing counters in one batch. Segment ids are + * derived from the version, so a retried job rewrites the same objects, and the + * batch only takes effect while the version is still unindexed. + */ +export async function indexVersion(ports: Ports, versionId: string): Promise { + const { db } = ports + const [version] = await db.select().from(schema.versions).where(eq(schema.versions.id, versionId)) + if (!version || version.refsIndexed) return + const [fork] = + version.seq === 1 + ? await db + .select() + .from(schema.forks) + .where(eq(schema.forks.childCollectionId, version.collectionId)) + : [] + const runId = `v-${versionId}` + const writer = new SegmentWriter(ports.stores.internal, (n) => `${runId}-${n}`) + let events = 0 + let bytes = 0 + if (!fork) { + for await (const e of sortedEvents(ports, `refs-${versionId}`, versionEvents(ports, version))) { + await writer.add(e) + events++ + bytes += eventBytes(e) + } + } + const segs = await writer.finish() + const unindexed = sql`(SELECT ${schema.versions.refsIndexed} FROM ${schema.versions} WHERE ${schema.versions.id} = ${versionId}) = 0` + const lit = (v: unknown) => sql`${v}` + await db.batch([ + ...segmentRows(segs, runId, 0, 'live').map((r) => + db.insert(schema.refSegments).select( + db + .select({ + id: lit(r.id).as('id'), + runId: lit(r.runId).as('run_id'), + tier: lit(0).as('tier'), + firstHash: lit(r.firstHash).as('first_hash'), + lastHash: lit(r.lastHash).as('last_hash'), + count: lit(r.count).as('count'), + bytes: lit(r.bytes).as('bytes'), + state: lit('live').as('state'), + createdAt: lit(Date.now()).as('created_at'), + }) + .from(schema.versions) + .where(and(eq(schema.versions.id, versionId), unindexed)) as never, + ), + ), + db + .update(schema.collections) + .set({ + refEvents: sql`${schema.collections.refEvents} + ${events}`, + refBytes: sql`${schema.collections.refBytes} + ${bytes}`, + }) + .where(and(eq(schema.collections.id, version.collectionId), unindexed)), + db.update(schema.versions).set({ refsIndexed: true }).where(eq(schema.versions.id, versionId)), + ] as unknown as Parameters[0]) + await ports.jobs.enqueue({ type: 'refs.compact' }) +} + +// --- Compaction --------------------------------------------------------------------- + +/** If some tier has FAN_IN live runs, start merging its oldest FAN_IN into one run of the next tier. */ +export async function planCompaction(ports: Ports): Promise { + const { db } = ports + const running = await db + .select() + .from(schema.refCompactions) + .where(eq(schema.refCompactions.status, 'running')) + .limit(1) + if (running.length) return + const runs = (await db.all(sql` + SELECT run_id, tier, sum(count) AS events, min(created_at) AS created + FROM ${schema.refSegments} WHERE state = 'live' + GROUP BY run_id, tier ORDER BY tier, created + `)) as { run_id: string; tier: number; events: number; created: number }[] + const byTier = new Map() + for (const r of runs) byTier.set(r.tier, [...(byTier.get(r.tier) ?? []), r]) + for (const [tier, list] of [...byTier].sort((a, b) => a[0] - b[0])) { + if (list.length < FAN_IN) continue + const inputs = list.slice(0, FAN_IN) + const total = inputs.reduce((n, r) => n + r.events, 0) + // Hash-range parts by hex prefix: 16^L parts of roughly PART_EVENTS each. + let prefixLength = 0 + while (16 ** prefixLength * PART_EVENTS < total && prefixLength < 4) prefixLength++ + const [comp] = await db + .insert(schema.refCompactions) + .values({ + tier, + inputRuns: inputs.map((r) => r.run_id), + outputRun: `c-${crypto.randomUUID()}`, + parts: 16 ** prefixLength, + prefixLength, + }) + .returning() + await ports.jobs.enqueueBatch( + Array.from({ length: comp!.parts }, (_, part) => ({ + type: 'refs.compactPart', + compactionId: comp!.id, + part, + })), + ) + return + } +} + +const partRange = (part: number, prefixLength: number) => { + if (prefixLength === 0) return { from: '', to: 'g' } + const from = part.toString(16).padStart(prefixLength, '0') + const to = + part + 1 === 16 ** prefixLength ? 'g' : (part + 1).toString(16).padStart(prefixLength, '0') + return { from, to } +} + +/** Merge one hash range of a compaction's input runs into pending segments of its output run. */ +export async function compactPart(ports: Ports, compactionId: string, part: number): Promise { + const { db } = ports + const [comp] = await db + .select() + .from(schema.refCompactions) + .where(eq(schema.refCompactions.id, compactionId)) + if (!comp || comp.status !== 'running') return + const range = partRange(part, comp.prefixLength) + const inputs = await db + .select() + .from(schema.refSegments) + .where( + and( + inArray(schema.refSegments.runId, comp.inputRuns), + lte(schema.refSegments.firstHash, range.to), + gte(schema.refSegments.lastHash, range.from), + ), + ) + .orderBy(asc(schema.refSegments.firstHash)) + // Each input run, in hash order, as one stream; then a k-way merge by event order. + const streams = comp.inputRuns.map((run) => + (async function* () { + for (const seg of inputs.filter((s) => s.runId === run)) + yield* readSegment(ports.stores.internal, seg.id, range) + })(), + ) + const heads = await Promise.all(streams.map(async (s) => ({ s, h: await s.next() }))) + const writer = new SegmentWriter(ports.stores.internal, (n) => `${comp.outputRun}-${part}-${n}`) + for (;;) { + let best: (typeof heads)[number] | null = null + for (const x of heads) + if (!x.h.done && (!best || compareEvents(x.h.value, best.h.value as RefEvent) < 0)) best = x + if (!best) break + await writer.add(best.h.value as RefEvent) + best.h = await best.s.next() + } + const segs = await writer.finish() + await db.batch([ + ...segmentRows(segs, comp.outputRun, comp.tier + 1, 'pending').map((r) => + db.insert(schema.refSegments).values(r).onConflictDoNothing(), + ), + db.insert(schema.refCompactionParts).values({ compactionId, part }).onConflictDoNothing(), + ] as unknown as Parameters[0]) + const [{ done } = { done: 0 }] = (await db.all( + sql`SELECT count(*) AS done FROM ${schema.refCompactionParts} WHERE compaction_id = ${compactionId}`, + )) as { done: number }[] + if (done === comp.parts) await ports.jobs.enqueue({ type: 'refs.finishCompaction', compactionId }) +} + +/** Swap a finished compaction in atomically, then delete the inputs' objects. */ +export async function finishCompaction(ports: Ports, compactionId: string): Promise { + const { db } = ports + const [comp] = await db + .select() + .from(schema.refCompactions) + .where(eq(schema.refCompactions.id, compactionId)) + if (!comp || comp.status !== 'running') return + const old = await db + .select({ id: schema.refSegments.id }) + .from(schema.refSegments) + .where(inArray(schema.refSegments.runId, comp.inputRuns)) + await db.batch([ + db + .update(schema.refSegments) + .set({ state: 'live' }) + .where(eq(schema.refSegments.runId, comp.outputRun)), + db + .update(schema.refSegments) + .set({ state: 'retired' }) + .where(inArray(schema.refSegments.runId, comp.inputRuns)), + db + .update(schema.refCompactions) + .set({ status: 'done' }) + .where(eq(schema.refCompactions.id, compactionId)), + ]) + for (const s of old) await deleteSegment(ports.stores.internal, s.id) + await db.delete(schema.refSegments).where(inArray(schema.refSegments.runId, comp.inputRuns)) + await ports.jobs.enqueue({ type: 'refs.compact' }) +} + +registerJob('refs.index', async (job, ports) => indexVersion(ports, String(job.versionId))) +registerJob('refs.compact', async (_job, ports) => planCompaction(ports)) +registerJob('refs.compactPart', async (job, ports) => + compactPart(ports, String(job.compactionId), Number(job.part)), +) +registerJob('refs.finishCompaction', async (job, ports) => + finishCompaction(ports, String(job.compactionId)), +) + +// --- Queries ------------------------------------------------------------------------ + +/** Every event for a hash across live segments. */ +export async function eventsFor(ports: Ports, hash: string): Promise { + const segs = await ports.db + .select({ id: schema.refSegments.id }) + .from(schema.refSegments) + .where( + and( + eq(schema.refSegments.state, 'live'), + lte(schema.refSegments.firstHash, hash), + gte(schema.refSegments.lastHash, hash), + ), + ) + const found = await Promise.all( + segs.map((s) => lookupSegment(ports.stores.internal, ports.cache, s.id, hash)), + ) + return found.flat().sort(compareEvents) +} + +export interface Presence { + collectionId: string + set: 'public' | 'private' + kind: 'r' | 'f' + type: string + id: string + /** First seq containing it, and the first seq that no longer does (null: still present). */ + from: number + to: number | null +} + +/** + * Fold events into presence intervals per (collection, set, type, id), and + * extend them into forks: a fork's first version contains whatever its parent's + * version at the fork point did, until the fork removes it. + */ +export async function presenceOf(ports: Ports, hash: string): Promise { + const events = await eventsFor(ports, hash) + const open = new Map() + const out: Presence[] = [] + // Removals with no addition in the same collection: a fork dropping something it inherited. + const orphanRemovals = new Map() + for (const e of events) { + const [, kind, collectionId, set, seq, op, type, id] = e + const key = `${collectionId}\u0000${set}\u0000${type}\u0000${id}` + if (op === '+') { + if (!open.has(key)) open.set(key, { collectionId, set, kind, type, id, from: seq, to: null }) + } else { + const p = open.get(key) + if (p) { + p.to = seq + out.push(p) + open.delete(key) + } else if (!orphanRemovals.has(key)) { + orphanRemovals.set(key, seq) + } + } + } + out.push(...open.values()) + // Forks: children whose parent contained it at the fork point. + const queue = [...out] + while (queue.length) { + const p = queue.shift()! + const children = await ports.db + .select() + .from(schema.forks) + .where(eq(schema.forks.parentCollectionId, p.collectionId)) + for (const f of children) { + if (f.parentSeq < p.from || (p.to !== null && f.parentSeq >= p.to)) continue + if ( + out.some( + (q) => + q.collectionId === f.childCollectionId && + q.type === p.type && + q.id === p.id && + q.set === p.set, + ) + ) + continue + const removed = orphanRemovals.get( + `${f.childCollectionId}\u0000${p.set}\u0000${p.type}\u0000${p.id}`, + ) + const child: Presence = { + ...p, + collectionId: f.childCollectionId, + from: 1, + to: removed ?? null, + } + out.push(child) + queue.push(child) + } + } + return out +} diff --git a/packages/server/src/refs/segments.ts b/packages/server/src/refs/segments.ts new file mode 100644 index 0000000..387367e --- /dev/null +++ b/packages/server/src/refs/segments.ts @@ -0,0 +1,240 @@ +/** + * Reference-log segments: immutable, sorted by hash, in the platform's internal + * area (never mirrored). + * + * refs/seg/.dat gzip blocks of NDJSON events, concatenated (~1,024 events each) + * refs/seg/.idx {count, bytes, blocks: [[first, last, offset, length]], m, k, bloom} + * + * The index is small (about 1.3 MB of filter at the 256k-event cap) and + * immutable, so it's cached; a lookup that passes the Bloom filter reads one + * block by range. + */ +import { gunzipText, gzip, splitLines } from '@underlay/repo' + +import type { BlobStore, Cache } from '../ports.js' + +/** [hash, kind, collectionId, set, seq, op, type, id]. kind: r(ecord) | f(ile). op: + | -. */ +export type RefEvent = [ + string, + 'r' | 'f', + string, + 'public' | 'private', + number, + '+' | '-', + string, + string, +] + +export const SEGMENT_MAX_EVENTS = 262_144 +const BLOCK_EVENTS = 1024 +const BLOOM_BITS_PER_KEY = 10 +const BLOOM_K = 7 + +const enc = new TextEncoder() +/** Billable bytes: the event's canonical encoding (a pure function of the event). */ +export const eventBytes = (e: RefEvent) => enc.encode(JSON.stringify(e)).byteLength + +export const compareEvents = (a: RefEvent, b: RefEvent) => + a[0] < b[0] + ? -1 + : a[0] > b[0] + ? 1 + : a[2] < b[2] + ? -1 + : a[2] > b[2] + ? 1 + : a[4] - b[4] || (a[5] < b[5] ? -1 : a[5] > b[5] ? 1 : 0) + +export interface SegmentIndex { + count: number + bytes: number + blocks: [string, string, number, number][] + m: number + k: number + bloom: string +} + +const keyOf = (id: string, ext: 'dat' | 'idx') => `refs/seg/${id}.${ext}` + +function bloomPositions(hash: string, m: number, k: number): number[] { + const h1 = parseInt(hash.slice(0, 8), 16) + // `| 1` alone would make values ≥ 2^31 negative (int32); keep them unsigned. + const h2 = (parseInt(hash.slice(8, 16), 16) | 1) >>> 0 + return Array.from({ length: k }, (_, i) => (h1 + i * h2) % m) +} + +export function bloomHas(idx: SegmentIndex, bits: Uint8Array, hash: string): boolean { + return bloomPositions(hash, idx.m, idx.k).every((p) => (bits[p >> 3]! & (1 << (p & 7))) !== 0) +} + +export interface WrittenSegment { + id: string + firstHash: string + lastHash: string + count: number + bytes: number +} + +/** Writes sorted events as segments of at most SEGMENT_MAX_EVENTS. */ +export class SegmentWriter { + readonly segments: WrittenSegment[] = [] + #blocks: Uint8Array[] = [] + #blockIndex: [string, string, number, number][] = [] + #offset = 0 + #lines: string[] = [] + #first: string | null = null + #last: string | null = null + #blockFirst: string | null = null + #count = 0 + #bytes = 0 + #prefixes: number[] = [] + #n = 0 + + constructor( + readonly store: BlobStore, + readonly idFor: (n: number) => string, + ) {} + + async add(e: RefEvent): Promise { + if (this.#last !== null && e[0] < this.#last) throw new Error('Reference events out of order') + const line = JSON.stringify(e) + this.#first ??= e[0] + this.#blockFirst ??= e[0] + this.#last = e[0] + this.#lines.push(line) + this.#count++ + this.#bytes += enc.encode(line).byteLength + this.#prefixes.push(parseInt(e[0].slice(0, 8), 16), parseInt(e[0].slice(8, 16), 16)) + if (this.#lines.length >= BLOCK_EVENTS) await this.#flushBlock() + if (this.#count >= SEGMENT_MAX_EVENTS) await this.#finishSegment() + } + + async #flushBlock() { + if (this.#lines.length === 0) return + const gz = await gzip(this.#lines.join('\n') + '\n') + this.#blockIndex.push([this.#blockFirst!, this.#last!, this.#offset, gz.byteLength]) + this.#blocks.push(gz) + this.#offset += gz.byteLength + this.#lines = [] + this.#blockFirst = null + } + + async #finishSegment() { + await this.#flushBlock() + if (this.#count === 0) return + const id = this.idFor(this.#n++) + const m = Math.max(64, this.#count * BLOOM_BITS_PER_KEY) + const bits = new Uint8Array(Math.ceil(m / 8)) + for (let i = 0; i < this.#prefixes.length; i += 2) { + const h1 = this.#prefixes[i]! + const h2 = (this.#prefixes[i + 1]! | 1) >>> 0 + for (let j = 0; j < BLOOM_K; j++) { + const p = (h1 + j * h2) % m + bits[p >> 3]! |= 1 << (p & 7) + } + } + const dat = new Uint8Array(this.#offset) + let off = 0 + for (const b of this.#blocks) { + dat.set(b, off) + off += b.byteLength + } + const idx: SegmentIndex = { + count: this.#count, + bytes: this.#bytes, + blocks: this.#blockIndex, + m, + k: BLOOM_K, + bloom: Buffer.from(bits).toString('base64'), + } + await this.store.put(keyOf(id, 'dat'), dat, { contentType: 'application/gzip' }) + await this.store.put(keyOf(id, 'idx'), JSON.stringify(idx), { contentType: 'application/json' }) + this.segments.push({ + id, + firstHash: this.#first!, + lastHash: this.#last!, + count: this.#count, + bytes: this.#bytes, + }) + this.#blocks = [] + this.#blockIndex = [] + this.#offset = 0 + this.#first = null + this.#count = 0 + this.#bytes = 0 + this.#prefixes = [] + } + + async finish(): Promise { + await this.#finishSegment() + return this.segments + } +} + +const idxCache = new Map() + +async function loadIndex(store: BlobStore, cache: Cache, id: string) { + const hit = idxCache.get(id) + if (hit) return hit + const key = `refidx/${id}` + let bytes = await cache.get(key) + if (!bytes) { + const obj = await store.get(keyOf(id, 'idx')) + if (!obj) throw new Error(`Missing reference segment index ${id}`) + bytes = await obj.bytes() + await cache.put(key, bytes) + } + const idx = JSON.parse(new TextDecoder().decode(bytes)) as SegmentIndex + const entry = { idx, bits: new Uint8Array(Buffer.from(idx.bloom, 'base64')) } + if (idxCache.size > 256) idxCache.delete(idxCache.keys().next().value!) + idxCache.set(id, entry) + return entry +} + +/** Events for one hash in one segment. */ +export async function lookupSegment( + store: BlobStore, + cache: Cache, + id: string, + hash: string, +): Promise { + const { idx, bits } = await loadIndex(store, cache, id) + if (!bloomHas(idx, bits, hash)) return [] + const out: RefEvent[] = [] + for (const [first, last, offset, length] of idx.blocks) { + if (hash < first || hash > last) continue + const obj = await store.get(keyOf(id, 'dat'), { offset, length }) + if (!obj) throw new Error(`Missing reference segment ${id}`) + for (const line of splitLines(await gunzipText(await obj.bytes()))) { + const e = JSON.parse(line) as RefEvent + if (e[0] === hash) out.push(e) + } + } + return out +} + +/** All events of a segment in order, a block at a time (for compaction). */ +export async function* readSegment( + store: BlobStore, + id: string, + range?: { from: string; to: string }, +): AsyncGenerator { + const obj = await store.get(keyOf(id, 'idx')) + if (!obj) throw new Error(`Missing reference segment index ${id}`) + const idx = JSON.parse(await obj.text()) as SegmentIndex + for (const [first, last, offset, length] of idx.blocks) { + if (range && (last < range.from || first >= range.to)) continue + const block = await store.get(keyOf(id, 'dat'), { offset, length }) + if (!block) throw new Error(`Missing reference segment ${id}`) + for (const line of splitLines(await gunzipText(await block.bytes()))) { + const e = JSON.parse(line) as RefEvent + if (range && (e[0] < range.from || e[0] >= range.to)) continue + yield e + } + } +} + +export async function deleteSegment(store: BlobStore, id: string): Promise { + await store.delete(keyOf(id, 'dat')) + await store.delete(keyOf(id, 'idx')) +} diff --git a/packages/server/src/versions/publish.ts b/packages/server/src/versions/publish.ts index f5037ad..019d03e 100644 --- a/packages/server/src/versions/publish.ts +++ b/packages/server/src/versions/publish.ts @@ -117,6 +117,7 @@ export async function publishVersion( hasPrivate: lit(v.hasPrivate ? 1 : 0).as('has_private'), publicRefsRoot: lit(v.publicRefsRoot).as('public_refs_root'), privateRefsRoot: lit(v.privateRefsRoot).as('private_refs_root'), + refsIndexed: lit(0).as('refs_indexed'), changes: lit(JSON.stringify(v.changes)).as('changes'), createdAt: lit(now).as('created_at'), }) diff --git a/packages/server/test/refs.test.ts b/packages/server/test/refs.test.ts new file mode 100644 index 0000000..3f0d70d --- /dev/null +++ b/packages/server/test/refs.test.ts @@ -0,0 +1,159 @@ +import { createHash } from 'node:crypto' + +import { hashRecord } from '@underlay/core' +import { noCache } from '@underlay/repo' +import { MemoryBlobStore } from '@underlay/repo/blob/memory' +import { eq } from 'drizzle-orm' +import { afterAll, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { eventsFor, FAN_IN } from '../src/refs/log.js' +import { lookupSegment, SegmentWriter } from '../src/refs/segments.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) + +const Author = { type: 'object', properties: { name: { type: 'string' } } } +const h_ = (id: string, data: unknown) => hashRecord(id, 'Author', data).hash + +async function json(res: Response) { + return (await res.json()) as any +} + +async function push( + h: Harness, + user: string, + base: string, + body: object, + records: object[], + deletes: object[] = [], +) { + const sid = (await json(await h.request(`${base}/push`, { method: 'POST', user, json: body }))) + .session_id + if (records.length) + await h.request(`${base}/push/${sid}/records`, { method: 'POST', user, ndjson: records }) + if (deletes.length) + await h.request(`${base}/push/${sid}/deletes`, { method: 'POST', user, ndjson: deletes }) + const res = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) + expect(res.status).toBe(201) + await h.drain() + return json(res) +} + +describe('reference log', () => { + it('answers provenance across versions, sets and forks, filtered by access', async () => { + const h = await harness() + const user = await h.member() + await h.collection('lib') + await h.ports.db.update(schema.collections).set({ public: true }) + const base = '/api/collections/org/lib' + await push(h, user, base, { schemas: { Author } }, [ + { id: 'a', type: 'Author', data: { name: 'A' } }, + { id: 'b', type: 'Author', data: { name: 'B' }, private: true }, + ]) + await push(h, user, base, { base: 'v1.0.0' }, [ + { id: 'a', type: 'Author', data: { name: 'A2' } }, + ]) + + const oldA = await json(await h.request(`/api/records/${h_('a', { name: 'A' })}/provenance`)) + expect(oldA).toMatchObject({ recordId: 'a', type: 'Author', data: { name: 'A' } }) + expect(oldA.references.map((r: any) => r.semver)).toEqual(['v1.0.0']) + const newA = await json(await h.request(`/api/records/${h_('a', { name: 'A2' })}/provenance`)) + expect(newA.references.map((r: any) => r.semver)).toEqual(['v1.1.0']) + + // Private records: members only, and nothing (not even a count) for others. + const bHash = h_('b', { name: 'B' }) + expect((await h.request(`/api/records/${bHash}/provenance`)).status).toBe(404) + expect( + (await json(await h.request(`/api/records/${bHash}/provenance`, { user }))).references.length, + ).toBe(2) + + // A fork inherits presence from the fork point, until it removes the record. + await h.ports.db.insert(schema.organization).values({ id: 'org2', name: 'Two', slug: 'two' }) + await h.ports.db + .insert(schema.member) + .values({ organizationId: 'org2', userId: user, role: 'owner' }) + expect( + (await h.request(`${base}/fork`, { method: 'POST', user, json: { targetOrg: 'two' } })) + .status, + ).toBe(201) + await h.drain() + await h.ports.db.update(schema.collections).set({ public: true }) + let refs = (await json(await h.request(`/api/records/${h_('a', { name: 'A2' })}/provenance`))) + .references + expect(refs.map((r: any) => `${r.owner}/${r.semver}`).sort()).toEqual([ + 'org/v1.1.0', + 'two/v1.0.0', + ]) + await push( + h, + user, + '/api/collections/two/lib', + { base: 'v1.0.0' }, + [], + [{ type: 'Author', id: 'a' }], + ) + refs = (await json(await h.request(`/api/records/${h_('a', { name: 'A2' })}/provenance`))) + .references + expect(refs.map((r: any) => `${r.owner}/${r.semver}`).sort()).toEqual([ + 'org/v1.1.0', + 'two/v1.0.0', + ]) + + // Batch fetch by hash, and the counters. + const batch = await h.request('/api/records/batch', { + method: 'POST', + json: { hashes: [h_('a', { name: 'A2' }), bHash] }, + }) + const lines = (await batch.text()) + .trim() + .split('\n') + .map((l) => JSON.parse(l)) + expect(lines).toEqual([ + { id: 'a', type: 'Author', data: { name: 'A2' }, hash: h_('a', { name: 'A2' }) }, + ]) + const [col] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.slug, 'lib')) + expect(col!.refEvents).toBeGreaterThan(0) + }) + + it('compacts runs without changing answers', async () => { + const h = await harness() + const user = await h.member() + await h.collection('lib') + const base = '/api/collections/org/lib' + await push(h, user, base, { schemas: { Author } }, [ + { id: 'k0', type: 'Author', data: { name: '0' } }, + ]) + for (let i = 1; i < FAN_IN + 2; i++) { + await push(h, user, base, {}, [{ id: `k${i}`, type: 'Author', data: { name: String(i) } }]) + } + const runs = await h.ports.db + .select() + .from(schema.refSegments) + .where(eq(schema.refSegments.state, 'live')) + expect(new Set(runs.map((r) => r.runId)).size).toBeLessThan(FAN_IN + 2) + expect(runs.some((r) => r.tier === 1)).toBe(true) + for (let i = 0; i < FAN_IN + 2; i++) { + const ev = await eventsFor(h.ports, h_(`k${i}`, { name: String(i) })) + expect(ev.map((e) => e[5])).toEqual(['+']) + } + }) +}) + +describe('segments', () => { + it('never has false negatives, for any hash', async () => { + const store = new MemoryBlobStore() + const hashes = Array.from({ length: 5000 }, (_, i) => + createHash('sha256').update(String(i)).digest('hex'), + ).sort() + const w = new SegmentWriter(store, () => 'seg') + for (const x of hashes) await w.add([x, 'r', 'c', 'public', 1, '+', 'T', x.slice(0, 6)]) + await w.finish() + for (const x of hashes.filter((_, i) => i % 50 === 0)) { + expect((await lookupSegment(store, noCache, 'seg', x)).length).toBe(1) + } + }) +}) From ec4b2a425092591ee098b3e94e197256f37d4d16 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 17:28:11 -0400 Subject: [PATCH 016/178] v2 export: streaming tar(.gz) sized from the trees, with README.md GET /api/collections/:owner/:slug/export streams manifest.json, README.md (metadata.readme), records/.ndjson and files/ for what the caller may read. Entry sizes come from tree bytes and counts, so nothing is buffered and there's no record cap (v1 refused over 2M). Long type names use PAX headers. format=tar skips gzip for very large exports (gzip costs Worker CPU per byte). --- packages/server/src/api/export.ts | 167 ++++++++++++++++++++++++++++ packages/server/src/app.ts | 2 + packages/server/src/lib/tar.ts | 80 +++++++++++++ packages/server/test/export.test.ts | 80 +++++++++++++ 4 files changed, 329 insertions(+) create mode 100644 packages/server/src/api/export.ts create mode 100644 packages/server/src/lib/tar.ts create mode 100644 packages/server/test/export.test.ts diff --git a/packages/server/src/api/export.ts b/packages/server/src/api/export.ts new file mode 100644 index 0000000..bad596d --- /dev/null +++ b/packages/server/src/api/export.ts @@ -0,0 +1,167 @@ +/** + * GET /api/collections/:owner/:slug/export?version=&format=tar|tar.gz + * + * A tar archive of what the caller may read (v1 layout, plus README.md): + * manifest.json collection, version, schemas, missing files + * README.md the version's metadata.readme, when there is one + * records/.ndjson one {id,type,data,hash} per line + * files/ file bytes + * + * Streams with no record count limit: tar needs sizes up front, and the trees + * give them without reading records (a type's NDJSON is its canonical bytes plus + * a fixed `,"hash":"…"` and newline per record). gzip costs Worker CPU per byte, + * so very large exports should ask for format=tar. + */ +import { fileTree, iterate } from '@underlay/core' +import { RepoSource } from '@underlay/repo' +import { inArray } from 'drizzle-orm' +import { Hono } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import { type TarEntry, tarStream } from '../lib/tar.js' +import { findVersion, loadView, typeRecords } from '../versions/view.js' +import { jsonError, requireCollection } from './access.js' + +const enc = new TextEncoder() +// `{"id":…}` → `{"id":…,"hash":"<64 hex>"}` adds 74 bytes; plus the newline. +const PER_RECORD_EXTRA = 75 + +export function exportRoutes() { + const app = new Hono() + + app.get('/:owner/:slug/export', async (c) => { + const access = await requireCollection(c, 'read') + if (access instanceof Response) return access + const ports = c.var.ports + const v = await findVersion( + ports.db, + access.collection.id, + c.req.query('version') ?? 'latest', + access.collection.headVersionId, + ) + if (!v) return jsonError(c, 404, 'No versions found') + const repo = await ports.stores.forCollection(access.collection.id) + const view = await loadView(repo, v, access.isMember) + const gzip = c.req.query('format') !== 'tar' + + // Files the caller may read, and which of them the platform has bytes for. + const fileSource = new RepoSource(fileTree, repo) + const fileHashes = new Map() + for (const r of [view.public.files.root, view.private?.files.root ?? null]) { + for await (const f of iterate(fileSource, r)) fileHashes.set(f.key, f.size) + } + const rows: (typeof schema.files.$inferSelect)[] = [] + const list = [...fileHashes.keys()] + for (let i = 0; i < list.length; i += 90) { + rows.push( + ...(await ports.db + .select() + .from(schema.files) + .where(inArray(schema.files.hash, list.slice(i, i + 90)))), + ) + } + const stored = new Map(rows.map((r) => [r.hash, r])) + + const schemas: Record = {} + for (const t of view.types) schemas[t.slug] = await repo.schema(t.schemaHash) + const recordCount = view.types.reduce((n, t) => n + t.count, 0) + const manifest = enc.encode( + JSON.stringify( + { + collection: { + owner: access.owner.slug, + slug: access.collection.slug, + name: access.collection.name, + description: access.collection.summary?.description ?? null, + }, + version: { + semver: v.semver, + hash: v.hash, + message: v.message, + recordCount, + fileCount: fileHashes.size, + totalBytes: + view.types.reduce((n, t) => n + t.bytes, 0) + + [...fileHashes.values()].reduce((a, b) => a + b, 0), + createdAt: v.createdAt, + }, + schemas, + files_missing: list.filter((h) => !stored.has(h)), + }, + null, + 2, + ), + ) + const readme = + typeof view.root.metadata?.readme === 'string' ? enc.encode(view.root.metadata.readme) : null + + const entries = async function* (): AsyncGenerator { + yield { + name: 'manifest.json', + size: manifest.byteLength, + body: async function* () { + yield manifest + }, + } + if (readme) + yield { + name: 'README.md', + size: readme.byteLength, + body: async function* () { + yield readme + }, + } + for (const t of view.types) { + yield { + name: `records/${t.slug}.ndjson`, + size: t.bytes + t.count * PER_RECORD_EXTRA, + body: async function* () { + let batch: string[] = [] + for await (const e of typeRecords(view, t, { bodies: true })) { + batch.push(`${e.body!.slice(0, -1)},"hash":"${e.hash}"}\n`) + if (batch.length === 512) { + yield enc.encode(batch.join('')) + batch = [] + } + } + if (batch.length) yield enc.encode(batch.join('')) + }, + } + } + for (const [hash, size] of fileHashes) { + const f = stored.get(hash) + if (!f) continue + yield { + name: `files/${hash}`, + size, + body: async function* () { + const obj = await ports.stores.fileBytes.get(f.storageKey) + if (!obj) throw new Error(`File ${hash} is missing from storage`) + const reader = obj.body.getReader() + for (;;) { + const { done, value } = await reader.read() + if (done) return + yield value + } + }, + } + } + } + + let body = tarStream(entries()) + if (gzip) + body = body.pipeThrough( + new CompressionStream('gzip') as unknown as TransformStream, + ) + const name = `${access.owner.slug}-${access.collection.slug}-${v.semver}.tar${gzip ? '.gz' : ''}` + return new Response(body, { + headers: { + 'content-type': gzip ? 'application/gzip' : 'application/x-tar', + 'content-disposition': `attachment; filename="${name}"`, + }, + }) + }) + + return app +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 38aa1a2..0845e0a 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -7,6 +7,7 @@ import { type Context, type ExecutionContext, Hono } from 'hono' import type { Principal } from './api/access.js' import { collectionRoutes } from './api/collections.js' +import { exportRoutes } from './api/export.js' import { fileRoutes } from './api/files.js' import { manageRoutes } from './api/manage.js' import { pushRoutes } from './api/push.js' @@ -90,6 +91,7 @@ export function createApp(setup: Setup) { // Before the :owner/:slug routes: /api/collections/files/:hash would match them. app.route('/', recordRoutes()) app.route('/api/collections', fileRoutes()) + app.route('/api/collections', exportRoutes()) app.route('/api/collections', pushRoutes()) app.route('/api/collections', versionRoutes()) app.route('/api/collections', webhookRoutes()) diff --git a/packages/server/src/lib/tar.ts b/packages/server/src/lib/tar.ts new file mode 100644 index 0000000..9147439 --- /dev/null +++ b/packages/server/src/lib/tar.ts @@ -0,0 +1,80 @@ +/** + * A streaming tar writer over Web Streams (Workers and Node). Each entry's size + * must be known before its bytes; long names use a PAX extended header. + */ +const enc = new TextEncoder() +const BLOCK = 512 + +function octal(n: number, width: number): string { + return n.toString(8).padStart(width - 1, '0') + '\0' +} + +function header(name: string, size: number, type: '0' | 'x', mtime: number): Uint8Array { + const h = new Uint8Array(BLOCK) + const put = (s: string, off: number, len: number) => h.set(enc.encode(s).subarray(0, len), off) + put(name, 0, 100) + put(octal(0o644, 8), 100, 8) + put(octal(0, 8), 108, 8) + put(octal(0, 8), 116, 8) + put(octal(size, 12), 124, 12) + put(octal(Math.floor(mtime / 1000), 12), 136, 12) + put(' ', 148, 8) // checksum placeholder + put(type, 156, 1) + put('ustar\0', 257, 6) + put('00', 263, 2) + let sum = 0 + for (const b of h) sum += b + put(octal(sum, 7) + ' ', 148, 8) + return h +} + +function pax(name: string): Uint8Array { + // " path=\n", where counts itself. + const body = (n: number) => `${n} path=${name}\n` + let len = enc.encode(body(0)).byteLength + while (enc.encode(body(len)).byteLength !== len) len = enc.encode(body(len)).byteLength + return enc.encode(body(len)) +} + +const pad = (size: number) => (size % BLOCK === 0 ? 0 : BLOCK - (size % BLOCK)) + +export interface TarEntry { + name: string + size: number + /** Produces exactly `size` bytes. */ + body: () => AsyncIterable +} + +export function tarStream( + entries: AsyncIterable | Iterable, + mtime = Date.now(), +): ReadableStream { + return new ReadableStream({ + async start(controller) { + try { + for await (const e of entries as AsyncIterable) { + const nameBytes = enc.encode(e.name) + if (nameBytes.byteLength > 100) { + const p = pax(e.name) + controller.enqueue(header('PaxHeader', p.byteLength, 'x', mtime)) + controller.enqueue(p) + controller.enqueue(new Uint8Array(pad(p.byteLength))) + } + controller.enqueue(header(e.name, e.size, '0', mtime)) + let written = 0 + for await (const chunk of e.body()) { + written += chunk.byteLength + controller.enqueue(chunk) + } + if (written !== e.size) + throw new Error(`tar: ${e.name} produced ${written} bytes, declared ${e.size}`) + controller.enqueue(new Uint8Array(pad(e.size))) + } + controller.enqueue(new Uint8Array(BLOCK * 2)) + controller.close() + } catch (err) { + controller.error(err) + } + }, + }) +} diff --git a/packages/server/test/export.test.ts b/packages/server/test/export.test.ts new file mode 100644 index 0000000..10f8226 --- /dev/null +++ b/packages/server/test/export.test.ts @@ -0,0 +1,80 @@ +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterAll, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { cleanup, harness } from './harness.js' + +afterAll(cleanup) + +const LONG = 'T'.repeat(120) + +async function json(res: Response) { + return (await res.json()) as any +} + +describe('export', () => { + it('streams a valid tar.gz of what the caller may read', async () => { + const h = await harness() + const user = await h.member() + await h.collection('lib') + await h.ports.db.update(schema.collections).set({ public: true }) + const base = '/api/collections/org/lib' + const file = 'file bytes' + const fileHash = createHash('sha256').update(file).digest('hex') + await h.request(`${base}/files/${fileHash}`, { method: 'PUT', user, body: file }) + const Doc = { type: 'object' } + const sid = ( + await json( + await h.request(`${base}/push`, { + method: 'POST', + user, + json: { schemas: { Doc, [LONG]: Doc }, metadata: { readme: '# Lib\n' } }, + }), + ) + ).session_id + await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [ + { id: 'd1', type: 'Doc', data: { title: 'Ünïcode ✓', f: { $file: `sha256:${fileHash}` } } }, + { id: 'd2', type: 'Doc', data: { title: 'hidden' }, private: true }, + { id: 'l1', type: LONG, data: { x: 1 } }, + ], + }) + expect((await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user })).status).toBe( + 201, + ) + + const res = await h.request(`${base}/export`) + expect(res.headers.get('content-disposition')).toContain('org-lib-v1.0.0.tar.gz') + const dir = await mkdtemp(join(tmpdir(), 'ul-export-')) + await writeFile(join(dir, 'a.tar.gz'), new Uint8Array(await res.arrayBuffer())) + execFileSync('tar', ['-xzf', 'a.tar.gz'], { cwd: dir }) + const manifest = JSON.parse(await readFile(join(dir, 'manifest.json'), 'utf8')) + expect(manifest.version).toMatchObject({ semver: 'v1.0.0', recordCount: 2, fileCount: 1 }) + expect(await readFile(join(dir, 'README.md'), 'utf8')).toBe('# Lib\n') + const docs = (await readFile(join(dir, 'records/Doc.ndjson'), 'utf8')) + .trim() + .split('\n') + .map((l) => JSON.parse(l)) + expect(docs.map((d) => d.id)).toEqual(['d1']) // private record excluded for anonymous readers + expect(docs[0].hash).toMatch(/^[0-9a-f]{64}$/) + expect( + (await readFile(join(dir, `records/${LONG}.ndjson`), 'utf8')).trim().split('\n').length, + ).toBe(1) + expect(await readFile(join(dir, `files/${fileHash}`), 'utf8')).toBe(file) + + // Members get the private record too; plain tar works as well. + const owner = await h.request(`${base}/export?format=tar`, { user }) + await writeFile(join(dir, 'b.tar'), new Uint8Array(await owner.arrayBuffer())) + execFileSync('mkdir', ['-p', 'b'], { cwd: dir }) + execFileSync('tar', ['-xf', '../b.tar'], { cwd: join(dir, 'b') }) + const all = (await readFile(join(dir, 'b/records/Doc.ndjson'), 'utf8')).trim().split('\n') + expect(all.length).toBe(2) + }) +}) From a4e247ec2f456fd61d6f95831109408672ea34a7 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 18:03:20 -0400 Subject: [PATCH 017/178] =?UTF-8?q?v2=20migration=20(phase=209):=20v1=20Po?= =?UTF-8?q?stgres=20=E2=86=92=20v2=20converter,=20tested=20on=20PGlite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @underlay/migrate copies accounts, settings, files, webhooks, ARK tables and comments row for row, and replays each collection's ready versions oldest first through the v2 commit engine. A version's changes are the diff between consecutive v1 record sets, computed in Postgres in COLLATE "C" (UTF-8 byte) order, so each version costs O(changes); metadata-patch versions reuse their record sets. Versions keep their v1 semver, time and hashes (as format 1 aliases); records re-hashed by JCS get legacy_hashes aliases; types with field-level privacy become private types and are reported. commitVersion gains a migrated option (semver, createdAt, legacy hashes; indexes the reference log without firing webhooks). @underlay/server gets its package entry (src/index.ts). The test runs v1's own migrations in PGlite and checks the replayed history, privacy, legacy file keys, provenance through an alias, the signed log and lookup by legacy version hash. src/main.ts runs it against a real v1 database into a SQLite file and an S3/R2 bucket. --- packages/migrate/package.json | 25 + packages/migrate/src/convert.ts | 422 ++++++++++++++++ packages/migrate/src/main.ts | 74 +++ packages/migrate/test/convert.test.ts | 178 +++++++ packages/migrate/tsconfig.json | 17 + packages/migrate/vitest.config.ts | 7 + packages/server/src/api/ark.ts | 621 ++++++++++++++++++++++++ packages/server/src/index.ts | 27 ++ packages/server/src/lib/ark.ts | 161 ++++++ packages/server/src/versions/commit.ts | 35 +- packages/server/src/versions/publish.ts | 11 +- packages/server/test/ark-lib.test.ts | 251 ++++++++++ pnpm-lock.yaml | 123 ++++- 13 files changed, 1928 insertions(+), 24 deletions(-) create mode 100644 packages/migrate/package.json create mode 100644 packages/migrate/src/convert.ts create mode 100644 packages/migrate/src/main.ts create mode 100644 packages/migrate/test/convert.test.ts create mode 100644 packages/migrate/tsconfig.json create mode 100644 packages/migrate/vitest.config.ts create mode 100644 packages/server/src/api/ark.ts create mode 100644 packages/server/src/index.ts create mode 100644 packages/server/src/lib/ark.ts create mode 100644 packages/server/test/ark-lib.test.ts diff --git a/packages/migrate/package.json b/packages/migrate/package.json new file mode 100644 index 0000000..f929628 --- /dev/null +++ b/packages/migrate/package.json @@ -0,0 +1,25 @@ +{ + "name": "@underlay/migrate", + "version": "0.0.0", + "private": true, + "description": "Convert an Underlay v1 (Postgres) instance into v2: accounts and settings into SQLite, every collection's version history through the v2 commit engine into a repository.", + "type": "module", + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@underlay/core": "workspace:*", + "@underlay/repo": "workspace:*", + "@underlay/server": "workspace:*", + "drizzle-orm": "^0.45.2", + "postgres": "^3.4.9" + }, + "devDependencies": { + "@electric-sql/pglite": "^0.5.8", + "@types/node": "^25.0.0", + "tsx": "^4.19.0", + "typescript": "^6.0.0", + "vitest": "^4.1.6" + } +} diff --git a/packages/migrate/src/convert.ts b/packages/migrate/src/convert.ts new file mode 100644 index 0000000..ca83857 --- /dev/null +++ b/packages/migrate/src/convert.ts @@ -0,0 +1,422 @@ +/** + * v1 (Postgres) → v2 conversion (edge-redesign.md, "Migration"). + * + * 1. Accounts and settings: better-auth tables, organizations, ARK tables, + * instance settings and comments are copied row for row (same fields). + * 2. Files: the `files` table is copied; objects stay at their storage keys. + * 3. Collections: each collection's ready versions are replayed oldest first + * through the v2 commit engine. Each version's changes are the diff between + * its v1 record set and the previous one, computed in Postgres and streamed + * in (type, id) order (COLLATE "C" = UTF-8 byte order, the trees' order), so + * the cost is O(changes) per version. Metadata-patch versions (shared record + * sets) produce no record changes. Versions keep their v1 semver, time and + * hashes (as format 1 aliases). + * + * Records are re-hashed under format 2. A record whose hash changes (integer-like + * keys; JCS) gets a legacy_hashes alias. Field-level privacy is gone in format 2: + * a type with private fields becomes a wholly private type, and the report says + * so (edge-redesign.md asks to check this is unused before migrating). + */ +import { + type Change, + hashRecord, + hashSchema, + newSalt, + type RecordEntry, + utf8ByteLength, +} from '@underlay/core' +import { OUT_OF_LINE_BYTES } from '@underlay/repo' +import { + type BaseVersion, + commitVersion, + dbSchema as schema, + type Ports, + type TypeInput, +} from '@underlay/server' +import { getTableColumns } from 'drizzle-orm' + +/** Anything that runs a parameterized query against the v1 database. */ +export interface V1Db { + query>(text: string, params?: unknown[]): Promise +} + +export interface MigrationReport { + collections: number + versions: number + skippedVersions: { collection: string; semver: string; reason: string }[] + recordUpserts: number + legacyRecordAliases: number + fieldPrivateTypes: { collection: string; type: string }[] + copied: Record +} + +export const newReport = (): MigrationReport => ({ + collections: 0, + versions: 0, + skippedVersions: [], + recordUpserts: 0, + legacyRecordAliases: 0, + fieldPrivateTypes: [], + copied: {}, +}) + +const camel = (s: string) => s.replace(/_([a-z])/g, (_, c: string) => c.toUpperCase()) + +/** Copy a table whose v1 and v2 columns have the same names. Unknown columns are dropped. */ +// `any`: pnpm resolves drizzle-orm twice (different peers), so its table types don't unify. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +async function copyTable( + v1: V1Db, + ports: Ports, + v1Table: string, + v2: any, + report: MigrationReport, + map?: (row: Record) => Record | null, +) { + const columns = new Set(Object.keys(getTableColumns(v2) as object)) + const rows = await v1.query(`SELECT * FROM "${v1Table}"`) + let n = 0 + for (let i = 0; i < rows.length; i += 50) { + const values = rows + .slice(i, i + 50) + .map((r) => { + const out: Record = {} + for (const [k, v] of Object.entries(r)) if (columns.has(camel(k))) out[camel(k)] = v + return map ? map(out) : out + }) + .filter((r): r is Record => r !== null) + if (values.length === 0) continue + await ports.db + .insert(v2) + .values(values as never) + .onConflictDoNothing() + n += values.length + } + report.copied[v1Table] = n +} + +export async function migrateAccounts( + v1: V1Db, + ports: Ports, + report: MigrationReport, +): Promise { + // Order follows foreign keys. + await copyTable(v1, ports, 'user', schema.user, report) + await copyTable(v1, ports, 'organization', schema.organization, report) + await copyTable(v1, ports, 'member', schema.member, report) + await copyTable(v1, ports, 'account', schema.account, report) + await copyTable(v1, ports, 'session', schema.session, report) + await copyTable(v1, ports, 'verification', schema.verification, report) + await copyTable(v1, ports, 'invitation', schema.invitation, report) + await copyTable(v1, ports, 'apikey', schema.apikey, report) + await copyTable(v1, ports, 'instance_settings', schema.instanceSettings, report) + await copyTable(v1, ports, 'files', schema.files, report, (r) => ({ + ...r, + verifiedAt: r.createdAt, + })) +} + +/** Collection-scoped tables, after the collections exist. */ +export async function migrateCollectionSettings( + v1: V1Db, + ports: Ports, + report: MigrationReport, +): Promise { + await copyTable(v1, ports, 'collection_webhooks', schema.collectionWebhooks, report) + await copyTable(v1, ports, 'ark_shoulders', schema.arkShoulders, report) + await copyTable(v1, ports, 'ark_collections', schema.arkCollections, report) + await copyTable(v1, ports, 'ark_record_types', schema.arkRecordTypes, report) + await copyTable(v1, ports, 'page_comments', schema.pageComments, report) + // Labels move from v1 schema ids to format 2 schema hashes. + const labels = await v1.query<{ label: string; schema: unknown; created_at: Date }>( + 'SELECT l.label, s.schema, l.created_at FROM schema_labels l JOIN schemas s ON s.id = l.schema_id', + ) + for (const l of labels) { + await ports.db + .insert(schema.schemaLabels) + .values({ schemaHash: hashSchema(l.schema), label: l.label, createdAt: l.created_at }) + .onConflictDoNothing() + } + report.copied.schema_labels = labels.length +} + +interface V1Version { + id: string + semver: string + hash: string + public_hash: string | null + base_semver: string | null + message: string | null + metadata: Record | null + pushed_by: string | null + app_id: string | null + actor_id: string | null + records_from_version_id: string | null + created_at: Date +} + +/** + * Format 2 refuses field-level privacy. A type that used it becomes a private + * type: its private fields are never exposed, at the cost of the public ones. + */ +function fixFieldPrivacy(s: Record): { + schema: Record + changed: boolean +} { + const props = s.properties as Record> | undefined + if (!props || !Object.values(props).some((p) => p && p.private === true)) + return { schema: s, changed: false } + const cleaned: Record = {} + for (const [k, p] of Object.entries(props)) { + const { private: _drop, ...rest } = p + void _drop + cleaned[k] = rest + } + return { schema: { ...s, properties: cleaned, private: true }, changed: true } +} + +const PAGE = 2000 + +/** One type's record changes between two v1 record sets, in id order. */ +async function* typeDelta( + v1: V1Db, + prev: string | null, + cur: string, + type: string, +): AsyncGenerator<{ + id: string + upsert: { data: unknown; private: boolean; hash: string } | null +}> { + let afterU = '' + let afterD = '' + let ups: { id: string; private: boolean; h: string; data: unknown }[] = [] + let dels: { id: string }[] = [] + let doneU = false + let doneD = prev === null + const fillU = async () => { + if (doneU || ups.length) return + ups = await v1.query( + `SELECT vr.record_id AS id, vr.private, vr.record_hash AS h, ro.data + FROM version_records vr JOIN record_objects ro ON ro.hash = vr.record_hash + WHERE vr.version_id = $1 AND vr.type = $2 AND vr.record_id COLLATE "C" > $3 + ${ + prev + ? `AND NOT EXISTS (SELECT 1 FROM version_records p WHERE p.version_id = $5 + AND p.type = vr.type AND p.record_id = vr.record_id AND p.record_hash = vr.record_hash + AND p.private = vr.private)` + : '' + } + ORDER BY vr.record_id COLLATE "C" LIMIT $4`, + prev ? [cur, type, afterU, PAGE, prev] : [cur, type, afterU, PAGE], + ) + if (ups.length < PAGE) doneU = true + if (ups.length) afterU = ups[ups.length - 1]!.id + } + const fillD = async () => { + if (doneD || dels.length) return + dels = await v1.query( + `SELECT p.record_id AS id FROM version_records p + WHERE p.version_id = $1 AND p.type = $2 AND p.record_id COLLATE "C" > $3 + AND NOT EXISTS (SELECT 1 FROM version_records vr WHERE vr.version_id = $4 + AND vr.type = p.type AND vr.record_id = p.record_id) + ORDER BY p.record_id COLLATE "C" LIMIT $5`, + [prev, type, afterD, cur, PAGE], + ) + if (dels.length < PAGE) doneD = true + if (dels.length) afterD = dels[dels.length - 1]!.id + } + // Byte order, to match COLLATE "C". + const lt = (a: string, b: string) => Buffer.compare(Buffer.from(a), Buffer.from(b)) < 0 + for (;;) { + await fillU() + await fillD() + const u = ups[0] + const d = dels[0] + if (!u && !d) return + if (u && (!d || lt(u.id, d.id))) { + ups.shift() + yield { id: u.id, upsert: { data: u.data, private: u.private, hash: u.h } } + } else { + dels.shift() + yield { id: d!.id, upsert: null } + } + } +} + +export async function migrateCollection( + v1: V1Db, + ports: Ports, + collectionId: string, + report: MigrationReport, +): Promise { + const [col] = await v1.query<{ + id: string + organization_id: string + slug: string + name: string + public: boolean + created_at: Date + updated_at: Date + }>('SELECT * FROM collections WHERE id = $1', [collectionId]) + if (!col) throw new Error(`v1 collection ${collectionId} not found`) + await ports.db.batch([ + ports.db.insert(schema.collections).values({ + id: col.id, + organizationId: col.organization_id, + slug: col.slug, + name: col.name, + public: col.public, + privateSalt: newSalt(), + createdAt: col.created_at, + updatedAt: col.updated_at, + }), + ports.db.insert(schema.placements).values({ + collectionId: col.id, + locationId: schema.PLATFORM_LOCATION_ID, + role: 'primary', + sets: 'public+private', + }), + ]) + report.collections++ + const repo = await ports.stores.forCollection(col.id) + + const versions = await v1.query( + `SELECT id::text, semver, hash, public_hash, base_semver, message, metadata, pushed_by, app_id, + actor_id, records_from_version_id::text, created_at + FROM versions WHERE collection_id = $1 AND status = 'ready' + ORDER BY created_at, major, minor, patch`, + [collectionId], + ) + + let base: BaseVersion | null = null + let prevRecords: string | null = null + let prevFiles = new Set() + const aliases: { legacyHash: string; hash: string }[] = [] + + for (const v of versions) { + const recordsId = v.records_from_version_id ?? v.id + const schemaRows = await v1.query<{ slug: string; schema: Record }>( + 'SELECT vs.slug, s.schema FROM version_schemas vs JOIN schemas s ON s.id = vs.schema_id WHERE vs.version_id = $1', + [v.id], + ) + const root = base ? await repo.root(base.hash) : null + const basePriv = root?.private ? await repo.privateSet(root.private) : null + const fileRows = await v1.query<{ file_hash: string }>( + 'SELECT file_hash FROM version_files WHERE version_id = $1', + [v.id], + ) + const files = new Set(fileRows.map((f) => f.file_hash)) + + const types: TypeInput[] = schemaRows.map((row) => { + const fixed = fixFieldPrivacy(row.schema) + if ( + fixed.changed && + !report.fieldPrivateTypes.some((t) => t.collection === col.id && t.type === row.slug) + ) { + report.fieldPrivateTypes.push({ collection: col.id, type: row.slug }) + } + const s = fixed.schema + const privateType = s.private === true + const hasPub = !!root?.public.types[row.slug]?.root + const hasPriv = !!basePriv?.types[row.slug]?.root + const unchanged = prevRecords === recordsId + const stream = async function* ( + set: 'public' | 'private', + ): AsyncGenerator> { + for await (const d of typeDelta(v1, prevRecords, recordsId, row.slug)) { + const inOther = set === 'public' ? hasPub : hasPriv + if (!d.upsert) { + if (inOther) yield { key: d.id, entry: null } + continue + } + const target = privateType || d.upsert.private ? 'private' : 'public' + if (target !== set) { + if (inOther) yield { key: d.id, entry: null } + continue + } + const { hash, canonical } = hashRecord(d.id, row.slug, d.upsert.data) + report.recordUpserts++ + if (hash !== d.upsert.hash) aliases.push({ legacyHash: d.upsert.hash, hash }) + const size = utf8ByteLength(canonical) + const body = + size > OUT_OF_LINE_BYTES ? await repo.putOutOfLine(hash, canonical) : canonical + yield { key: d.id, entry: { key: d.id, hash, size, body } } + } + } + return { + slug: row.slug, + schema: s, + schemaHash: hashSchema(s), + public: unchanged || privateType ? null : stream('public'), + private: unchanged ? null : stream('private'), + } + }) + + const r = await commitVersion(ports, { + collectionId: col.id, + base, + types, + metadata: v.metadata, + declaredFiles: { + add: [...files].filter((h) => !prevFiles.has(h)), + remove: [...prevFiles].filter((h) => !files.has(h)), + }, + message: v.message, + pushedBy: v.pushed_by, + appId: v.app_id, + actorId: v.actor_id, + migrated: { + semver: v.semver, + createdAt: v.created_at, + legacyHash: v.hash, + legacyPublicHash: v.public_hash, + }, + }) + if (r.status === 'committed') { + report.versions++ + const nv = r.version + base = { + id: nv.id, + seq: nv.seq, + semver: nv.semver, + hash: nv.hash, + publicRefsRoot: nv.publicRefsRoot, + privateRefsRoot: nv.privateRefsRoot, + } + prevRecords = recordsId + prevFiles = files + } else { + // e.g. two v1 versions that differ only in what format 2 no longer records. + report.skippedVersions.push({ collection: col.slug, semver: v.semver, reason: r.status }) + prevRecords = recordsId + prevFiles = files + } + } + + for (let i = 0; i < aliases.length; i += 50) { + await ports.db + .insert(schema.legacyHashes) + .values(aliases.slice(i, i + 50).map((a) => ({ ...a, kind: 'record' as const }))) + .onConflictDoNothing() + } + report.legacyRecordAliases += aliases.length +} + +/** Everything: accounts, collections (each with its history), then collection settings. */ +export async function migrateAll( + v1: V1Db, + ports: Ports, + opts: { onCollection?: (slug: string) => void } = {}, +): Promise { + const report = newReport() + await migrateAccounts(v1, ports, report) + const cols = await v1.query<{ id: string; slug: string }>( + 'SELECT id::text, slug FROM collections ORDER BY created_at', + ) + for (const c of cols) { + opts.onCollection?.(c.slug) + await migrateCollection(v1, ports, c.id, report) + } + await migrateCollectionSettings(v1, ports, report) + return report +} diff --git a/packages/migrate/src/main.ts b/packages/migrate/src/main.ts new file mode 100644 index 0000000..c827c67 --- /dev/null +++ b/packages/migrate/src/main.ts @@ -0,0 +1,74 @@ +/** + * Run a v1 → v2 migration. + * + * V1_DATABASE_URL=postgres://… the v1 database (read only) + * TARGET_DB=file:./migrated.sqlite the v2 SQLite database to fill + * S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY the v2 bucket (R2) + * REPO_PREFIX (repo), INTERNAL_PREFIX (internal) + * SIGNING_KEY the target deployment's log key + * npx tsx packages/migrate/src/main.ts > report.json + * + * Then load the SQLite file into D1 (wrangler d1 export/import, or `.dump` and + * `wrangler d1 execute --file --remote`). File objects aren't copied: point the + * v2 bucket at the v1 bucket's keys or copy them first (build doc finding 16). + */ +import { ed25519Signer, generateSigningKey } from '@underlay/repo' +import { S3BlobStore } from '@underlay/repo/blob/s3' +import { + createStores, + drainSqliteJobs, + MemoryCache, + openNodeDb, + type Ports, + SqliteJobs, +} from '@underlay/server' +import postgres from 'postgres' + +import { migrateAll, type V1Db } from './convert.js' + +const env = process.env +if (!env.V1_DATABASE_URL || !env.S3_ENDPOINT) { + console.error( + 'Set V1_DATABASE_URL and the S3_* target bucket variables (see the header of this file).', + ) + process.exit(2) +} + +const sql = postgres(env.V1_DATABASE_URL, { max: 2 }) +const v1: V1Db = { + query: async (text, params) => (await sql.unsafe(text, (params ?? []) as never[])) as never, +} + +const db = await openNodeDb(env.TARGET_DB ?? 'file:./migrated.sqlite') +const cache = new MemoryCache() +const signer = await ed25519Signer(env.SIGNING_KEY ?? (await generateSigningKey())) +const ports: Ports = { + db, + cache, + stores: createStores(db, cache, { + bucket: new S3BlobStore({ + endpoint: env.S3_ENDPOINT, + bucket: env.S3_BUCKET ?? 'underlay', + accessKeyId: env.S3_ACCESS_KEY ?? '', + secretAccessKey: env.S3_SECRET_KEY ?? '', + region: env.S3_REGION ?? 'auto', + }), + repoPrefix: env.REPO_PREFIX ?? 'repo', + internalPrefix: env.INTERNAL_PREFIX ?? 'internal', + }), + jobs: new SqliteJobs(db), + signer: async () => signer, + outboundFetch: () => Promise.reject(new Error('No outbound requests during migration')), + waitUntil: (p) => void p.catch((err) => console.error(err)), +} + +const started = Date.now() +const report = await migrateAll(v1, ports, { + onCollection: (slug) => console.error(`[migrate] ${slug}`), +}) +// Reference-log indexing and compaction run as jobs; finish them here. +await drainSqliteJobs(ports) +console.log( + JSON.stringify({ ...report, seconds: Math.round((Date.now() - started) / 1000) }, null, 2), +) +await sql.end() diff --git a/packages/migrate/test/convert.test.ts b/packages/migrate/test/convert.test.ts new file mode 100644 index 0000000..4f75a0c --- /dev/null +++ b/packages/migrate/test/convert.test.ts @@ -0,0 +1,178 @@ +/** + * The converter against a real v1 schema: PGlite (Postgres in WASM) runs v1's + * own migrations, a fixture collection is written the way v1 stores it, and the + * result is checked through the v2 API. + */ +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' + +import { PGlite } from '@electric-sql/pglite' +import { hashRecord, hashSchema, legacyRecordHash } from '@underlay/core' +import { verifyLog } from '@underlay/repo' +import { afterAll, describe, expect, it } from 'vitest' + +import { cleanup, harness } from '../../server/test/harness.js' +import { migrateAll, type V1Db } from '../src/convert.js' + +afterAll(cleanup) + +const root = join(import.meta.dirname, '../../..') + +async function v1Database(): Promise<{ pg: PGlite; db: V1Db }> { + const pg = new PGlite() + const journal = JSON.parse( + await readFile(join(root, 'src/db/migrations/meta/_journal.json'), 'utf8'), + ) as { entries: { tag: string }[] } + for (const { tag } of journal.entries) { + const sql = await readFile(join(root, `src/db/migrations/${tag}.sql`), 'utf8') + for (const stmt of sql.split('--> statement-breakpoint')) if (stmt.trim()) await pg.exec(stmt) + } + const db: V1Db = { + query: async (text, params) => (await pg.query(text, params ?? [])).rows as never, + } + return { pg, db } +} + +const Author = { + type: 'object', + properties: { name: { type: 'string' }, scores: { type: 'object' }, photo: {} }, +} +const FILE = 'f'.repeat(64) + +describe('v1 → v2 migration', () => { + it('replays a collection history with its privacy, files and legacy hashes', async () => { + const { pg, db } = await v1Database() + const q = (s: string, p: unknown[] = []) => pg.query(s, p) + await q(`INSERT INTO "user" (id, name, email) VALUES ('u1', 'Ada', 'ada@example.org')`) + await q(`INSERT INTO organization (id, name, slug) VALUES ('o1', 'Org', 'org')`) + await q( + `INSERT INTO member (id, organization_id, user_id, role) VALUES ('m1', 'o1', 'u1', 'owner')`, + ) + await q( + `INSERT INTO collections (id, organization_id, slug, name, public) VALUES ('11111111-1111-1111-1111-111111111111', 'o1', 'lib', 'Lib', true)`, + ) + await q( + `INSERT INTO files (hash, size, mime_type, storage_key) VALUES ($1, 3, 'image/png', 'files/ff/ff/legacy')`, + [FILE], + ) + const s = await q(`INSERT INTO schemas (schema, schema_hash) VALUES ($1, $2) RETURNING id`, [ + Author, + hashSchema(Author), + ]) + const schemaId = (s.rows[0] as { id: string }).id + + const records = { + a1: { id: 'a', data: { name: 'A', photo: { $file: `sha256:${FILE}` } } }, + a2: { id: 'a', data: { name: 'A2', photo: { $file: `sha256:${FILE}` } } }, + b: { id: 'b', data: { name: 'B', scores: { 10: 1, 9: 2 } } }, // integer-like keys: re-hashed + c: { id: 'c', data: { name: 'C' } }, + } + const v1hash = (r: { id: string; data: unknown }) => legacyRecordHash(r.id, 'Author', r.data) + for (const r of Object.values(records)) { + await q( + `INSERT INTO record_objects (hash, record_id, type, data, size) VALUES ($1, $2, 'Author', $3, 10) ON CONFLICT DO NOTHING`, + [v1hash(r), r.id, r.data], + ) + } + const version = async ( + semver: string, + members: [{ id: string; data: unknown }, boolean][], + extra: { recordsFrom?: number; metadata?: object; at: string }, + ) => { + const [maj, min, pat] = semver.slice(1).split('.').map(Number) + const v = await q( + `INSERT INTO versions (collection_id, semver, major, minor, patch, hash, public_hash, metadata, record_count, file_count, total_bytes, records_from_version_id, status, created_at) + VALUES ('11111111-1111-1111-1111-111111111111', $1, $2, $3, $4, $5, $6, $7, $8, 1, 0, $9, 'ready', $10) RETURNING id`, + [ + semver, + maj, + min, + pat, + `private:${semver}`, + `public:${semver}`, + extra.metadata ?? null, + members.length, + extra.recordsFrom ?? null, + extra.at, + ], + ) + const id = (v.rows[0] as { id: number }).id + await q( + `INSERT INTO version_schemas (version_id, slug, schema_id) VALUES ($1, 'Author', $2)`, + [id, schemaId], + ) + await q(`INSERT INTO version_files (version_id, file_hash) VALUES ($1, $2)`, [id, FILE]) + for (const [r, priv] of members) { + await q( + `INSERT INTO version_records (version_id, record_hash, record_id, type, private) VALUES ($1, $2, $3, 'Author', $4)`, + [id, v1hash(r), r.id, priv], + ) + } + return id + } + await version( + 'v1.0.0', + [ + [records.a1, false], + [records.b, false], + [records.c, true], + ], + { metadata: { title: 'Lib' }, at: '2026-01-01T00:00:00Z' }, + ) + const v11 = await version( + 'v1.1.0', + [ + [records.a2, false], + [records.b, true], + ], + { metadata: { title: 'Lib' }, at: '2026-02-01T00:00:00Z' }, + ) + await version('v1.1.1', [], { + recordsFrom: v11, + metadata: { title: 'Lib', readme: 'hi' }, + at: '2026-03-01T00:00:00Z', + }) + + const h = await harness() + const report = await migrateAll(db, h.ports) + await h.drain() + expect(report).toMatchObject({ + collections: 1, + versions: 3, + skippedVersions: [], + fieldPrivateTypes: [], + }) + expect(report.legacyRecordAliases).toBeGreaterThan(0) + + const json = async (path: string, user?: string) => + (await (await h.request(path, user ? { user } : {})).json()) as any + const versions = await json('/api/collections/org/lib/versions', 'u1') + expect(versions.map((v: any) => v.semver)).toEqual(['v1.1.1', 'v1.1.0', 'v1.0.0']) + expect(versions.map((v: any) => v.recordCount)).toEqual([2, 2, 3]) + // Anonymous readers see only the public set: b went private in v1.1.0. + const pub = await json('/api/collections/org/lib/versions/v1.1.1/records') + expect(pub.records.map((r: any) => r.id)).toEqual(['a']) + const latest = await json('/api/collections/org/lib/versions/latest', 'u1') + expect(latest.metadata).toEqual({ title: 'Lib', readme: 'hi' }) + // The file kept its v1 storage key and is downloadable. + const file = await h.request(`/api/collections/org/lib/files/${FILE}`) + expect(file.status).toBe(302) + expect(file.headers.get('location')).toContain('files/ff/ff/legacy') + // Provenance by the format 1 hash of b resolves through the alias (members only now). + const prov = await json(`/api/records/${v1hash(records.b)}/provenance`, 'u1') + expect(prov.recordHash).toBe(hashRecord('b', 'Author', records.b.data).hash) + expect(prov.references.map((r: any) => r.semver)).toEqual(['v1.0.0', 'v1.1.0', 'v1.1.1']) + // Legacy version hashes, and the signed log over the replayed history. + const repo = await h.ports.stores.forCollection('11111111-1111-1111-1111-111111111111') + const { entries } = await verifyLog(repo, '11111111-1111-1111-1111-111111111111', [ + h.signer.publicKey, + ]) + expect(entries.map((e) => [e.semver, e.createdAt.slice(0, 10)])).toEqual([ + ['v1.0.0', '2026-01-01'], + ['v1.1.0', '2026-02-01'], + ['v1.1.1', '2026-03-01'], + ]) + const byLegacy = await json('/api/collections/org/lib/versions/private:v1.1.0', 'u1') + expect(byLegacy.semver).toBe('v1.1.0') + }) +}) diff --git a/packages/migrate/tsconfig.json b/packages/migrate/tsconfig.json new file mode 100644 index 0000000..96f05dd --- /dev/null +++ b/packages/migrate/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "target": "ES2024", + "lib": ["ES2024", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "skipLibCheck": true, + "isolatedModules": true, + "resolveJsonModule": true, + "types": ["node"], + "noEmit": true + }, + "include": ["src", "test", "drizzle.config.ts"] +} diff --git a/packages/migrate/vitest.config.ts b/packages/migrate/vitest.config.ts new file mode 100644 index 0000000..3e42797 --- /dev/null +++ b/packages/migrate/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + include: ['test/**/*.test.ts'], + }, +}) diff --git a/packages/server/src/api/ark.ts b/packages/server/src/api/ark.ts new file mode 100644 index 0000000..187b229 --- /dev/null +++ b/packages/server/src/api/ark.ts @@ -0,0 +1,621 @@ +/** + * ARKs (v1 shapes). + * + * GET /api/ark/resolve?path=ark:NAAN/… {type:'redirect', url, metadata} | 404 {type:'not_found'} + * GET /api/collections/:owner/:slug/ark {enabled, customUrl, arkUrl, shoulder, arkId} + * PATCH /api/collections/:owner/:slug/ark {enabled?, customUrl?} + * GET /api/collections/:owner/:slug/ark/record-types [{recordType, redirectUrlField}] + * PATCH /api/collections/:owner/:slug/ark/record-types {recordType, redirectUrlField | null} + * PUT /api/collections/:owner/:slug/ark/record-types {recordType, redirectUrlField} + * DELETE /api/collections/:owner/:slug/ark/record-types/:type + * PATCH /api/accounts/:slug/ark {naan | null} + * GET /ark:NAAN/… 302, ?info / ?? ERC text, ?json metadata + * + * Resolution sees what the caller may read: a non-member resolves only public + * collections and their public set, as everywhere else in v2. + */ +import { and, asc, eq, ne, sql } from 'drizzle-orm' +import { type Context, Hono, type MiddlewareHandler } from 'hono' + +import type { AppEnv } from '../app.js' +import * as schema from '../db/schema.js' +import { + buildArkUrl, + buildErc, + collectionToArkId, + DEFAULT_NAAN, + formatErcDate, + nextShoulderCounter, + parseArkPath, +} from '../lib/ark.js' +import type { Db, Ports } from '../ports.js' +import { findVersion, getRecord, loadView, type VersionRow } from '../versions/view.js' +import { type CollectionAccess, collectionAccess, jsonError, type Principal } from './access.js' + +// --- Shoulders and collection ARKs (also for wiring ARK fields into other routes) --- + +/** The org's shoulder, if it has one. */ +export async function orgShoulder(db: Db, organizationId: string): Promise { + const [row] = await db + .select({ shoulder: schema.arkShoulders.shoulder }) + .from(schema.arkShoulders) + .where(eq(schema.arkShoulders.organizationId, organizationId)) + .orderBy(asc(schema.arkShoulders.createdAt)) + .limit(1) + return row?.shoulder ?? null +} + +export async function getOrMintShoulder(db: Db, organizationId: string): Promise { + const existing = await orgShoulder(db, organizationId) + if (existing) return existing + + for (let attempt = 0; attempt < 10; attempt++) { + const [countRow] = await db.select({ count: sql`count(*)` }).from(schema.arkShoulders) + const counter = nextShoulderCounter(countRow?.count ?? 0) + const digit = Math.floor(Math.random() * 10).toString() + const shoulder = `ul${counter}${digit}` + // A concurrent mint can take the same shoulder; nothing inserted means retry. + const inserted = await db + .insert(schema.arkShoulders) + .values({ organizationId, shoulder }) + .onConflictDoNothing() + .returning({ shoulder: schema.arkShoulders.shoulder }) + if (inserted[0]) return inserted[0].shoulder + } + throw new Error('Failed to mint ARK shoulder after 10 attempts') +} + +/** Mint a collection's ARK (and its org's shoulder) if it has none. For collection creation. */ +export async function ensureCollectionArk( + db: Db, + collection: { id: string; organizationId: string }, +): Promise { + await getOrMintShoulder(db, collection.organizationId) + await db + .insert(schema.arkCollections) + .values({ collectionId: collection.id, arkId: collectionToArkId(collection.id) }) + .onConflictDoNothing() +} + +export interface CollectionArkInfo { + naan: string + shoulder: string + arkId: string +} + +/** + * What a collection's ARK URLs are built from, or null when it has no enabled + * ARK. Build URLs with buildArkUrl(naan, shoulder, arkId, semver?, type?, id?). + */ +export async function collectionArkInfo( + db: Db, + collectionId: string, +): Promise { + const [row] = await db + .select({ + arkId: schema.arkCollections.arkId, + orgId: schema.collections.organizationId, + naan: schema.organization.arkNaan, + }) + .from(schema.arkCollections) + .innerJoin(schema.collections, eq(schema.arkCollections.collectionId, schema.collections.id)) + .innerJoin(schema.organization, eq(schema.collections.organizationId, schema.organization.id)) + .where( + and( + eq(schema.arkCollections.collectionId, collectionId), + eq(schema.arkCollections.enabled, true), + ), + ) + .limit(1) + if (!row) return null + const shoulder = await orgShoulder(db, row.orgId) + if (!shoulder) return null + return { naan: row.naan ?? DEFAULT_NAAN, shoulder, arkId: row.arkId } +} + +// --- Resolution --- + +export type ArkResolution = + | { type: 'redirect'; url: string; metadata: Record } + | { type: 'not_found'; error?: string } + +const notFound: ArkResolution = { type: 'not_found' } + +/** + * Resolve `ark:NAAN/name` (anything before "ark:" is ignored) for a caller. + * Returns null when the string isn't an ARK at all. + */ +export async function resolveArk( + ports: Ports, + principal: Principal | null, + path: string, +): Promise { + const { db } = ports + const arkLabelIdx = path.indexOf('ark:') + if (arkLabelIdx === -1) return null + + let afterLabel = path.slice(arkLabelIdx + 4) + // "ark:/NAAN/…" is the older spelling of "ark:NAAN/…"; the ARK spec treats them as one. + if (afterLabel.startsWith('/')) afterLabel = afterLabel.slice(1) + const slashIdx = afterLabel.indexOf('/') + if (slashIdx === -1) return notFound + const naan = afterLabel.slice(0, slashIdx) + const pathAfterNaan = afterLabel.slice(slashIdx + 1) + if (!pathAfterNaan) return notFound + + let components + try { + components = parseArkPath(pathAfterNaan) + } catch { + // decodeURIComponent throws on malformed escapes + return notFound + } + if (!components) return notFound + const { shoulder, collectionArkId, version, recordType, recordId } = components + + const [shoulderRow] = await db + .select({ organizationId: schema.arkShoulders.organizationId }) + .from(schema.arkShoulders) + .where(eq(schema.arkShoulders.shoulder, shoulder)) + .limit(1) + if (!shoulderRow) return notFound + + const [row] = await db + .select({ + ark: schema.arkCollections, + collection: schema.collections, + owner: schema.organization, + }) + .from(schema.arkCollections) + .innerJoin(schema.collections, eq(schema.arkCollections.collectionId, schema.collections.id)) + .innerJoin(schema.organization, eq(schema.collections.organizationId, schema.organization.id)) + .where(eq(schema.arkCollections.arkId, collectionArkId)) + .limit(1) + if (!row || !row.ark.enabled) return notFound + // The shoulder must be the collection owner's: another org's shoulder in + // front of this collection's id is not its ARK. + if (shoulderRow.organizationId !== row.collection.organizationId) return notFound + + // Same visibility as the collection page: a private collection doesn't + // resolve (or reveal its name, owner and versions) to non-members. + const access = await collectionAccess(db, principal, row.owner.slug, row.collection.slug) + if (!access?.canRead) return notFound + const member = access.isMember + + const { collection, owner } = row + const resolvedNaan = owner.arkNaan ?? naan + + let v: VersionRow | null = null + if (version !== undefined) { + v = await findVersion(db, collection.id, version, collection.headVersionId) + if (!v) return notFound + } else if (collection.headVersionId) { + v = await findVersion(db, collection.id, 'latest', collection.headVersionId) + } + + const arkUrl = buildArkUrl( + resolvedNaan, + shoulder, + collectionArkId, + version !== undefined ? v!.semver : undefined, + recordType, + recordId, + ) + const base = { + who: owner.name, + where: arkUrl, + naan: resolvedNaan, + collectionName: collection.name, + ownerName: owner.name, + } + // Who pushed is for members only, as on the versions routes (v1 leaked it here). + const versionFields = (x: VersionRow) => ({ + semver: x.semver, + message: x.message, + ...(member ? { pushedBy: x.pushedBy, actorId: x.actorId } : {}), + appId: x.appId, + createdAt: x.createdAt, + }) + + if (recordType && recordId) { + const [rt] = await db + .select({ redirectUrlField: schema.arkRecordTypes.redirectUrlField }) + .from(schema.arkRecordTypes) + .where( + and( + eq(schema.arkRecordTypes.collectionId, collection.id), + eq(schema.arkRecordTypes.recordType, recordType), + ), + ) + .limit(1) + if (!rt || !v) return notFound + + // Non-members get the public set only: a private-set record is not found. + const repo = await ports.stores.forCollection(collection.id) + const view = await loadView(repo, v, member) + const type = view.types.find((t) => t.slug === recordType) + const rec = type ? await getRecord(view, type, recordId) : null + if (!type || !rec) return notFound + const { data } = JSON.parse(rec.body!) as { data: Record } + const typeSchema = await repo.schema(type.schemaHash) + + // Only http(s) targets: anything else (javascript:, data:) is an open redirect. + const redirectUrl = data?.[rt.redirectUrlField] + if (!isHttpUrl(redirectUrl)) return { type: 'not_found', error: 'No URL found for this record' } + + return { + type: 'redirect', + url: redirectUrl, + metadata: { + type: 'record', + ...base, + what: `${recordType} ${recordId} in ${collection.name}`, + when: formatErcDate(v.createdAt), + semver: v.semver, + recordType, + recordId, + schema: typeSchema, + data, + createdAt: v.createdAt, + arkUrl, + }, + } + } + + const kind = version !== undefined ? 'version' : 'collection' + if (row.ark.customUrl) { + return { + type: 'redirect', + url: row.ark.customUrl, + metadata: { + type: kind, + ...base, + what: v ? `${collection.name} ${v.semver}` : collection.name, + when: v ? formatErcDate(v.createdAt) : '(:unkn)', + ...(v ? versionFields(v) : {}), + arkUrl, + }, + } + } + + if (version !== undefined && v) { + // Page URLs use the bare semver (/v/1.0.0); stored semver is "v1.0.0". + return { + type: 'redirect', + url: `/${owner.slug}/${collection.slug}/v/${v.semver.replace(/^v/, '')}`, + metadata: { + type: 'version', + ...base, + what: `${collection.name} ${v.semver}`, + when: formatErcDate(v.createdAt), + ...versionFields(v), + arkUrl, + }, + } + } + + return { + type: 'redirect', + url: `/${owner.slug}/${collection.slug}`, + metadata: { + type: 'collection', + ...base, + what: collection.name, + when: v ? formatErcDate(v.createdAt) : '(:unkn)', + ...(v ? { semver: v.semver, createdAt: v.createdAt } : {}), + arkUrl, + }, + } +} + +// --- /ark: URLs --- + +const policy = (naan: string) => + [ + `The Underlay assigns identifiers within the ARK domain ${naan} with the following principles:`, + '', + '1. Persistence: ARKs are never reassigned. Once minted, an ARK will always resolve to the same collection or record, or return a tombstone response if the object has been deleted.', + '', + '2. Transparency: Appending ?info or ?? to any ARK returns an Electronic Resource Citation (ERC) describing the identified object.', + '', + '3. Openness: ARKs are free, open identifiers requiring no licensing fees. The Underlay uses the ARK scheme as specified by the ARK Alliance.', + '', + '4. Scope: Underlay ARKs primarily identify versioned data collections and the records within them. Collection ARKs redirect to the collection overview; version-qualified ARKs redirect to specific version pages; record ARKs redirect to the canonical URL of the identified record.', + '', + `For more information, see: https://underlay.org/ark:${naan}/`, + ].join('\n') + +const text = (body: string, status = 200) => + new Response(body, { status, headers: { 'content-type': 'text/plain; charset=utf-8' } }) + +/** GET /ark:NAAN/…: the NAAN's policy, an ERC (?info, ??), the metadata (?json), or a 302. */ +async function arkPage(c: Context): Promise { + const url = new URL(c.req.url) + const fullPath = url.pathname.slice(1) + let afterLabel = fullPath.slice(4) + if (afterLabel.startsWith('/')) afterLabel = afterLabel.slice(1) + const slashIdx = afterLabel.indexOf('/') + const naan = slashIdx === -1 ? afterLabel : afterLabel.slice(0, slashIdx) + const afterNaan = slashIdx === -1 ? '' : afterLabel.slice(slashIdx + 1) + if (!afterNaan.trim()) return text(policy(naan)) + + // In process, as the caller: a Worker can't fetch its own zone, and members + // should resolve what they can read. + const res = await resolveArk(c.var.ports, c.var.principal, fullPath) + if (!res || res.type === 'not_found') return text('ARK not found', 404) + + const { metadata } = res + const search = url.search + if (search === '?info' || search === '??' || search === '%3F%3F') { + const m = metadata as Record + return text( + buildErc({ + type: metadata.type as 'collection' | 'version' | 'record', + who: m.who ?? m.ownerName ?? '(:unkn)', + what: m.what ?? m.collectionName ?? '(:unkn)', + when: m.when ?? '(:unkn)', + where: m.where ?? m.arkUrl ?? '(:unkn)', + naan: m.naan ?? DEFAULT_NAAN, + }), + ) + } + if (search === '?json') { + return new Response(JSON.stringify(metadata, null, 2), { + headers: { 'content-type': 'application/json' }, + }) + } + // The deployment's public origin, not the request's: behind a proxy the + // request URL can be the internal one. + const target = res.url.startsWith('/') ? `${c.var.config.appUrl}${res.url}` : res.url + return c.redirect(target, 302) +} + +/** + * The /ark: handler as middleware, for mounting outside arkRoutes(). Must run + * after the middleware that sets ports, config and principal. + */ +export const arkMiddleware: MiddlewareHandler = async (c, next) => { + if (c.req.method !== 'GET' && c.req.method !== 'HEAD') return next() + if (!c.req.path.startsWith('/ark:')) return next() + return arkPage(c) +} + +// --- Settings access --- + +/** + * ARK settings are for members of the owning org (v1: any role). A private + * collection stays a 404 to everyone else; a public one is 401/403. + */ +async function requireArkMember( + c: Context, + write: boolean, +): Promise { + const access = await collectionAccess( + c.var.ports.db, + c.var.principal, + c.req.param('owner') ?? '', + c.req.param('slug') ?? '', + ) + if (!access?.canRead) return jsonError(c, 404, 'Collection not found') + if (!access.isMember || (write && !access.canWrite)) { + return c.var.principal + ? jsonError(c, 403, 'Forbidden') + : jsonError(c, 401, 'Authentication required') + } + return access +} + +async function setRecordType( + c: Context, + collectionId: string, + recordType: string, + redirectUrlField: string | null, +) { + const { db } = c.var.ports + if (redirectUrlField === null) { + await db + .delete(schema.arkRecordTypes) + .where( + and( + eq(schema.arkRecordTypes.collectionId, collectionId), + eq(schema.arkRecordTypes.recordType, recordType), + ), + ) + } else { + await db + .insert(schema.arkRecordTypes) + .values({ collectionId, recordType, redirectUrlField }) + .onConflictDoUpdate({ + target: [schema.arkRecordTypes.collectionId, schema.arkRecordTypes.recordType], + set: { redirectUrlField }, + }) + } + return c.json({ ok: true }) +} + +export function arkRoutes() { + const app = new Hono() + + app.get('/api/ark/resolve', async (c) => { + const path = c.req.query('path') + if (!path) return jsonError(c, 400, 'Missing path') + const res = await resolveArk(c.var.ports, c.var.principal, path) + if (!res) return jsonError(c, 400, 'Invalid ARK path') + return res.type === 'not_found' ? c.json(res, 404) : c.json(res) + }) + + app.get('/api/collections/:owner/:slug/ark', async (c) => { + const access = await requireArkMember(c, false) + if (access instanceof Response) return access + const { db } = c.var.ports + const [row] = await db + .select() + .from(schema.arkCollections) + .where(eq(schema.arkCollections.collectionId, access.collection.id)) + .limit(1) + const shoulder = row ? await orgShoulder(db, access.owner.id) : null + if (!row || !shoulder) + return c.json({ enabled: false, customUrl: null, arkUrl: null, shoulder: null, arkId: null }) + return c.json({ + enabled: row.enabled, + customUrl: row.customUrl, + arkUrl: buildArkUrl(access.owner.arkNaan ?? DEFAULT_NAAN, shoulder, row.arkId), + shoulder, + arkId: row.arkId, + }) + }) + + app.patch('/api/collections/:owner/:slug/ark', async (c) => { + const access = await requireArkMember(c, true) + if (access instanceof Response) return access + const body = (await c.req.json().catch(() => null)) as { + enabled?: unknown + customUrl?: unknown + } | null + if (!body) return jsonError(c, 400, 'Invalid JSON') + const { enabled, customUrl } = body + if (enabled !== undefined && typeof enabled !== 'boolean') + return jsonError(c, 400, 'enabled must be a boolean') + // The resolver redirects to customUrl, so only http(s) targets are allowed — + // anything else (javascript:, data:, protocol-relative) is an open redirect. + if (customUrl != null && customUrl !== '' && !isHttpUrl(customUrl)) + return jsonError(c, 422, 'customUrl must be an http(s) URL') + const url = customUrl === undefined ? undefined : (customUrl as string | null) || null + + const { db } = c.var.ports + const coll = access.collection + const [existing] = await db + .select({ collectionId: schema.arkCollections.collectionId }) + .from(schema.arkCollections) + .where(eq(schema.arkCollections.collectionId, coll.id)) + .limit(1) + if (!existing) { + // v2 collections aren't minted an ARK at creation (yet): mint on first enable. + await getOrMintShoulder(db, coll.organizationId) + await db.insert(schema.arkCollections).values({ + collectionId: coll.id, + arkId: collectionToArkId(coll.id), + enabled: enabled ?? true, + customUrl: url ?? null, + }) + } else { + const updates: { enabled?: boolean; customUrl?: string | null } = {} + if (enabled !== undefined) updates.enabled = enabled + if (url !== undefined) updates.customUrl = url + if (Object.keys(updates).length > 0) + await db + .update(schema.arkCollections) + .set(updates) + .where(eq(schema.arkCollections.collectionId, coll.id)) + } + return c.json({ ok: true }) + }) + + app.get('/api/collections/:owner/:slug/ark/record-types', async (c) => { + const access = await requireArkMember(c, false) + if (access instanceof Response) return access + const rows = await c.var.ports.db + .select({ + recordType: schema.arkRecordTypes.recordType, + redirectUrlField: schema.arkRecordTypes.redirectUrlField, + }) + .from(schema.arkRecordTypes) + .where(eq(schema.arkRecordTypes.collectionId, access.collection.id)) + .orderBy(asc(schema.arkRecordTypes.recordType)) + return c.json(rows) + }) + + // PATCH is v1's (null removes); PUT sets and DELETE removes, for clients that prefer them. + for (const method of ['patch', 'put'] as const) { + app[method]('/api/collections/:owner/:slug/ark/record-types', async (c) => { + const access = await requireArkMember(c, true) + if (access instanceof Response) return access + const body = (await c.req.json().catch(() => null)) as { + recordType?: unknown + redirectUrlField?: unknown + } | null + const { recordType, redirectUrlField } = body ?? {} + if (typeof recordType !== 'string' || !recordType) + return jsonError(c, 400, 'recordType required') + const removes = method === 'patch' && redirectUrlField === null + if (!removes && (typeof redirectUrlField !== 'string' || !redirectUrlField)) + return jsonError(c, 400, 'redirectUrlField required') + return setRecordType( + c, + access.collection.id, + recordType, + removes ? null : (redirectUrlField as string), + ) + }) + } + + app.delete('/api/collections/:owner/:slug/ark/record-types/:recordType', async (c) => { + const access = await requireArkMember(c, true) + if (access instanceof Response) return access + return setRecordType(c, access.collection.id, c.req.param('recordType'), null) + }) + + app.patch('/api/accounts/:slug/ark', async (c) => { + const { db } = c.var.ports + const body = (await c.req.json().catch(() => null)) as { naan?: unknown } | null + const naan = body?.naan + if (naan !== null && (typeof naan !== 'string' || !/^\d{1,16}$/.test(naan))) + return jsonError(c, 400, 'NAAN must be numeric (up to 16 digits)') + + const [org] = await db + .select() + .from(schema.organization) + .where(eq(schema.organization.slug, c.req.param('slug'))) + .limit(1) + if (!org) return jsonError(c, 404, 'Org not found') + + // Owner or admin of the org, as in v1. Scoped and read-only keys can't. + const p = c.var.principal + if (!p) return jsonError(c, 401, 'Authentication required') + let role: string | null = null + if (!p.collectionIds && p.scope !== 'read') { + if (p.orgId) role = p.orgId === org.id ? 'owner' : null + else { + const [m] = await db + .select({ role: schema.member.role }) + .from(schema.member) + .where(and(eq(schema.member.organizationId, org.id), eq(schema.member.userId, p.userId))) + .limit(1) + role = m?.role ?? null + } + } + if (role !== 'owner' && role !== 'admin') return jsonError(c, 403, 'Forbidden') + + // Whether a NAAN is registered to the org can't be checked here, but it must + // not collide with the instance's own NAAN or another org's claim — ARK + // resolution keys on it. + if (naan !== null) { + const [taken] = await db + .select({ id: schema.organization.id }) + .from(schema.organization) + .where(and(eq(schema.organization.arkNaan, naan), ne(schema.organization.id, org.id))) + .limit(1) + if (naan === DEFAULT_NAAN || taken) return jsonError(c, 409, 'That NAAN is already in use') + } + await db + .update(schema.organization) + .set({ arkNaan: naan }) + .where(eq(schema.organization.id, org.id)) + return c.json({ ok: true }) + }) + + // `{ark:.+}` matches across slashes: /ark:NAAN/name/Type/id. + app.get('/:ark{ark:.+}', arkPage) + + return app +} + +function isHttpUrl(value: unknown): value is string { + if (typeof value !== 'string') return false + try { + const { protocol } = new URL(value) + return protocol === 'https:' || protocol === 'http:' + } catch { + return false + } +} diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts new file mode 100644 index 0000000..59669b0 --- /dev/null +++ b/packages/server/src/index.ts @@ -0,0 +1,27 @@ +/** What other packages (web, migrate, tools) use from the server. */ +export { + type App, + type AppConfig, + type AppEnv, + type Authenticate, + createApp, + type RenderPage, + type Setup, +} from './app.js' +export type { Principal } from './api/access.js' +export { MemoryCache } from './cache.js' +export { openNodeDb } from './db/node.js' +export * as dbSchema from './db/schema.js' +export { drainSqliteJobs, registerJob, runJob, SqliteJobs } from './jobs.js' +export type { BlobStore, Cache, Db, JobMessage, Jobs, Ports, Stores } from './ports.js' +export { createStores, type PlatformStorage } from './stores.js' +export { + type BaseVersion, + commitVersion, + type CommitInput, + type CommitResult, + type TypeInput, +} from './versions/commit.js' +export { createCollectionRows } from './versions/fork.js' +export { compareSemver, parseSemver } from './versions/semver.js' +import './handlers.js' diff --git a/packages/server/src/lib/ark.ts b/packages/server/src/lib/ark.ts new file mode 100644 index 0000000..f4f8d70 --- /dev/null +++ b/packages/server/src/lib/ark.ts @@ -0,0 +1,161 @@ +/** + * ARK (Archival Resource Key) helpers: minting ids, check characters, parsing + * and building ARK URLs, and ERC text. Pure, so they behave the same on Node + * and Workers; the database side (shoulders, settings, resolution) is in + * api/ark.ts. Ported from v1 unchanged in behaviour: existing ARKs must keep + * resolving. + */ +import { createHash } from 'node:crypto' + +// Workers only have process.env under nodejs_compat; don't crash without it. +export const DEFAULT_NAAN = globalThis.process?.env?.ARK_DEFAULT_NAAN ?? '12345' +// ARKs are persistent identifiers, so they always name the canonical site, +// whichever deployment minted or serves them. +const SITE_URL = 'https://underlay.org' + +// Betanumeric: consonants (no 'l') + digits +export const BETANUMERIC = 'bcdfghjkmnpqrstvwxz0123456789' // 29 chars +export const BETANUMERIC_CONSONANTS = 'bcdfghjkmnpqrstvwxz' // 19 chars + +const ARK_ID_LENGTH = 10 + +// NCDA (Noid Check Digit Algorithm): computed over betanumeric characters only. +// Multiply each character's alphabet index by its 1-based position, sum, mod 29. +export function computeNcdaCheckChar(name: string): string { + let total = 0 + for (let i = 0; i < name.length; i++) { + total += BETANUMERIC.indexOf(name[i]!) * (i + 1) + } + return BETANUMERIC[total % BETANUMERIC.length]! +} + +// Converts a collection UUID to a 10-char betanumeric string. +// Uses SHA-256 of the UUID encoded in base-29; guarantees first char is a consonant +// so the primordinal shoulder parsing is always unambiguous. +export function collectionToArkId(collectionId: string): string { + const hash = createHash('sha256').update(collectionId).digest() + let n = BigInt('0x' + hash.subarray(0, 8).toString('hex')) + const base = BigInt(BETANUMERIC.length) + const chars: string[] = [] + for (let i = 0; i < ARK_ID_LENGTH; i++) { + chars.unshift(BETANUMERIC[Number(n % base)]!) + n = n / base + } + // Primordinal shoulder parsing requires collection IDs start with a consonant + if (!BETANUMERIC_CONSONANTS.includes(chars[0]!)) { + chars[0] = BETANUMERIC_CONSONANTS[hash[8]! % BETANUMERIC_CONSONANTS.length]! + } + return chars.join('') +} + +// Converts a 0-indexed count to a bijective base-19 consonant string. +// 0→"b", 1→"c", …, 18→"z", 19→"bb", 20→"bc", … +export function nextShoulderCounter(count: number): string { + const base = BETANUMERIC_CONSONANTS.length + let n = count + 1 + let result = '' + while (n > 0) { + n -= 1 + result = BETANUMERIC_CONSONANTS[n % base]! + result + n = Math.floor(n / base) + } + return result +} + +export interface ArkComponents { + shoulder: string + collectionArkId: string + version?: string + recordType?: string + recordId?: string +} + +// Parses the portion of an ARK URL after "ark:NAAN/". +// Handles: shoulder+arkId, optional .vN version suffix, optional /recordType/recordId. +export function parseArkPath(pathAfterNaan: string): ArkComponents | null { + const parts = pathAfterNaan.split('/') + const firstSeg = parts[0]! + + if (!firstSeg.startsWith('ul')) return null + + // Shoulder = "ul" + consonant counter + single digit + let i = 2 + while (i < firstSeg.length && BETANUMERIC_CONSONANTS.includes(firstSeg[i]!)) i++ + if (i >= firstSeg.length || !/^\d$/.test(firstSeg[i]!)) return null + const shoulder = firstSeg.slice(0, i + 1) + const remainder = firstSeg.slice(i + 1) + + // remainder = arkId + check char (with optional .vX.Y.Z suffix) + const dotVMatch = remainder.match(/\.v(\d+\.\d+\.\d+)$/) + let arkIdWithCheck: string + let version: string | undefined + if (dotVMatch) { + arkIdWithCheck = remainder.slice(0, dotVMatch.index!) + version = `v${dotVMatch[1]}` + } else { + arkIdWithCheck = remainder + } + + if (arkIdWithCheck.length < 2) return null + const collectionArkId = arkIdWithCheck.slice(0, -1) + const checkChar = arkIdWithCheck.slice(-1) + if (computeNcdaCheckChar(collectionArkId) !== checkChar) return null + + const result: ArkComponents = { shoulder, collectionArkId } + if (version !== undefined) result.version = version + if (parts.length >= 3) { + result.recordType = decodeURIComponent(parts[1]!) + result.recordId = parts.slice(2).map(decodeURIComponent).join('/') + } + return result +} + +export function buildArkUrl( + naan: string, + shoulder: string, + collectionArkId: string, + semver?: string, + recordType?: string, + recordId?: string, +): string { + const check = computeNcdaCheckChar(collectionArkId) + let name = shoulder + collectionArkId + check + if (semver !== undefined) name += `.${semver}` + if (recordType && recordId) + name += `/${encodeURIComponent(recordType)}/${encodeURIComponent(recordId)}` + return `${SITE_URL}/ark:${naan}/${name}` +} + +// Formats a date as YYYYMMDD for ERC responses. +export function formatErcDate(date: Date | string): string { + const d = new Date(date) + const y = d.getUTCFullYear() + const m = String(d.getUTCMonth() + 1).padStart(2, '0') + const day = String(d.getUTCDate()).padStart(2, '0') + return `${y}${m}${day}` +} + +export interface ErcMetadata { + type: 'collection' | 'version' | 'record' + who: string + what: string + when: string + where: string + naan: string +} + +export function buildErc(meta: ErcMetadata): string { + return [ + 'erc:', + `who: ${meta.who}`, + `what: ${meta.what}`, + `when: ${meta.when}`, + `where: ${meta.where}`, + '', + 'erc-support:', + 'who: Underlay', + 'what: Underlay ARK Service', + 'when: 20260504', + `where: ${SITE_URL}/ark:${meta.naan}/`, + ].join('\n') +} diff --git a/packages/server/src/versions/commit.ts b/packages/server/src/versions/commit.ts index 9489d0d..594bd63 100644 --- a/packages/server/src/versions/commit.ts +++ b/packages/server/src/versions/commit.ts @@ -50,7 +50,7 @@ import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' import { applyFileSet, FileRefDelta, fileSizes, type SetName } from './file-refs.js' import { publishVersion, type SchemaUsageChange } from './publish.js' -import { bumpType, deriveSemver } from './semver.js' +import { bumpType, deriveSemver, parseSemver } from './semver.js' export type ChangeSource = Iterable> | AsyncIterable> @@ -87,6 +87,17 @@ export interface CommitInput { actorId?: string | null /** Validate a record's data against its type's schema; errors or null. Used when a schema changes. */ validate?: (schema: Record, data: unknown) => string[] | null + /** + * Migration only: keep a v1 version's identity. Its semver (v1's rules may + * have differed over time), creation time and format 1 hashes; and skip the + * post-publish job (webhooks would fire for history). + */ + migrated?: { + semver: string + createdAt: Date + legacyHash: string | null + legacyPublicHash: string | null + } } export type CommitResult = @@ -336,7 +347,9 @@ export async function commitVersion(ports: Ports, input: CommitInput): Promise { const v = p.version const now = Date.now() + const createdAt = v.createdAt?.getTime() ?? now // Literal values for INSERT … SELECT. Raw SQL bypasses column mapping, so JSON // and booleans are given in their stored form. const lit = (value: unknown) => sql`${value}` @@ -96,8 +101,8 @@ export async function publishVersion( minor: lit(v.minor).as('minor'), patch: lit(v.patch).as('patch'), hash: lit(v.hash).as('hash'), - legacyHash: lit(null).as('legacy_hash'), - legacyPublicHash: lit(null).as('legacy_public_hash'), + legacyHash: lit(v.legacyHash ?? null).as('legacy_hash'), + legacyPublicHash: lit(v.legacyPublicHash ?? null).as('legacy_public_hash'), baseSemver: lit(v.baseSemver).as('base_semver'), message: lit(v.message).as('message'), pushedBy: lit(v.pushedBy).as('pushed_by'), @@ -119,7 +124,7 @@ export async function publishVersion( privateRefsRoot: lit(v.privateRefsRoot).as('private_refs_root'), refsIndexed: lit(0).as('refs_indexed'), changes: lit(JSON.stringify(v.changes)).as('changes'), - createdAt: lit(now).as('created_at'), + createdAt: lit(createdAt).as('created_at'), }) .from(schema.collections) .where(headIsBase) as never, diff --git a/packages/server/test/ark-lib.test.ts b/packages/server/test/ark-lib.test.ts new file mode 100644 index 0000000..16c5418 --- /dev/null +++ b/packages/server/test/ark-lib.test.ts @@ -0,0 +1,251 @@ +import { describe, expect, test } from 'vitest' + +import { + BETANUMERIC, + BETANUMERIC_CONSONANTS, + buildArkUrl, + buildErc, + collectionToArkId, + computeNcdaCheckChar, + formatErcDate, + nextShoulderCounter, + parseArkPath, +} from '../src/lib/ark.js' + +describe('computeNcdaCheckChar', () => { + test('returns a betanumeric character', () => { + const ch = computeNcdaCheckChar('bcdf') + expect(BETANUMERIC).toContain(ch) + }) + + test('is deterministic', () => { + expect(computeNcdaCheckChar('test123')).toBe(computeNcdaCheckChar('test123')) + }) + + test('different inputs produce different check chars', () => { + const a = computeNcdaCheckChar('abc') + const b = computeNcdaCheckChar('xyz') + // Not guaranteed for all inputs, but these particular ones differ + expect(a).not.toBe(b) + }) +}) + +describe('collectionToArkId', () => { + test('returns a 10-character string', () => { + const id = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + expect(id).toHaveLength(10) + }) + + test('uses only betanumeric characters', () => { + const id = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + for (const ch of id) { + expect(BETANUMERIC).toContain(ch) + } + }) + + test('first character is always a consonant', () => { + // Test several UUIDs to exercise the fallback path + const uuids = [ + '550e8400-e29b-41d4-a716-446655440000', + '6ba7b810-9dad-11d1-80b4-00c04fd430c8', + 'f47ac10b-58cc-4372-a567-0e02b2c3d479', + '00000000-0000-0000-0000-000000000000', + 'ffffffff-ffff-ffff-ffff-ffffffffffff', + ] + for (const uuid of uuids) { + const id = collectionToArkId(uuid) + expect(BETANUMERIC_CONSONANTS).toContain(id[0]) + } + }) + + test('is deterministic', () => { + const uuid = '550e8400-e29b-41d4-a716-446655440000' + expect(collectionToArkId(uuid)).toBe(collectionToArkId(uuid)) + }) + + // ARKs v1 minted are in circulation; the derivation must never change. + test('matches the ids v1 minted', () => { + expect(collectionToArkId('550e8400-e29b-41d4-a716-446655440000')).toBe('f623mpkw3m') + }) + + test('different UUIDs produce different IDs', () => { + const a = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + const b = collectionToArkId('6ba7b810-9dad-11d1-80b4-00c04fd430c8') + expect(a).not.toBe(b) + }) +}) + +describe('nextShoulderCounter', () => { + test('maps 0 to first consonant', () => { + expect(nextShoulderCounter(0)).toBe('b') + }) + + test('maps sequential counts to single consonants', () => { + expect(nextShoulderCounter(1)).toBe('c') + expect(nextShoulderCounter(2)).toBe('d') + }) + + test('maps 18 to last single consonant', () => { + expect(nextShoulderCounter(18)).toBe('z') + }) + + test('wraps to two characters at 19', () => { + expect(nextShoulderCounter(19)).toBe('bb') + expect(nextShoulderCounter(20)).toBe('bc') + }) + + test('uses only consonant characters', () => { + for (let i = 0; i < 50; i++) { + const result = nextShoulderCounter(i) + for (const ch of result) { + expect(BETANUMERIC_CONSONANTS).toContain(ch) + } + } + }) +}) + +describe('parseArkPath', () => { + // Helper: build a valid path from components + function makeValidPath(shoulder: string, arkId: string, semver?: string) { + const check = computeNcdaCheckChar(arkId) + let path = `${shoulder}${arkId}${check}` + if (semver !== undefined) path += `.${semver}` + return path + } + + test('parses a basic collection ARK', () => { + const arkId = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + const path = makeValidPath('ulb3', arkId) + const result = parseArkPath(path) + expect(result).toEqual({ + shoulder: 'ulb3', + collectionArkId: arkId, + }) + }) + + test('parses ARK with version suffix', () => { + const arkId = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + const path = makeValidPath('ulb3', arkId, 'v5.0.0') + const result = parseArkPath(path) + expect(result).toMatchObject({ + shoulder: 'ulb3', + collectionArkId: arkId, + version: 'v5.0.0', + }) + }) + + test('parses ARK with record type and record ID', () => { + const arkId = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + const check = computeNcdaCheckChar(arkId) + const path = `ulb3${arkId}${check}/Article/rec-001` + const result = parseArkPath(path) + expect(result).toMatchObject({ + shoulder: 'ulb3', + collectionArkId: arkId, + recordType: 'Article', + recordId: 'rec-001', + }) + }) + + test('rejects paths not starting with ul', () => { + expect(parseArkPath('xxb3abcdefghijk')).toBeNull() + }) + + test('rejects paths with invalid check digit', () => { + const arkId = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + const badCheck = arkId + 'x' // wrong check char (almost certainly) + expect(parseArkPath(`ulb3${badCheck}`)).toBeNull() + }) + + test('rejects version 0', () => { + const arkId = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + const path = makeValidPath('ulb3', arkId) + '.v0' + // version 0 is replaced by the full arkIdWithCheck since .v0 causes vNum < 1 + expect( + parseArkPath(path.replace(makeValidPath('ulb3', arkId), makeValidPath('ulb3', arkId))), + ).toBeNull() + }) + + test('handles multi-character shoulder counters', () => { + const arkId = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + const path = makeValidPath('ulbc5', arkId) + const result = parseArkPath(path) + expect(result).toMatchObject({ + shoulder: 'ulbc5', + collectionArkId: arkId, + }) + }) +}) + +describe('buildArkUrl', () => { + test('builds a basic ARK URL', () => { + const url = buildArkUrl('12345', 'ulb3', 'bcdfghjkmn') + const check = computeNcdaCheckChar('bcdfghjkmn') + expect(url).toBe(`https://underlay.org/ark:12345/ulb3bcdfghjkmn${check}`) + }) + + test('includes version suffix', () => { + const url = buildArkUrl('12345', 'ulb3', 'bcdfghjkmn', 'v2.0.0') + const check = computeNcdaCheckChar('bcdfghjkmn') + expect(url).toBe(`https://underlay.org/ark:12345/ulb3bcdfghjkmn${check}.v2.0.0`) + }) + + test('includes record type and record ID', () => { + const url = buildArkUrl('12345', 'ulb3', 'bcdfghjkmn', undefined, 'Article', 'rec-1') + const check = computeNcdaCheckChar('bcdfghjkmn') + expect(url).toBe(`https://underlay.org/ark:12345/ulb3bcdfghjkmn${check}/Article/rec-1`) + }) + + test('roundtrips with parseArkPath', () => { + const naan = '12345' + const shoulder = 'ulb3' + const arkId = collectionToArkId('550e8400-e29b-41d4-a716-446655440000') + + const url = buildArkUrl(naan, shoulder, arkId, 'v3.0.0', 'Article', 'rec-001') + // Extract the path after "ark:NAAN/" + const pathAfterNaan = url.split(`ark:${naan}/`)[1]! + const parsed = parseArkPath(pathAfterNaan) + + expect(parsed).toMatchObject({ + shoulder, + collectionArkId: arkId, + version: 'v3.0.0', + recordType: 'Article', + recordId: 'rec-001', + }) + }) +}) + +describe('formatErcDate', () => { + test('formats a Date object as YYYYMMDD', () => { + expect(formatErcDate(new Date('2026-05-04T00:00:00Z'))).toBe('20260504') + }) + + test('formats a date string', () => { + expect(formatErcDate('2024-01-15T12:00:00Z')).toBe('20240115') + }) + + test('pads month and day with zeros', () => { + expect(formatErcDate(new Date('2026-01-02T00:00:00Z'))).toBe('20260102') + }) +}) + +describe('buildErc', () => { + test('produces a valid ERC record', () => { + const erc = buildErc({ + type: 'collection', + who: 'Test Author', + what: 'Test Collection', + when: '20260504', + where: 'https://underlay.org/ark:12345/ulb3test', + naan: '12345', + }) + expect(erc).toContain('erc:') + expect(erc).toContain('who: Test Author') + expect(erc).toContain('what: Test Collection') + expect(erc).toContain('when: 20260504') + expect(erc).toContain('where: https://underlay.org/ark:12345/ulb3test') + expect(erc).toContain('erc-support:') + expect(erc).toContain('who: Underlay') + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5253cd4..6ffe0b8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,7 +16,7 @@ importers: version: 3.1104.0 '@better-auth/api-key': specifier: ^1.6.11 - version: 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)))) + version: 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)))) '@codemirror/autocomplete': specifier: ^6.20.1 version: 6.20.2 @@ -49,7 +49,7 @@ importers: version: 3.0.1(ajv@8.20.0) better-auth: specifier: ^1.6.11 - version: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) + version: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) better-sqlite3: specifier: ^12.9.0 version: 12.9.0 @@ -58,7 +58,7 @@ importers: version: 14.0.3 drizzle-orm: specifier: ^0.45.0 - version: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) + version: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) hono: specifier: ^4 version: 4.12.18 @@ -200,6 +200,40 @@ importers: specifier: ^4.1.6 version: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + packages/migrate: + dependencies: + '@underlay/core': + specifier: workspace:* + version: link:../core + '@underlay/repo': + specifier: workspace:* + version: link:../repo + '@underlay/server': + specifier: workspace:* + version: link:../server + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) + postgres: + specifier: ^3.4.9 + version: 3.4.9 + devDependencies: + '@electric-sql/pglite': + specifier: ^0.5.8 + version: 0.5.8 + '@types/node': + specifier: ^25.0.0 + version: 25.6.2 + tsx: + specifier: ^4.19.0 + version: 4.21.0 + typescript: + specifier: ^6.0.0 + version: 6.0.3 + vitest: + specifier: ^4.1.6 + version: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + packages/repo: dependencies: '@underlay/core': @@ -223,7 +257,7 @@ importers: dependencies: '@better-auth/api-key': specifier: ^1.6.11 - version: 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)))) + version: 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)))) '@hono/node-server': specifier: ^1.19.14 version: 1.19.14(hono@4.12.18) @@ -238,10 +272,10 @@ importers: version: link:../repo better-auth: specifier: ^1.6.11 - version: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) + version: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) drizzle-orm: specifier: ^0.45.2 - version: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) + version: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) hono: specifier: ^4.12.18 version: 4.12.18 @@ -271,6 +305,61 @@ importers: specifier: ^4.147.0 version: 4.147.0(@cloudflare/workers-types@5.20261003.1)(@types/node@25.6.2) + packages/web: + dependencies: + '@better-auth/api-key': + specifier: ^1.6.11 + version: 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)))) + '@tanstack/react-query': + specifier: ^5.101.0 + version: 5.101.0(react@19.2.6) + better-auth: + specifier: ^1.6.11 + version: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) + marked: + specifier: ^18.0.3 + version: 18.0.3 + react: + specifier: ^19.2.6 + version: 19.2.6 + react-dom: + specifier: ^19.2.6 + version: 19.2.6(react@19.2.6) + react-router: + specifier: ^7.15.0 + version: 7.15.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + devDependencies: + '@tailwindcss/vite': + specifier: ^4.3.0 + version: 4.3.0(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + '@types/node': + specifier: ^25.6.2 + version: 25.6.2 + '@types/react': + specifier: ^19.2.14 + version: 19.2.14 + '@types/react-dom': + specifier: ^19.2.3 + version: 19.2.3(@types/react@19.2.14) + '@vitejs/plugin-react': + specifier: ^5.2.0 + version: 5.2.0(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) + babel-plugin-react-compiler: + specifier: ^1.0.0 + version: 1.0.0 + tailwindcss: + specifier: ^4.3.0 + version: 4.3.0 + tsx: + specifier: ^4.21.0 + version: 4.21.0 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vite: + specifier: ^6.4.2 + version: 6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0) + packages: '@asamuzakjp/css-color@5.1.11': @@ -754,6 +843,9 @@ packages: '@drizzle-team/brocli@0.10.2': resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} + '@electric-sql/pglite@0.5.8': + resolution: {integrity: sha512-n9tsbUOhwx2epK1V0ZG9Ar4SHWUju04dhmzZXiSBXwBoleOvIfals33NAaWgagQVAL4Rbvx/Ptsu3P+pA09f6Q==} + '@emnapi/runtime@1.11.3': resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} @@ -4442,11 +4534,11 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.28.5 - '@better-auth/api-key@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))))': + '@better-auth/api-key@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))))': dependencies: '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/utils': 0.4.0 - better-auth: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) + better-auth: 1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))) zod: 4.4.3 '@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)': @@ -4463,12 +4555,12 @@ snapshots: optionalDependencies: '@cloudflare/workers-types': 5.20261003.1 - '@better-auth/drizzle-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))': + '@better-auth/drizzle-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))': dependencies: '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/utils': 0.4.0 optionalDependencies: - drizzle-orm: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) + drizzle-orm: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) '@better-auth/kysely-adapter@1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(kysely@0.28.17)': dependencies: @@ -4608,6 +4700,8 @@ snapshots: '@drizzle-team/brocli@0.10.2': {} + '@electric-sql/pglite@0.5.8': {} + '@emnapi/runtime@1.11.3': dependencies: tslib: 2.8.1 @@ -6010,10 +6104,10 @@ snapshots: baseline-browser-mapping@2.10.29: {} - better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))): + better-auth@1.6.11(@cloudflare/workers-types@5.20261003.1)(better-sqlite3@12.9.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1))(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(vitest@4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0))): dependencies: '@better-auth/core': 1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) - '@better-auth/drizzle-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1)) + '@better-auth/drizzle-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1)) '@better-auth/kysely-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0)(kysely@0.28.17) '@better-auth/memory-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0) '@better-auth/mongo-adapter': 1.6.11(@better-auth/core@1.6.11(@better-auth/utils@0.4.0)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@5.20261003.1)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.0) @@ -6032,7 +6126,7 @@ snapshots: optionalDependencies: better-sqlite3: 12.9.0 drizzle-kit: 0.31.10 - drizzle-orm: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) + drizzle-orm: 0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1) react: 19.2.6 react-dom: 19.2.6(react@19.2.6) vitest: 4.1.6(@types/node@25.6.2)(happy-dom@20.9.0)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@6.4.2(@types/node@25.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.9.0)) @@ -6155,9 +6249,10 @@ snapshots: esbuild: 0.25.12 tsx: 4.21.0 - drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1): + drizzle-orm@0.45.2(@cloudflare/workers-types@5.20261003.1)(@electric-sql/pglite@0.5.8)(@libsql/client@0.18.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@12.9.0)(kysely@0.28.17)(postgres@3.4.9)(sql.js@1.14.1): optionalDependencies: '@cloudflare/workers-types': 5.20261003.1 + '@electric-sql/pglite': 0.5.8 '@libsql/client': 0.18.0 '@types/better-sqlite3': 7.6.13 better-sqlite3: 12.9.0 From fa0c8038524717bdf1de2a542cc450987679df69 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 3 Oct 2026 18:06:49 -0400 Subject: [PATCH 018/178] v2 UI on Workers and Node (packages/web), ARK mounted @underlay/web is the v1 React Router UI rendered on the server with renderToReadableStream. Loaders call the API in-process through the renderPage(request, api) contract (a Worker can't fetch its own zone), the HTML template and asset tags are inlined at build time, and dist/client is served as Workers Static Assets (wrangler assets) or by Node's serveStatic. The SQL query tool and mirror admin are gone; pages for APIs that don't exist yet degrade instead of breaking. Records tables are now in the server-rendered HTML. Verified under workerd (wrangler dev): home, explore, collection, records, versions and schemas pages render; scripts/ssr-smoke.ts passes 25 checks on Node. ARK (ported by a subagent; helpers and tests landed in a4e247e) is mounted; new collections mint an ARK as in v1, and the UI's ARK settings are on. --- packages/repo/test/fake-s3-server.ts | 7 +- packages/repo/test/fake-s3.ts | 4 +- packages/server/package.json | 1 + packages/server/src/api/manage.ts | 1 + packages/server/src/app.ts | 2 + packages/server/src/node/main.ts | 30 +- packages/server/src/web.d.ts | 4 + packages/server/src/worker.ts | 3 + packages/server/test/ark.test.ts | 465 +++++++++ packages/server/wrangler.jsonc | 3 + packages/web/index.html | 27 + packages/web/package.json | 37 + packages/web/public/favicon.svg | 9 + packages/web/public/llms.txt | 845 +++++++++++++++++ packages/web/public/logoLight.svg | 9 + packages/web/scripts/ssr-smoke.ts | 260 +++++ packages/web/src/App.tsx | 43 + packages/web/src/components/ApiPlayground.tsx | 299 ++++++ packages/web/src/components/BaseLayout.tsx | 72 ++ .../web/src/components/CollectionExplorer.tsx | 364 +++++++ packages/web/src/components/CreateMenu.tsx | 47 + .../web/src/components/DiscussionDrawer.tsx | 506 ++++++++++ packages/web/src/components/DocsLayout.tsx | 81 ++ packages/web/src/components/DocsSearch.tsx | 192 ++++ .../web/src/components/ExploreTagsAdmin.tsx | 135 +++ .../components/FeaturedCollectionsAdmin.tsx | 168 ++++ packages/web/src/components/NotFound.tsx | 110 +++ packages/web/src/components/Root.tsx | 44 + packages/web/src/components/SchemaBrowser.tsx | 177 ++++ .../web/src/components/SchemaLabelManager.tsx | 138 +++ packages/web/src/components/SchemaList.tsx | 268 ++++++ .../web/src/components/SettingsLayout.tsx | 192 ++++ packages/web/src/components/Unavailable.tsx | 23 + packages/web/src/components/UserMenu.tsx | 123 +++ .../web/src/components/WebhooksSettings.tsx | 330 +++++++ .../web/src/components/collection-nav.tsx | 254 +++++ .../src/components/collection-overview.tsx | 387 ++++++++ packages/web/src/components/share-panel.tsx | 271 ++++++ packages/web/src/components/ui.tsx | 368 ++++++++ packages/web/src/components/version-views.tsx | 541 +++++++++++ packages/web/src/entry-client.tsx | 33 + packages/web/src/entry-server.tsx | 187 ++++ packages/web/src/env.d.ts | 8 + packages/web/src/global.css | 363 +++++++ packages/web/src/lib/api-keys.ts | 19 + packages/web/src/lib/app-context.ts | 9 + packages/web/src/lib/auth-client.ts | 9 + packages/web/src/lib/auth-middleware.ts | 13 + packages/web/src/lib/features.ts | 30 + packages/web/src/lib/fetch-base.ts | 60 ++ packages/web/src/lib/format.ts | 61 ++ packages/web/src/lib/kf-updates.ts | 8 + packages/web/src/lib/markdown.ts | 54 ++ packages/web/src/lib/records-page.ts | 43 + packages/web/src/lib/schemas.ts | 41 + packages/web/src/lib/share-token.tsx | 48 + packages/web/src/lib/use-dismissable.ts | 36 + packages/web/src/lib/use-is-owner.ts | 21 + packages/web/src/route-gen.ts | 81 ++ packages/web/src/routes/404.tsx | 10 + .../routes/[owner]/[collection]/diff.data.ts | 7 + .../src/routes/[owner]/[collection]/diff.tsx | 4 + .../routes/[owner]/[collection]/files.data.ts | 30 + .../src/routes/[owner]/[collection]/files.tsx | 45 + .../routes/[owner]/[collection]/index.data.ts | 17 + .../src/routes/[owner]/[collection]/index.tsx | 92 ++ .../[owner]/[collection]/records.data.ts | 32 + .../routes/[owner]/[collection]/records.tsx | 51 + .../[owner]/[collection]/schemas.data.ts | 36 + .../routes/[owner]/[collection]/schemas.tsx | 34 + .../[owner]/[collection]/settings.data.ts | 33 + .../routes/[owner]/[collection]/settings.tsx | 654 +++++++++++++ .../[owner]/[collection]/v/[n]/files.data.ts | 32 + .../[owner]/[collection]/v/[n]/files.tsx | 35 + .../[owner]/[collection]/v/[n]/index.data.ts | 47 + .../[owner]/[collection]/v/[n]/index.tsx | 47 + .../[collection]/v/[n]/records.data.ts | 34 + .../[owner]/[collection]/v/[n]/records.tsx | 41 + .../[collection]/v/[n]/schemas.data.ts | 31 + .../[owner]/[collection]/v/[n]/schemas.tsx | 34 + .../[owner]/[collection]/versions.data.ts | 23 + .../routes/[owner]/[collection]/versions.tsx | 115 +++ .../[collection]/versions/compare.data.ts | 23 + .../[owner]/[collection]/versions/compare.tsx | 382 ++++++++ packages/web/src/routes/[owner]/index.data.ts | 27 + packages/web/src/routes/[owner]/index.tsx | 212 +++++ .../src/routes/[owner]/settings/index.data.ts | 27 + .../web/src/routes/[owner]/settings/index.tsx | 472 ++++++++++ .../src/routes/[owner]/settings/keys.data.ts | 21 + .../web/src/routes/[owner]/settings/keys.tsx | 264 ++++++ .../routes/[owner]/settings/members.data.ts | 7 + .../src/routes/[owner]/settings/members.tsx | 290 ++++++ .../web/src/routes/admin/discussion.data.ts | 4 + packages/web/src/routes/admin/discussion.tsx | 178 ++++ .../web/src/routes/admin/explore-tags.tsx | 31 + packages/web/src/routes/dashboard.data.ts | 26 + packages/web/src/routes/dashboard.tsx | 231 +++++ .../web/src/routes/docs/api/accounts.data.ts | 1 + packages/web/src/routes/docs/api/accounts.tsx | 149 +++ .../src/routes/docs/api/collections.data.ts | 1 + .../web/src/routes/docs/api/collections.tsx | 422 +++++++++ .../web/src/routes/docs/api/files.data.ts | 1 + packages/web/src/routes/docs/api/files.tsx | 204 ++++ .../web/src/routes/docs/api/index.data.ts | 1 + packages/web/src/routes/docs/api/index.tsx | 213 +++++ .../web/src/routes/docs/api/versions.data.ts | 1 + packages/web/src/routes/docs/api/versions.tsx | 837 +++++++++++++++++ packages/web/src/routes/docs/concepts.data.ts | 1 + packages/web/src/routes/docs/concepts.tsx | 206 ++++ packages/web/src/routes/docs/index.data.ts | 1 + packages/web/src/routes/docs/index.tsx | 108 +++ .../web/src/routes/docs/integration.data.ts | 1 + packages/web/src/routes/docs/integration.tsx | 500 ++++++++++ .../web/src/routes/docs/quickstart.data.ts | 1 + packages/web/src/routes/docs/quickstart.tsx | 271 ++++++ .../web/src/routes/docs/self-host.data.ts | 1 + packages/web/src/routes/docs/self-host.tsx | 236 +++++ packages/web/src/routes/explore.data.ts | 38 + packages/web/src/routes/explore.tsx | 22 + .../web/src/routes/forgot-password.data.ts | 6 + packages/web/src/routes/forgot-password.tsx | 4 + packages/web/src/routes/index.data.ts | 27 + packages/web/src/routes/index.tsx | 299 ++++++ .../web/src/routes/invitations/accept.data.ts | 1 + .../web/src/routes/invitations/accept.tsx | 124 +++ packages/web/src/routes/login.data.ts | 1 + packages/web/src/routes/login.tsx | 51 + packages/web/src/routes/logout.data.ts | 1 + packages/web/src/routes/logout.tsx | 27 + packages/web/src/routes/new-org.tsx | 157 ++++ packages/web/src/routes/new.data.ts | 4 + packages/web/src/routes/new.tsx | 176 ++++ packages/web/src/routes/protocol.data.ts | 23 + packages/web/src/routes/protocol.tsx | 886 ++++++++++++++++++ .../web/src/routes/records/[hash].data.ts | 19 + packages/web/src/routes/records/[hash].tsx | 275 ++++++ .../web/src/routes/reset-password.data.ts | 6 + packages/web/src/routes/reset-password.tsx | 4 + packages/web/src/routes/schemas/[id].data.ts | 21 + packages/web/src/routes/schemas/[id].tsx | 204 ++++ packages/web/src/routes/schemas/index.data.ts | 1 + packages/web/src/routes/schemas/index.tsx | 18 + .../web/src/routes/settings/index.data.ts | 4 + packages/web/src/routes/settings/index.tsx | 330 +++++++ packages/web/src/routes/settings/keys.data.ts | 4 + packages/web/src/routes/settings/keys.tsx | 217 +++++ .../web/src/routes/settings/sessions.data.ts | 4 + packages/web/src/routes/settings/sessions.tsx | 109 +++ packages/web/src/routes/signup.data.ts | 6 + packages/web/src/routes/signup.tsx | 4 + packages/web/src/routes/superadmin.data.ts | 4 + packages/web/src/routes/superadmin.tsx | 67 ++ packages/web/tsconfig.json | 23 + packages/web/vite.config.ts | 100 ++ pnpm-lock.yaml | 3 + 155 files changed, 18062 insertions(+), 7 deletions(-) create mode 100644 packages/server/src/web.d.ts create mode 100644 packages/server/test/ark.test.ts create mode 100644 packages/web/index.html create mode 100644 packages/web/package.json create mode 100644 packages/web/public/favicon.svg create mode 100644 packages/web/public/llms.txt create mode 100644 packages/web/public/logoLight.svg create mode 100644 packages/web/scripts/ssr-smoke.ts create mode 100644 packages/web/src/App.tsx create mode 100644 packages/web/src/components/ApiPlayground.tsx create mode 100644 packages/web/src/components/BaseLayout.tsx create mode 100644 packages/web/src/components/CollectionExplorer.tsx create mode 100644 packages/web/src/components/CreateMenu.tsx create mode 100644 packages/web/src/components/DiscussionDrawer.tsx create mode 100644 packages/web/src/components/DocsLayout.tsx create mode 100644 packages/web/src/components/DocsSearch.tsx create mode 100644 packages/web/src/components/ExploreTagsAdmin.tsx create mode 100644 packages/web/src/components/FeaturedCollectionsAdmin.tsx create mode 100644 packages/web/src/components/NotFound.tsx create mode 100644 packages/web/src/components/Root.tsx create mode 100644 packages/web/src/components/SchemaBrowser.tsx create mode 100644 packages/web/src/components/SchemaLabelManager.tsx create mode 100644 packages/web/src/components/SchemaList.tsx create mode 100644 packages/web/src/components/SettingsLayout.tsx create mode 100644 packages/web/src/components/Unavailable.tsx create mode 100644 packages/web/src/components/UserMenu.tsx create mode 100644 packages/web/src/components/WebhooksSettings.tsx create mode 100644 packages/web/src/components/collection-nav.tsx create mode 100644 packages/web/src/components/collection-overview.tsx create mode 100644 packages/web/src/components/share-panel.tsx create mode 100644 packages/web/src/components/ui.tsx create mode 100644 packages/web/src/components/version-views.tsx create mode 100644 packages/web/src/entry-client.tsx create mode 100644 packages/web/src/entry-server.tsx create mode 100644 packages/web/src/env.d.ts create mode 100644 packages/web/src/global.css create mode 100644 packages/web/src/lib/api-keys.ts create mode 100644 packages/web/src/lib/app-context.ts create mode 100644 packages/web/src/lib/auth-client.ts create mode 100644 packages/web/src/lib/auth-middleware.ts create mode 100644 packages/web/src/lib/features.ts create mode 100644 packages/web/src/lib/fetch-base.ts create mode 100644 packages/web/src/lib/format.ts create mode 100644 packages/web/src/lib/kf-updates.ts create mode 100644 packages/web/src/lib/markdown.ts create mode 100644 packages/web/src/lib/records-page.ts create mode 100644 packages/web/src/lib/schemas.ts create mode 100644 packages/web/src/lib/share-token.tsx create mode 100644 packages/web/src/lib/use-dismissable.ts create mode 100644 packages/web/src/lib/use-is-owner.ts create mode 100644 packages/web/src/route-gen.ts create mode 100644 packages/web/src/routes/404.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/diff.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/diff.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/files.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/files.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/index.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/index.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/records.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/records.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/schemas.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/schemas.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/settings.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/settings.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/v/[n]/files.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/v/[n]/files.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/v/[n]/index.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/v/[n]/index.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/v/[n]/records.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/v/[n]/records.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/v/[n]/schemas.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/v/[n]/schemas.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/versions.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/versions.tsx create mode 100644 packages/web/src/routes/[owner]/[collection]/versions/compare.data.ts create mode 100644 packages/web/src/routes/[owner]/[collection]/versions/compare.tsx create mode 100644 packages/web/src/routes/[owner]/index.data.ts create mode 100644 packages/web/src/routes/[owner]/index.tsx create mode 100644 packages/web/src/routes/[owner]/settings/index.data.ts create mode 100644 packages/web/src/routes/[owner]/settings/index.tsx create mode 100644 packages/web/src/routes/[owner]/settings/keys.data.ts create mode 100644 packages/web/src/routes/[owner]/settings/keys.tsx create mode 100644 packages/web/src/routes/[owner]/settings/members.data.ts create mode 100644 packages/web/src/routes/[owner]/settings/members.tsx create mode 100644 packages/web/src/routes/admin/discussion.data.ts create mode 100644 packages/web/src/routes/admin/discussion.tsx create mode 100644 packages/web/src/routes/admin/explore-tags.tsx create mode 100644 packages/web/src/routes/dashboard.data.ts create mode 100644 packages/web/src/routes/dashboard.tsx create mode 100644 packages/web/src/routes/docs/api/accounts.data.ts create mode 100644 packages/web/src/routes/docs/api/accounts.tsx create mode 100644 packages/web/src/routes/docs/api/collections.data.ts create mode 100644 packages/web/src/routes/docs/api/collections.tsx create mode 100644 packages/web/src/routes/docs/api/files.data.ts create mode 100644 packages/web/src/routes/docs/api/files.tsx create mode 100644 packages/web/src/routes/docs/api/index.data.ts create mode 100644 packages/web/src/routes/docs/api/index.tsx create mode 100644 packages/web/src/routes/docs/api/versions.data.ts create mode 100644 packages/web/src/routes/docs/api/versions.tsx create mode 100644 packages/web/src/routes/docs/concepts.data.ts create mode 100644 packages/web/src/routes/docs/concepts.tsx create mode 100644 packages/web/src/routes/docs/index.data.ts create mode 100644 packages/web/src/routes/docs/index.tsx create mode 100644 packages/web/src/routes/docs/integration.data.ts create mode 100644 packages/web/src/routes/docs/integration.tsx create mode 100644 packages/web/src/routes/docs/quickstart.data.ts create mode 100644 packages/web/src/routes/docs/quickstart.tsx create mode 100644 packages/web/src/routes/docs/self-host.data.ts create mode 100644 packages/web/src/routes/docs/self-host.tsx create mode 100644 packages/web/src/routes/explore.data.ts create mode 100644 packages/web/src/routes/explore.tsx create mode 100644 packages/web/src/routes/forgot-password.data.ts create mode 100644 packages/web/src/routes/forgot-password.tsx create mode 100644 packages/web/src/routes/index.data.ts create mode 100644 packages/web/src/routes/index.tsx create mode 100644 packages/web/src/routes/invitations/accept.data.ts create mode 100644 packages/web/src/routes/invitations/accept.tsx create mode 100644 packages/web/src/routes/login.data.ts create mode 100644 packages/web/src/routes/login.tsx create mode 100644 packages/web/src/routes/logout.data.ts create mode 100644 packages/web/src/routes/logout.tsx create mode 100644 packages/web/src/routes/new-org.tsx create mode 100644 packages/web/src/routes/new.data.ts create mode 100644 packages/web/src/routes/new.tsx create mode 100644 packages/web/src/routes/protocol.data.ts create mode 100644 packages/web/src/routes/protocol.tsx create mode 100644 packages/web/src/routes/records/[hash].data.ts create mode 100644 packages/web/src/routes/records/[hash].tsx create mode 100644 packages/web/src/routes/reset-password.data.ts create mode 100644 packages/web/src/routes/reset-password.tsx create mode 100644 packages/web/src/routes/schemas/[id].data.ts create mode 100644 packages/web/src/routes/schemas/[id].tsx create mode 100644 packages/web/src/routes/schemas/index.data.ts create mode 100644 packages/web/src/routes/schemas/index.tsx create mode 100644 packages/web/src/routes/settings/index.data.ts create mode 100644 packages/web/src/routes/settings/index.tsx create mode 100644 packages/web/src/routes/settings/keys.data.ts create mode 100644 packages/web/src/routes/settings/keys.tsx create mode 100644 packages/web/src/routes/settings/sessions.data.ts create mode 100644 packages/web/src/routes/settings/sessions.tsx create mode 100644 packages/web/src/routes/signup.data.ts create mode 100644 packages/web/src/routes/signup.tsx create mode 100644 packages/web/src/routes/superadmin.data.ts create mode 100644 packages/web/src/routes/superadmin.tsx create mode 100644 packages/web/tsconfig.json create mode 100644 packages/web/vite.config.ts diff --git a/packages/repo/test/fake-s3-server.ts b/packages/repo/test/fake-s3-server.ts index 0585117..28849ca 100644 --- a/packages/repo/test/fake-s3-server.ts +++ b/packages/repo/test/fake-s3-server.ts @@ -1,5 +1,8 @@ -/** Run the fake S3 server standalone (for `wrangler dev` smoke tests): tsx test/fake-s3-server.ts [port] */ +/** + * Run the fake S3 server standalone, for `wrangler dev` smoke tests: + * npx tsx test/fake-s3-server.ts [port] [bucket] + */ import { startFakeS3 } from './fake-s3.js' -const s3 = await startFakeS3(process.argv[3] ?? 'underlay-v2-staging') +const s3 = await startFakeS3(process.argv[3] ?? 'underlay-v2-staging', Number(process.argv[2] ?? 0)) console.log(`fake S3 at ${s3.url}`) diff --git a/packages/repo/test/fake-s3.ts b/packages/repo/test/fake-s3.ts index a901834..c06e614 100644 --- a/packages/repo/test/fake-s3.ts +++ b/packages/repo/test/fake-s3.ts @@ -13,7 +13,7 @@ export interface FakeS3 { close(): Promise } -export async function startFakeS3(bucket = 'test'): Promise { +export async function startFakeS3(bucket = 'test', listenPort = 0): Promise { const objects = new Map() const uploads = new Map }>() const requests: { method: string; url: string }[] = [] @@ -135,7 +135,7 @@ export async function startFakeS3(bucket = 'test'): Promise { } res.writeHead(400).end() }) - await new Promise((r) => server.listen(0, '127.0.0.1', r)) + await new Promise((r) => server.listen(listenPort, '127.0.0.1', r)) const port = (server.address() as { port: number }).port return { url: `http://127.0.0.1:${port}`, diff --git a/packages/server/package.json b/packages/server/package.json index 95b1c9e..29e7e11 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -19,6 +19,7 @@ "@libsql/client": "^0.18.0", "@underlay/core": "workspace:*", "@underlay/repo": "workspace:*", + "@underlay/web": "workspace:*", "better-auth": "^1.6.11", "drizzle-orm": "^0.45.2", "hono": "^4.12.18" diff --git a/packages/server/src/api/manage.ts b/packages/server/src/api/manage.ts index ef1c7d6..5af3fb8 100644 --- a/packages/server/src/api/manage.ts +++ b/packages/server/src/api/manage.ts @@ -21,6 +21,7 @@ import { headBase } from '../push/delta.js' import { commitVersion } from '../versions/commit.js' import { createCollectionRows, forkCollection } from '../versions/fork.js' import { jsonError, requireCollection } from './access.js' +import { ensureCollectionArk } from './ark.js' async function membership(c: Context, orgSlug: string) { const p = c.var.principal diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 0845e0a..0aa2f01 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -6,6 +6,7 @@ import { type Context, type ExecutionContext, Hono } from 'hono' import type { Principal } from './api/access.js' +import { arkRoutes } from './api/ark.js' import { collectionRoutes } from './api/collections.js' import { exportRoutes } from './api/export.js' import { fileRoutes } from './api/files.js' @@ -88,6 +89,7 @@ export function createApp(setup: Setup) { }), ) + app.route('/', arkRoutes()) // Before the :owner/:slug routes: /api/collections/files/:hash would match them. app.route('/', recordRoutes()) app.route('/api/collections', fileRoutes()) diff --git a/packages/server/src/node/main.ts b/packages/server/src/node/main.ts index c143db4..8898300 100644 --- a/packages/server/src/node/main.ts +++ b/packages/server/src/node/main.ts @@ -13,13 +13,18 @@ * SESSION_SECRET, OIDC_ISSUER_URL, OIDC_ISSUER_INTERNAL_URL, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET * better-auth and KF Auth (same names as v1's .env files) */ +import { relative } from 'node:path' +import { fileURLToPath } from 'node:url' + import { serve } from '@hono/node-server' +import { serveStatic } from '@hono/node-server/serve-static' import { ed25519Signer, generateSigningKey, type Signer } from '@underlay/repo' import { FsBlobStore, serveSignedBlob } from '@underlay/repo/blob/fs' import { S3BlobStore } from '@underlay/repo/blob/s3' +import { Hono } from 'hono' import '../handlers.js' -import { createApp } from '../app.js' +import { createApp, type RenderPage } from '../app.js' import { authenticator, createAuth } from '../auth/auth.js' import { MemoryCache } from '../cache.js' import { openNodeDb } from '../db/node.js' @@ -105,6 +110,17 @@ kick = () => void runJobs() setInterval(kick, 5000).unref() const config = { appUrl, deployment: env.DEPLOYMENT ?? 'dev' } + +// The UI (packages/web, built with `pnpm --filter @underlay/web build`). Without +// a build, the API still runs and pages are 404. +let renderPage: RenderPage | undefined +const clientDir = fileURLToPath(new URL('../../../web/dist/client', import.meta.url)) +try { + renderPage = (await import('@underlay/web')).renderPage +} catch { + console.warn('[underlay] @underlay/web is not built; serving the API only') +} +const staticFiles = serveStatic({ root: relative(process.cwd(), clientDir) }) const auth = createAuth( db, { @@ -124,13 +140,21 @@ const app = createApp(() => ({ config, authenticate: authenticator(() => auth), authHandler: (req) => auth.handler(req), + ...(renderPage ? { renderPage } : {}), })) +/** Static files from the client build, falling back to the app. */ +const staticApp = new Hono() +staticApp.use('*', staticFiles) +staticApp.all('*', (c) => app.fetch(c.req.raw)) +const staticOrApp = (req: Request) => staticApp.fetch(req) + serve({ port, fetch: (req) => { - if (fsBlobs && new URL(req.url).pathname.startsWith('/_blob/')) - return serveSignedBlob(fsBlobs, req) + const path = new URL(req.url).pathname + if (fsBlobs && path.startsWith('/_blob/')) return serveSignedBlob(fsBlobs, req) + if (!path.startsWith('/api/') && /\.[a-z0-9]+$/i.test(path)) return staticOrApp(req) return app.fetch(req) }, }) diff --git a/packages/server/src/web.d.ts b/packages/server/src/web.d.ts new file mode 100644 index 0000000..c12a902 --- /dev/null +++ b/packages/server/src/web.d.ts @@ -0,0 +1,4 @@ +// @underlay/web ships built JavaScript (dist/server/entry-server.js); this is its contract. +declare module '@underlay/web' { + export const renderPage: import('./app.js').RenderPage +} diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index 9795e99..34b9823 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -13,6 +13,8 @@ import { ed25519Signer, type Signer } from '@underlay/repo' import { S3BlobStore } from '@underlay/repo/blob/s3' import './handlers.js' +import { renderPage } from '@underlay/web' + import { createApp } from './app.js' import { type Auth, authenticator, createAuth } from './auth/auth.js' import { CfCache } from './cache.js' @@ -99,6 +101,7 @@ const app = createApp((c) => { config: { appUrl: env.APP_URL, deployment: env.DEPLOYMENT }, authenticate: authenticator(() => authFor(env, ports)), authHandler: (req) => authFor(env, ports).handler(req), + renderPage, } }) diff --git a/packages/server/test/ark.test.ts b/packages/server/test/ark.test.ts new file mode 100644 index 0000000..e9327f0 --- /dev/null +++ b/packages/server/test/ark.test.ts @@ -0,0 +1,465 @@ +import { eq } from 'drizzle-orm' +import { Hono } from 'hono' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { arkRoutes, getOrMintShoulder } from '../src/api/ark.js' +import type { AppEnv } from '../src/app.js' +import * as schema from '../src/db/schema.js' +import { buildArkUrl, collectionToArkId, computeNcdaCheckChar } from '../src/lib/ark.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) + +const Link = { + type: 'object', + properties: { title: { type: 'string' }, url: { type: 'string' } }, + required: ['title'], +} + +/** + * The ARK routes on the harness's ports, mounted as app.ts would: after the + * middleware that sets ports, config and principal. `x-test-user` signs in; + * `x-test-scope` makes it an API key with that scope. + */ +function arkApp(h: Harness) { + const app = new Hono() + app.use('*', async (c, next) => { + c.set('ports', h.ports) + c.set('config', { appUrl: 'https://ul.test', deployment: 'test' }) + const user = c.req.header('x-test-user') + const scope = (c.req.header('x-test-scope') ?? 'session') as 'session' | 'read' | 'write' + c.set('principal', user ? { userId: user, scope, collectionIds: null } : null) + await next() + }) + app.route('/', arkRoutes()) + return app +} + +interface Ctx { + h: Harness + user: string + collectionId: string + ark: ReturnType + /** fetch against the ARK routes. */ + req( + path: string, + init?: { method?: string; user?: string; scope?: string; json?: unknown }, + ): Promise + /** The path after "ark:NAAN/" for this collection, optionally a version and record. */ + name(semver?: string, type?: string, id?: string): string +} + +const base = '/api/collections/org/links' +let t: Ctx + +async function body(res: Response) { + return (await res.json()) as Record +} + +beforeAll(async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('links') + await h.ports.db + .update(schema.collections) + .set({ public: true }) + .where(eq(schema.collections.id, c.id)) + + let res = await h.request(`${base}/push`, { + method: 'POST', + user, + json: { schemas: { Link } }, + }) + const sid = (await body(res)).session_id + await h.request(`${base}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [ + { id: 'a', type: 'Link', data: { title: 'A', url: 'https://example.org/a' } }, + { id: 'b', type: 'Link', data: { title: 'B', url: 'https://example.org/b' }, private: true }, + { id: 'evil', type: 'Link', data: { title: 'E', url: 'javascript:alert(1)' } }, + { id: 'none', type: 'Link', data: { title: 'N' } }, + ], + }) + res = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) + expect(res.status).toBe(201) + + const ark = arkApp(h) + const req: Ctx['req'] = async (path, init = {}) => { + const headers = new Headers() + if (init.user) headers.set('x-test-user', init.user) + if (init.scope) headers.set('x-test-scope', init.scope) + if (init.json !== undefined) headers.set('content-type', 'application/json') + return ark.fetch( + new Request(`http://test${path}`, { + method: init.method ?? 'GET', + headers, + ...(init.json !== undefined ? { body: JSON.stringify(init.json) } : {}), + }), + ) + } + t = { + h, + user, + collectionId: c.id, + ark, + req, + name: () => '', + } + + // Enabling the ARK mints the org's shoulder and the collection's id. + res = await req(`${base}/ark`, { method: 'PATCH', user, json: { enabled: true } }) + expect(await body(res)).toEqual({ ok: true }) + res = await req(`${base}/ark`, { user }) + const settings = await body(res) + t.name = (semver, type, id) => + new URL(buildArkUrl('12345', settings.shoulder, settings.arkId, semver, type, id)).pathname + .split('/') + .slice(2) + .join('/') +}) + +const addOrg2 = () => + t.h.ports.db + .insert(schema.organization) + .values({ id: 'org2', name: 'Two', slug: 'two' }) + .onConflictDoNothing() + +const resolve = (path: string, user?: string) => + t.req(`/api/ark/resolve?${new URLSearchParams({ path })}`, user ? { user } : {}) + +describe('collection ARK settings', () => { + it('reports the minted ARK to members', async () => { + const res = await t.req(`${base}/ark`, { user: t.user }) + expect(await body(res)).toEqual({ + enabled: true, + customUrl: null, + arkUrl: expect.stringMatching(/^https:\/\/underlay\.org\/ark:12345\/ulb\d/), + shoulder: expect.stringMatching(/^ulb\d$/), + arkId: collectionToArkId(t.collectionId), + }) + }) + + it('is for members only', async () => { + expect((await t.req(`${base}/ark`)).status).toBe(401) + expect((await t.req(`${base}/ark`, { user: 'stranger' })).status).toBe(403) + // A read-scoped key may read the settings but not change them. + expect((await t.req(`${base}/ark`, { user: t.user, scope: 'read' })).status).toBe(200) + const res = await t.req(`${base}/ark`, { + method: 'PATCH', + user: t.user, + scope: 'read', + json: { enabled: false }, + }) + expect(res.status).toBe(403) + expect((await t.req(`/api/collections/org/nope/ark`, { user: t.user })).status).toBe(404) + }) + + it('reports no ARK for a collection that has none', async () => { + await t.h.collection('bare') + const res = await t.req(`/api/collections/org/bare/ark`, { user: t.user }) + expect(await body(res)).toEqual({ + enabled: false, + customUrl: null, + arkUrl: null, + shoulder: null, + arkId: null, + }) + // A private collection stays hidden from non-members. + expect((await t.req(`/api/collections/org/bare/ark`, { user: 'stranger' })).status).toBe(404) + }) + + it('rejects custom URLs that are not http(s)', async () => { + for (const customUrl of ['javascript:alert(1)', '//evil.example', 'data:text/html,x']) { + const res = await t.req(`${base}/ark`, { method: 'PATCH', user: t.user, json: { customUrl } }) + expect(res.status).toBe(422) + } + const res = await t.req(`${base}/ark`, { method: 'PATCH', user: t.user, json: { enabled: 1 } }) + expect(res.status).toBe(400) + }) + + it('sets record-type redirect fields', async () => { + const rt = `${base}/ark/record-types` + let res = await t.req(rt, { method: 'PUT', user: t.user, json: { recordType: 'X' } }) + expect(res.status).toBe(400) + res = await t.req(rt, { + method: 'PUT', + user: t.user, + json: { recordType: 'X', redirectUrlField: 'href' }, + }) + expect(await body(res)).toEqual({ ok: true }) + res = await t.req(rt, { + method: 'PATCH', + user: t.user, + json: { recordType: 'Y', redirectUrlField: 'link' }, + }) + expect(res.status).toBe(200) + expect(await (await t.req(rt, { user: t.user })).json()).toEqual([ + { recordType: 'X', redirectUrlField: 'href' }, + { recordType: 'Y', redirectUrlField: 'link' }, + ]) + // v1's PATCH with null removes; DELETE does the same. + await t.req(rt, { + method: 'PATCH', + user: t.user, + json: { recordType: 'X', redirectUrlField: null }, + }) + await t.req(`${rt}/Y`, { method: 'DELETE', user: t.user }) + expect(await (await t.req(rt, { user: t.user })).json()).toEqual([]) + expect((await t.req(rt, { user: 'stranger' })).status).toBe(403) + res = await t.req(rt, { + method: 'PUT', + user: 'stranger', + json: { recordType: 'X', redirectUrlField: 'href' }, + }) + expect(res.status).toBe(403) + }) +}) + +describe('resolve', () => { + it('needs an ARK path', async () => { + expect((await t.req('/api/ark/resolve')).status).toBe(400) + expect((await resolve('nothing here')).status).toBe(400) + expect(await body(await resolve('ark:12345/'))).toEqual({ type: 'not_found' }) + }) + + it('resolves a collection ARK to its page', async () => { + const res = await resolve(`ark:12345/${t.name()}`) + expect(res.status).toBe(200) + const r = await body(res) + expect(r).toMatchObject({ + type: 'redirect', + url: '/org/links', + metadata: { + type: 'collection', + who: 'Org', + what: 'links', + where: `https://underlay.org/ark:12345/${t.name()}`, + naan: '12345', + collectionName: 'links', + ownerName: 'Org', + semver: 'v1.0.0', + arkUrl: `https://underlay.org/ark:12345/${t.name()}`, + }, + }) + expect(r.metadata.when).toMatch(/^\d{8}$/) + // The older "ark:/NAAN/" spelling is the same ARK. + expect((await body(await resolve(`ark:/12345/${t.name()}`))).url).toBe('/org/links') + }) + + it('resolves a version ARK, showing who pushed only to members', async () => { + const anon = await body(await resolve(`ark:12345/${t.name('v1.0.0')}`)) + expect(anon).toMatchObject({ + type: 'redirect', + url: '/org/links/v/1.0.0', + metadata: { type: 'version', what: 'links v1.0.0', semver: 'v1.0.0' }, + }) + expect(anon.metadata).not.toHaveProperty('pushedBy') + expect(anon.metadata).not.toHaveProperty('actorId') + const member = await body(await resolve(`ark:12345/${t.name('v1.0.0')}`, t.user)) + expect(member.metadata).toHaveProperty('pushedBy') + expect(member.metadata).toHaveProperty('actorId') + + expect((await resolve(`ark:12345/${t.name('v9.0.0')}`)).status).toBe(404) + }) + + it('rejects bad check characters and other orgs’ shoulders', async () => { + const name = t.name() + const bad = name.slice(0, -1) + (name.endsWith('b') ? 'c' : 'b') + expect((await resolve(`ark:12345/${bad}`)).status).toBe(404) + + await addOrg2() + const other = await getOrMintShoulder(t.h.ports.db, 'org2') + const arkId = collectionToArkId(t.collectionId) + const res = await resolve(`ark:12345/${other}${arkId}${computeNcdaCheckChar(arkId)}`) + expect(res.status).toBe(404) + }) + + it('resolves record ARKs of configured types, by set', async () => { + // Not configured yet: no record ARKs for the type. + expect((await resolve(`ark:12345/${t.name(undefined, 'Link', 'a')}`)).status).toBe(404) + await t.req(`${base}/ark/record-types`, { + method: 'PUT', + user: t.user, + json: { recordType: 'Link', redirectUrlField: 'url' }, + }) + + const pub = await body(await resolve(`ark:12345/${t.name(undefined, 'Link', 'a')}`)) + expect(pub).toMatchObject({ + type: 'redirect', + url: 'https://example.org/a', + metadata: { + type: 'record', + what: 'Link a in links', + semver: 'v1.0.0', + recordType: 'Link', + recordId: 'a', + schema: Link, + data: { title: 'A', url: 'https://example.org/a' }, + }, + }) + // Versioned record ARKs resolve at that version. + const at = await body(await resolve(`ark:12345/${t.name('v1.0.0', 'Link', 'a')}`)) + expect(at.url).toBe('https://example.org/a') + + // A private-set record resolves for members only. + const priv = `ark:12345/${t.name(undefined, 'Link', 'b')}` + expect(await body(await resolve(priv))).toEqual({ type: 'not_found' }) + expect(await body(await resolve(priv, 'stranger'))).toEqual({ type: 'not_found' }) + expect((await body(await resolve(priv, t.user))).url).toBe('https://example.org/b') + + // Missing records, and fields that aren't http(s) URLs, don't redirect. + expect((await resolve(`ark:12345/${t.name(undefined, 'Link', 'zzz')}`)).status).toBe(404) + for (const id of ['evil', 'none']) { + const res = await resolve(`ark:12345/${t.name(undefined, 'Link', id)}`) + expect(res.status).toBe(404) + expect(await body(res)).toEqual({ type: 'not_found', error: 'No URL found for this record' }) + } + }) + + it('hides private collections from non-members', async () => { + const db = t.h.ports.db + const set = (pub: boolean) => + db + .update(schema.collections) + .set({ public: pub }) + .where(eq(schema.collections.id, t.collectionId)) + await set(false) + try { + expect((await resolve(`ark:12345/${t.name()}`)).status).toBe(404) + expect((await resolve(`ark:12345/${t.name()}`, 'stranger')).status).toBe(404) + expect((await body(await resolve(`ark:12345/${t.name()}`, t.user))).url).toBe('/org/links') + } finally { + await set(true) + } + }) + + it('follows a custom URL and stops when disabled', async () => { + const patch = (json: unknown) => t.req(`${base}/ark`, { method: 'PATCH', user: t.user, json }) + await patch({ customUrl: 'https://custom.example/links' }) + try { + const r = await body(await resolve(`ark:12345/${t.name('v1.0.0')}`)) + expect(r).toMatchObject({ + url: 'https://custom.example/links', + metadata: { type: 'version', semver: 'v1.0.0' }, + }) + expect(r.metadata).not.toHaveProperty('pushedBy') + await patch({ enabled: false }) + expect((await resolve(`ark:12345/${t.name()}`)).status).toBe(404) + expect((await body(await t.req(`${base}/ark`, { user: t.user }))).enabled).toBe(false) + } finally { + await patch({ enabled: true, customUrl: '' }) + } + expect((await body(await resolve(`ark:12345/${t.name()}`))).url).toBe('/org/links') + }) +}) + +describe('/ark: URLs', () => { + it('answers the bare NAAN with its policy', async () => { + for (const p of ['/ark:12345', '/ark:12345/']) { + const res = await t.req(p) + expect(res.status).toBe(200) + expect(res.headers.get('content-type')).toMatch(/^text\/plain/) + expect(await res.text()).toMatch( + /^The Underlay assigns identifiers within the ARK domain 12345/, + ) + } + }) + + it('redirects to the deployment origin', async () => { + const res = await t.req(`/ark:12345/${t.name()}`) + expect(res.status).toBe(302) + expect(res.headers.get('location')).toBe('https://ul.test/org/links') + const v = await t.req(`/ark:12345/${t.name('v1.0.0')}`) + expect(v.headers.get('location')).toBe('https://ul.test/org/links/v/1.0.0') + }) + + it('redirects record ARKs to the record URL, by set', async () => { + const path = `/ark:12345/${t.name(undefined, 'Link', 'a')}` + expect((await t.req(path)).headers.get('location')).toBe('https://example.org/a') + const priv = `/ark:12345/${t.name(undefined, 'Link', 'b')}` + expect((await t.req(priv)).status).toBe(404) + expect((await t.req(priv, { user: t.user })).headers.get('location')).toBe( + 'https://example.org/b', + ) + }) + + it('answers ?info and ?? with an ERC, and ?json with the metadata', async () => { + for (const q of ['?info', '??']) { + const res = await t.req(`/ark:12345/${t.name('v1.0.0')}${q}`) + expect(res.status).toBe(200) + const erc = await res.text() + expect(erc).toMatch(/^erc:\nwho: Org\nwhat: links v1\.0\.0\nwhen: \d{8}\n/) + expect(erc).toContain(`where: https://underlay.org/ark:12345/${t.name('v1.0.0')}`) + expect(erc).toContain('where: https://underlay.org/ark:12345/') + } + const res = await t.req(`/ark:12345/${t.name()}?json`) + expect(res.headers.get('content-type')).toBe('application/json') + expect(await res.json()).toMatchObject({ type: 'collection', collectionName: 'links' }) + }) + + it('is a plain 404 for unknown ARKs', async () => { + for (const p of [ + '/ark:12345/nope', + '/ark:12345/ulb9xxxxx', + `/ark:12345/${t.name()}/Link/%E0%A4%A`, + ]) { + const res = await t.req(p) + expect(res.status).toBe(404) + expect(await res.text()).toBe('ARK not found') + } + }) +}) + +describe('organization NAAN', () => { + it('is set by org owners and admins, and used in ARK URLs', async () => { + const naanPath = '/api/accounts/org/ark' + const patch = (json: unknown, user?: string, scope?: string) => + t.req(naanPath, { + method: 'PATCH', + json, + ...(user ? { user } : {}), + ...(scope ? { scope } : {}), + }) + expect((await patch({ naan: '99999' })).status).toBe(401) + expect((await patch({ naan: '99999' }, 'stranger')).status).toBe(403) + expect((await patch({ naan: '99999' }, t.user, 'read')).status).toBe(403) + expect((await patch({ naan: 'abc' }, t.user)).status).toBe(400) + expect((await patch({ naan: '12345' }, t.user)).status).toBe(409) + expect( + ( + await t.req('/api/accounts/nope/ark', { + method: 'PATCH', + user: t.user, + json: { naan: null }, + }) + ).status, + ).toBe(404) + + expect((await patch({ naan: '99999' }, t.user)).status).toBe(200) + try { + const settings = await body(await t.req(`${base}/ark`, { user: t.user })) + expect(settings.arkUrl).toMatch(/^https:\/\/underlay\.org\/ark:99999\//) + // Resolution answers with the org's NAAN, whichever the ARK was written with. + const r = await body(await resolve(`ark:12345/${t.name()}`)) + expect(r.metadata.naan).toBe('99999') + expect(r.metadata.arkUrl).toBe(`https://underlay.org/ark:99999/${t.name()}`) + // Another org can't claim it. + await addOrg2() + await t.h.ports.db + .insert(schema.member) + .values({ organizationId: 'org2', userId: t.user, role: 'admin' }) + expect( + ( + await t.req('/api/accounts/two/ark', { + method: 'PATCH', + user: t.user, + json: { naan: '99999' }, + }) + ).status, + ).toBe(409) + } finally { + await patch({ naan: null }, t.user) + } + }) +}) diff --git a/packages/server/wrangler.jsonc b/packages/server/wrangler.jsonc index a096494..21ce124 100644 --- a/packages/server/wrangler.jsonc +++ b/packages/server/wrangler.jsonc @@ -19,6 +19,9 @@ "compatibility_date": "2026-09-30", "compatibility_flags": ["nodejs_compat"], "limits": { "cpu_ms": 300000 }, + // The UI's client build (pnpm --filter @underlay/web build). Matching files are + // served before the Worker runs; everything else reaches the Worker. + "assets": { "directory": "../web/dist/client" }, "observability": { "enabled": true }, "env": { "staging": { diff --git a/packages/web/index.html b/packages/web/index.html new file mode 100644 index 0000000..5e354c7 --- /dev/null +++ b/packages/web/index.html @@ -0,0 +1,27 @@ + + + + + + + + + + Underlay + + + + +
+ + + + diff --git a/packages/web/package.json b/packages/web/package.json new file mode 100644 index 0000000..5c9652d --- /dev/null +++ b/packages/web/package.json @@ -0,0 +1,37 @@ +{ + "name": "@underlay/web", + "version": "0.0.0", + "private": true, + "description": "Underlay v2 UI: React Router pages, a client bundle, and renderPage for SSR on Workers and Node.", + "type": "module", + "exports": { + ".": "./dist/server/entry-server.js", + "./package.json": "./package.json" + }, + "scripts": { + "build": "vite build && vite build --ssr", + "typecheck": "tsc --noEmit", + "smoke": "tsx scripts/ssr-smoke.ts" + }, + "dependencies": { + "@better-auth/api-key": "^1.6.11", + "@tanstack/react-query": "^5.101.0", + "better-auth": "^1.6.11", + "marked": "^18.0.3", + "react": "^19.2.6", + "react-dom": "^19.2.6", + "react-router": "^7.15.0" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.3.0", + "@types/node": "^25.6.2", + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^5.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "tailwindcss": "^4.3.0", + "tsx": "^4.21.0", + "typescript": "^6.0.3", + "vite": "^6.4.2" + } +} diff --git a/packages/web/public/favicon.svg b/packages/web/public/favicon.svg new file mode 100644 index 0000000..6c64b24 --- /dev/null +++ b/packages/web/public/favicon.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/packages/web/public/llms.txt b/packages/web/public/llms.txt new file mode 100644 index 0000000..0ac46d0 --- /dev/null +++ b/packages/web/public/llms.txt @@ -0,0 +1,845 @@ +# Underlay - AI Integration Guide + +Underlay is a versioned, content-addressed registry for structured knowledge. +Apps push snapshots of their data; Underlay preserves them and serves them via HTTPS API. +Built by Knowledge Futures (501c3): https://www.knowledgefutures.org + +Base URL: https://underlay.org/api + +--- + +## Authentication + +There are two auth methods: + +1. API Key (for programmatic access): + Header: Authorization: Bearer + Keys are prefixed with "ul_" (e.g. ul_abc123...) when created through the UI. + Keys have scopes: read, write, admin. + Keys can optionally be scoped to specific collections via metadata. + Create keys at https://underlay.org/settings/keys (personal) or /:owner/settings/keys (organization). + Keys are managed via better-auth's apiKey plugin at /api/auth/api-key/*. + +2. Session cookie (for browser use): + Users sign in via KF Auth SSO (OAuth2/PKCE) at https://underlay.org/login. + Accounts are created automatically on first sign-in, along with a default organization. + GET /api/accounts/me returns the current user and their organization memberships. + +All GET requests are public — no auth required to read public data. +All write requests (POST, PATCH, PUT, DELETE) require authentication. +If a Bearer token is provided but invalid, the request is rejected immediately (401). + +## Rate Limits + +All API requests are rate-limited per IP (unauthenticated) or per account (authenticated). + +| Auth status | Limit | +|-------------------|---------------| +| Unauthenticated | 60 req/min | +| Authenticated | 5,000 req/min | + +Rate limit headers are included on every response: +- X-RateLimit-Limit: max requests in the window +- X-RateLimit-Remaining: requests left +- X-RateLimit-Reset: seconds until the window resets + +When exceeded, you'll get a 429 response with a Retry-After header. +To get the higher limit, authenticate with an API key (recommended for any automated access). + +--- + +## Core Concepts + +- Organization: an entity that owns collections. Every user gets a default organization on signup. Identified by :slug. Managed via better-auth's organization plugin at /api/auth/organization/*. +- Collection: a named, versioned body of data owned by an organization. Identified by :owner/:slug. +- Version: an immutable snapshot containing a JSON Schema, records, and file references. Identified by semver (e.g. "v1.0.0"). Major = schema change, Minor = records/files change, Patch = metadata-only change. +- Record: a flat JSON object with { id, type, data }. Records are content-addressed: the SHA-256 hash of the canonical JSON `{"id":...,"type":...,"data":...}` is the record's identity. Records are stored globally and deduplicated — the same record in ten collections is stored once. Records reference other records by id and files by hash. Wire format is JSONL (one record per line). +- File: a binary blob stored by SHA-256 hash, referenced in record data as {"$file": "sha256:"}. +- Schema: a JSON Schema document for a single record type, stored as a global, immutable, content-addressed entity. Each type gets its own schema. Schema changes trigger a major version bump. +- Schema labeling: schemas can be labeled post-hoc with URIs or names (e.g. "schema.org/Person") for cross-collection discovery. + +--- + +## Web URLs (for linking humans to a view) + +The API paths above are for fetching data. When you need to point a person at +something in the browser, use these page URLs. The rule: a version is a path +prefix, a view is a path segment, and omitting the version prefix means "latest". +Semver appears WITHOUT the leading "v" in page URLs (/v/1.2.0), unlike API paths +(/versions/v1.2.0). Both forms resolve, but the bare form is canonical. + +https://underlay.org/:owner → account/organization profile +https://underlay.org/:owner/:slug → collection overview (latest): README, record types, stats +https://underlay.org/:owner/:slug/records → browse records (latest) — add ?type=TypeName to pick a type +https://underlay.org/:owner/:slug/schemas → schemas (latest) +https://underlay.org/:owner/:slug/files → files (latest) +https://underlay.org/:owner/:slug/versions → version history +https://underlay.org/:owner/:slug/versions/compare?from=v1.0.0&to=v1.1.0 → diff two versions +https://underlay.org/:owner/:slug/v/1.2.0 → overview pinned to v1.2.0 +https://underlay.org/:owner/:slug/v/1.2.0/records → records in v1.2.0 (?type=TypeName) +https://underlay.org/:owner/:slug/v/1.2.0/schemas → schemas as pinned in v1.2.0 +https://underlay.org/:owner/:slug/v/1.2.0/files → files in v1.2.0 +https://underlay.org/records/:hash → a record's content-addressed permalink + provenance +https://underlay.org/schemas/:id → a schema's detail page + which collections use it + +Prefer a version-pinned URL when citing data, since /records (no prefix) follows +the latest version and its contents will change on the next push. ARK identifiers +(ark:NAAN/id.v1.2.0) also resolve to these pages and are the most durable option. + +--- + +## Reading Data + +### Browse collections +GET /api/collections → list public collections (?q=search&limit=50&offset=0) +GET /api/collections/:owner/:slug → collection metadata + latest version summary + +### Read versions +GET /api/collections/:owner/:slug/versions → list versions (newest first, ?limit=50&offset=0) +GET /api/collections/:owner/:slug/versions/latest → latest version with full metadata +GET /api/collections/:owner/:slug/versions/:semver → specific version by semver (e.g. /versions/v1.2.0) + +### Read records and files +GET /api/collections/:owner/:slug/versions/:semver/records.ndjson → ALL records, streamed as NDJSON in one request (?type=TypeName&after=recordId) +GET /api/collections/:owner/:slug/versions/:semver/records → records for a version (?type=TypeName&limit=100&after=) +GET /api/collections/:owner/:slug/versions/:semver/manifest → manifest: record ids/types/hashes + file hashes + schema hashes (?since=v1.0.0 for delta) +GET /api/collections/:owner/:slug/versions/:semver/files → list files for a version (hash, size, content type) +GET /api/collections/:owner/:slug/files/:hash → download a file: access-checked, then 302-redirects to a short-lived presigned URL (follow it, e.g. curl -L). Public files are anonymous; private needs a session or a Bearer share/agent token. This API path is the durable locator — the redirect target is ephemeral, never persist it. +HEAD /api/collections/:owner/:slug/files/:hash → check if a file exists/is accessible (returns Content-Length, Content-Type) +POST /api/collections/:owner/:slug/files/presign → presign many files at once: body {"hashes":[...]} → {hash: presignedUrl | null}. One round trip for a page full of files. + +### Records (global, content-addressed) +GET /api/records/:hash/provenance → find all collections/versions containing this record hash +POST /api/records/batch → fetch records by hash: {"hashes": ["abc..."]} → JSONL stream + +### Diff +GET /api/collections/:owner/:slug/versions/:semver/diff?from=:semver → diff between two versions (added, updated, removed records) + Returns full record bodies, keyset-paginated: limit defaults to 500, maxes at 5000, and + ?cursor= walks the rest. When you only need hashes, prefer + .../manifest?since= — its entries are ~120 bytes instead of whole records. + +### Export +GET /api/collections/:owner/:slug/export → download .tar.gz archive (manifest.json + records/*.ndjson + files/*) +GET /api/collections/:owner/:slug/export?version=v2.0.0 → export a specific version + +Archive layout: manifest.json, records/.ndjson per record type, and files/. +Types too large for a single archive entry are split into numbered parts — +records/.0000.ndjson, records/.0001.ndjson, ... — at 25,000 records per part. +A type that fits in one part keeps the unnumbered records/.ndjson name, so archives +of ordinary collections are unchanged. Read every records/*.ndjson entry and you have the +version, whichever form it took. + +The archive streams: the download starts immediately and is produced as you read it, so +pipe it to disk or straight into tar rather than buffering it. Exporting the same version +again gives a byte-identical archive (entries carry the version's creation time), so a +checksum of one export holds for the next. manifest.json is the last entry; its +files_missing lists any file that couldn't be fetched. A failure partway through cuts the +response off instead of finishing it, so an archive that gunzips and untars without error +is complete. Versions over 2,000,000 records return 413 — use records.ndjson for those. + +Export is capped at 2,000,000 records and returns 413 above that — a guard against handing +back a multi-gigabyte tarball from a single GET, not a memory limit. For bulk reads prefer +records.ndjson (below): it streams, resumes, and needs no unpacking. + +The SQL explorer (/api/query/...) has a lower limit of 250,000 records, because unlike +export it genuinely does hold the whole version in memory to build a SQLite copy. It is a +UI feature rather than a documented API; on a large collection use records.ndjson, or Hot, +which hydrates a collection into a queryable database built for the purpose. + +### Fork +POST /api/collections/:owner/:slug/fork → fork collection into caller's org (requires write auth; a collection-scoped key is refused). 403 if you are NOT a member of the source org and the source's latest version holds any private record, private type, or private field — a fork copies the full record bodies, and there is no redacted-fork path. + Body: { "targetOrg": "my-org", "slug": "optional-new-slug" } + Creates a new collection under targetOrg with the source's latest version. + Records, schemas, and files are referenced (not copied) — zero additional storage. + Response includes { id, owner, slug, forkedFrom: { owner, slug, version } }. + +--- + +## Writing Data: The Push Flow + +This is the canonical workflow for syncing an app's data to Underlay: + +### Step 1: Get current state +GET /api/collections/:owner/:slug/versions/latest + +Response includes { semver, hash, recordCount, fileCount }. +If 404, no versions exist yet — your first push should use base_version: null. + +### Step 2: Fetch the manifest (optional, for diffing) +GET /api/collections/:owner/:slug/versions/:semver/manifest?limit=10000 + +Response: +{ + "semver": "v1.2.0", + "hash": "abc123...", + "schemas": {"Article": "schema-hash...", ...}, + "records": [{"id": "rec-1", "type": "Article", "hash": "record-hash..."}, ...], + "files": ["deadbeef...", ...], + "pagination": {"limit": 10000, "hasMore": true, "nextCursor": "eyJhZGRlZCI6..."} +} + +The manifest is paginated: limit defaults to 10000 and maxes at 100000. Pass +?cursor= to continue; repeat until hasMore is false. The cursor is +opaque — pass back exactly what you were given, don't construct or parse it. + +This is by far the cheapest way to learn what a version contains: manifest entries are ~120 +bytes each, so a million records is one request-order-of-magnitude smaller than fetching the +records themselves. Prefer it over walking /records when you only need hashes. + +For delta manifests, add ?since= to get only the changes between two versions: +GET /api/collections/:owner/:slug/versions/:semver/manifest?since=:semver +Response: +{ + ..., + "delta": { + "added": [{"id": "rec-9", "type": "Article", "hash": "..."}], + "updated": [{"id": "rec-1", "type": "Article", "hash": "...", "previousHash": "..."}], + "removed": [{"id": "rec-4", "type": "Article", "hash": "..."}] + }, + "pagination": {"limit": 10000, "hasMore": false, "nextCursor": null}, + "truncated": false +} + +Deltas are keyset-paginated the same way, so a delta of any size can be walked to completion +with ?cursor=. The three lists drain independently, so a page late in the walk may contain +only "updated" entries. "truncated" is legacy: it now just mirrors pagination.hasMore, and +older clients treated it as "give up and rebuild from the full manifest". If you understand +the cursor, page instead of rebuilding. + +Compare this against your local data to determine what changed. + +### Step 3: Upload new files (if any) +For each file your app has that Underlay doesn't: + +PUT /api/collections/:owner/:slug/files/sha256: +Content-Type: application/octet-stream +Body: raw file bytes + +The server verifies the SHA-256 hash matches the body. Existing hashes are idempotent (200 OK). +Check existence first with HEAD if you want to skip uploads. + +### Step 4: Push the version (negotiate protocol) + +All pushes use the negotiate protocol — a three-step flow similar to git's pack negotiation. +For very large collections two of those steps can be broken up (the manifest uploads in chunks, +the commit runs in the background) — see 4a-chunked and 4c-async below; the shape is the same. +The client sends a manifest of record hashes; the server says which it needs; the client sends +only those records; then commits. + +#### Step 4a: Negotiate +POST /api/collections/:owner/:slug/versions/negotiate +Content-Type: application/json +Authorization: Bearer ul_ + +{ + "base_version": "v1.2.0", + "message": "Daily archive 2026-04-27", + "app_id": "my-app", + "actor_id": "my-app:cron-job", + "schemas": { + "Article": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "body": {"type": "string"}, + "publishedAt": {"type": "string", "format": "date-time"}, + "authorId": {"type": "string", "x-ref-type": "Author"} + } + }, + "Author": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string", "private": true} + } + } + }, + "manifest": [ + {"id": "article-42", "type": "Article", "hash": "abc123..."}, + {"id": "article-10", "type": "Article", "hash": "def456..."} + ], + "files": ["7a8b9c..."], + "metadata": { + "description": "Daily archive of publications" + } +} + +Each record hash is SHA-256 of the canonical JSON — see "Record Hashing" section below for +the exact algorithm. Clients MUST canonicalize data (sort object keys recursively) before +hashing, or the server will reject the records. + +Field reference: +- base_version: the semver string of the version you diffed against (e.g. "v1.2.0"). null for first push. Used for optimistic locking. +- schemas: per-type JSON Schema map. Required on every push. +- manifest: array of {id, type, hash, private?} for every record in the new version. Capped at 500,000 entries — above that, upload it in chunks instead (see 4a-chunked below). Omit when using manifest_expected. +- private (per manifest entry): optional boolean. true hides that record from non-owners in THIS version. OMITTING IT MEANS PUBLIC — it is not inherited from the base version and must be re-sent on every push. Applies identically to inline manifests and JSONL manifest chunks. See "Private Records" below. +- manifest_expected: number of distinct record hashes you will upload in chunks. Mutually exclusive with manifest; sending both returns 400. +- files: array of file hashes (SHA-256 hex strings) referenced by records. +- metadata: optional JSON object for version metadata (description, readme, license, etc.). Merged with previous version's metadata. +- message: human-readable commit message (optional). +- app_id: identifier for the pushing application (optional). +- actor_id: identifier for the user or process that triggered the push (optional). +- strip_unknown_fields: if true, records with fields not defined in the schema will have those fields silently stripped. Default: false (returns 422 with extra field list). + +Response: +{ + "session_id": "uuid", + "needed_records": ["def456..."], + "needed_files": [], + "total_records": 2, + "total_files": 1, + "already_have_records": 1, + "already_have_files": 1 +} + +#### Step 4a-chunked: Large collections — upload the manifest in chunks + +The manifest above is one JSON body. That is fine up to 500,000 entries; past that it would be +hundreds of megabytes parsed in one go, so upload it in chunks instead. Omit "manifest" and +declare the count: + +POST /api/collections/:owner/:slug/versions/negotiate +{ + "base_version": null, + "schemas": {...}, + "manifest_expected": 3110000, + "message": "arXiv metadata" +} + +Response — nothing here is proportional to the collection: +{ + "session_id": "uuid", + "manifest_expected": 3110000, + "manifest_received": 0, + "needed_files": [], + "total_files": 0, + "already_have_files": 0, + "next": "POST .../versions/negotiate/uuid/manifest" +} + +Then send the manifest as JSONL, up to 50,000 entries per request: + +POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/manifest +Content-Type: application/x-ndjson +Authorization: Bearer ul_ + +{"id":"article-42","type":"Article","hash":"abc123..."} +{"id":"article-10","type":"Article","hash":"def456..."} + +Response: +{ + "received": 50000, + "needed_records": ["def456..."], + "manifest_received": 150000, + "manifest_expected": 3110000 +} + +Each response tells you which records from THAT chunk the server needs, so you can start +sending record bodies (step 4b) before the whole manifest is uploaded. + +Chunks are idempotent: entries are keyed by hash, so re-sending a chunk after a timeout is +safe and manifest_received will not move. Commit refuses to build a version until +manifest_received equals manifest_expected, so a client that dies partway through cannot +silently produce a version that dropped records. + +#### Step 4b: Send needed records +POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/records +Content-Type: application/x-ndjson +Authorization: Bearer ul_ + +{"id":"article-10","type":"Article","data":{"title":"Updated Title","body":"..."}} + +Each line is one JSON record. Only send records whose hashes appear in needed_records. +Call this endpoint multiple times for large datasets (up to 10,000 records per batch). +If needed_records was empty, skip this step entirely. + +Response: +{ "received": 1, "remaining": 0 } + +When remaining reaches 0, all needed records have been received. + +#### Step 4c: Commit +POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/commit +Authorization: Bearer ul_ + +No request body needed. The server validates all records against schemas, computes +version hashes, and creates the new immutable version. + +Response (201): +{ "semver": "v1.3.0", "hash": "def456...", "recordCount": 2, "fileCount": 1 } + +#### Step 4c-async: Large collections — commit in the background + +Commit work is proportional to collection size, so on a very large collection it can run for +minutes — longer than a proxy or client will hold a request open. Add ?async=true (or send +{"async": true}) and the server accepts the commit and builds the version in the background: + +POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/commit?async=true + +Response (202): +{ + "session_id": "uuid", + "status": "committing", + "message": "Commit accepted. Poll GET .../versions/negotiate/uuid until status is \"committed\" or \"failed\"." +} + +Then poll GET .../versions/negotiate/:sessionId until status is "committed" or "failed": + +{ + "session_id": "uuid", + "status": "committed", + "finalize_started_at": "2026-07-31T12:00:00.000Z", + "result": {"semver": "v1.3.0", "hash": "private:def456...", "recordCount": 3110000, "fileCount": 0}, + "error": null +} + +On success "result" holds exactly what the synchronous 201 would have returned. On failure +"status" is "failed" and "error" holds the rejection body the synchronous path would have +returned (same statusCode and shape), so the two paths are interchangeable apart from timing. + +The version is invisible to readers until the finalize completes — there is no window in which +a half-built version can be read. The finalize is server-side work and does not depend on your +connection staying open: a client that disconnects while polling can reconnect and read the +result. A finalize whose server process dies is swept and marked "failed". + +### Step 5: Handle errors + +Conflict (409 — someone pushed while you were diffing): +{ "error": "Version conflict", "currentVersion": "v1.3.0", "statusCode": 409 } +→ Re-negotiate with the new base_version. + +Missing records (400 — commit called before all records submitted): +{ "error": "Missing records", "missing_hashes": ["def456..."], "statusCode": 400 } +→ Send the remaining records via the /records endpoint, then retry commit. + +Missing files (422 — records reference files not yet uploaded): +{ "error": "Missing files", "filesNeeded": ["sha256:abc..."], "statusCode": 422 } +→ Upload the listed files, then retry commit. + +Extra fields (422 — records contain fields not in the schema): +{ "error": "Records contain fields not defined in schema", "extraFields": [...], "totalRecords": 12, "statusCode": 422 } +→ Either fix the records, or re-negotiate with "strip_unknown_fields": true. + extraFields lists at most the first 100; totalRecords is how many were affected. + Schema validation failures (422) are reported the same way, with "totalErrors". + +Manifest incomplete (400 — chunked upload, commit called before every chunk arrived): +{ "error": "Manifest incomplete", "manifest_expected": 3110000, "manifest_received": 3050000, "statusCode": 400 } +→ Upload the remaining chunks, then retry commit. + +Manifest too large (413 — inline manifest over 500,000 entries): +{ "error": "Inline manifests are limited to 500000 entries...", "statusCode": 413 } +→ Re-negotiate with manifest_expected and upload the manifest in chunks. + +Sessions expire after 10 minutes of INACTIVITY. Every manifest chunk and record batch pushes +the expiry back, so a push that legitimately runs for an hour will not expire underneath you. +If a session does expire, re-negotiate. + +### Session management +GET /api/collections/:owner/:slug/versions/negotiate/:sessionId → check session status +DELETE /api/collections/:owner/:slug/versions/negotiate/:sessionId → cancel session (204) + +Session status is one of: +- open — accepting manifest chunks and records +- committing — async commit accepted, finalize running in the background +- committed — done; "result" holds the version +- failed — finalize rejected or died; "error" holds why +- expired — timed out or cancelled + +### First push (no existing versions) +Set base_version to null. Include all records in the manifest. Include schemas for all types. +The first version will be v1.0.0. + +--- + +## Pagination (Records Endpoint) + +The records endpoint uses cursor-based pagination for efficient traversal of large collections. + +GET /api/collections/:owner/:slug/versions/:semver/records?limit=100&after= + +Response: +{ + "records": [ ...up to `limit` records... ], + "pagination": { + "limit": 100, + "hasMore": true, + "nextCursor": "eyJyIjpbInB1Yi0wMDIiLCJkZWY0NTYiXX0", + "total": 2000000 + } +} + +Parameters: +- limit: max records per page (default 100, max 2000) +- after: opaque keyset cursor — pass pagination.nextCursor back unchanged. Records are + ordered by (record id, record hash), so records sharing an id across types are never + skipped at a page boundary. This is the canonical, scalable method: it is an index seek + and stays fast at any depth. `cursor` is accepted as an alias for `after`. A bare record + ID is still accepted (records with IDs strictly after it), but it skips other records + that share the last ID. +- offset: legacy offset-based pagination. Cost grows with the offset, so it is capped: + an offset greater than 10000 returns 400. Use `after` to page deeper. +- type: filter by record type + +Notes: +- `pagination.total` respects the `type` filter and excludes private types. On collections + that mark individual records private it is an upper bound for anonymous callers, since + those records are hidden but still counted — use `hasMore` for an exact end-of-set signal. +- A query that exceeds the server's statement timeout returns 503 with a Retry-After + header. If you hit this on `offset`, switch to `after`. + +To paginate through all records (works at any collection size): +1. First request: GET .../records?limit=2000 +2. If pagination.hasMore is true, use pagination.nextCursor for the next request: + GET .../records?limit=2000&after= +3. Repeat until hasMore is false. + +BUT: if you want the whole collection, do not page it. Use the NDJSON stream below — +paging costs one round trip per page purely to re-establish a cursor the server just had. +Paging is for browsing a slice; streaming is for reading everything. + +Ask for the largest page you can handle. Walking a whole collection by paging is bounded by +request count, not bytes — 60 requests/minute unauthenticated, 5,000 authenticated — so a +3-million-record collection is 6,200 requests at 500/page and 1,550 at 2,000/page. +Authenticate for any full-collection walk. + +Do NOT paginate large collections with ?offset=; it is capped at 10000 and will 400 +beyond that. Use ?after= keyset pagination instead. + +--- + +## Bulk Read (the fast way to get a whole collection) + +GET /api/collections/:owner/:slug/versions/:semver/records.ndjson + +Streams every record in the version as newline-delimited JSON in a single response. One +request regardless of size: a 3.1M-record collection is one call here versus 1,556 paged +ones. The server reads through a database cursor and writes as it goes, so memory is +constant on both ends — you can start processing the first line before the last is sent. + +Content-Type: application/x-ndjson +X-Underlay-Record-Count: 3113504 ← how many lines to expect + +One JSON object per line: +{"id":"arxiv:0704.0001","type":"Preprint","data":{...},"hash":"abc123..."} + +Parameters: +- type: restrict to one record type +- after: resume — return records with ids strictly after this value + +Guarantees you can rely on: +- Records are ordered by id, ascending. This is what makes `after` work. +- `hash` is the same content-address the records endpoint serves: the full record hash for + owners, the public hash for everyone else. +- Privacy filtering is identical to /records — private types and private records are + absent, private fields are stripped. + +Verify completeness yourself. A stream that dies halfway cannot report an error: the 200 +status and headers were already sent. Count the lines and compare against +X-Underlay-Record-Count. If they differ, resume with +?after= rather than starting over. + +X-Underlay-Record-Count is the count for THIS request — privacy-filtered for your access +level and scoped to ?type= if you passed one — so the comparison is exact for every caller. +Do NOT compare against the version's `recordCount`: that is the full total and includes +private records and private types you may not be receiving. + +That resume behaviour makes this strictly better than paging for bulk reads: the same +recovery from a dropped connection, at a fraction of the requests. + +One edge: a record id is not guaranteed unique within a version — the same id can appear +under more than one hash. `after` resumes strictly past the id, so if a stream broke +between two lines sharing an id, resuming from it skips the second. Rare, and identical to +how ?after= behaves on the paged endpoint, but if you need exactness compare the final line +count against X-Underlay-Record-Count and re-read from an earlier id if it falls short. + +### Compression + +All /api/ responses are compressed when you send Accept-Encoding: gzip — about 3x on +record data. Most HTTP clients do this automatically. It applies to the NDJSON stream too. + +### When to use which + +- Whole collection, one pass → records.ndjson +- A page, or browsing → records?limit=2000&after= +- Only ids/types/hashes → manifest (≈120 bytes/record, far smaller than bodies) +- What changed since a version → manifest?since= (delta) +- Specific records you know hashes for → POST /api/records/batch (up to 10,000 per call) +- An archive to keep → export (.tar.gz) + +--- + +## Record Format + +{ + "id": "unique-stable-id", + "type": "TypeName", + "data": { + "title": "Some value", + "authorId": "author-123", + "attachment": {"$file": "sha256:abc123def456..."} + } +} + +- id: stable, unique within the collection. Use your app's primary key or generate a deterministic one. +- type: groups records by kind (e.g. "Article", "Author", "Grant"). +- data: flat JSON object. Reference other records by id. Reference files with {"$file": "sha256:"}. + +Records are flat — no nested joins. Relationships are expressed by storing the id of the related record. +The schema declares which fields are references, so tools can resolve them at read time. + +--- + +## Record Hashing (required for push) + +Records are content-addressed. The hash is the record's identity across the system — it determines +deduplication, manifest membership, and version integrity. Any client that pushes data must compute +hashes exactly as the server does, or the push will fail with "Unexpected record hash". + +### Algorithm + +1. Build the canonical object: `{"id": , "type": , "data": }` + - The top-level keys MUST appear in this exact order: id, type, data. + - The `data` value MUST be canonicalized (see below). + - The `private` flag is NOT part of the hash. Two records with identical id/type/data + but different privacy flags produce the same hash. + +2. Serialize to JSON with no extra whitespace (standard JSON.stringify behavior). + +3. Compute SHA-256 over the UTF-8 bytes of that JSON string. + +4. Encode as lowercase hex (64 characters). + +### Canonicalization + +Canonicalization recursively sorts all object keys alphabetically. This ensures that +`{"b":1,"a":2}` and `{"a":2,"b":1}` produce the same hash. + +Rules: +- Objects: sort keys lexicographically (by Unicode code point), recurse into values. +- Arrays: preserve order, recurse into elements. +- Primitives (strings, numbers, booleans, null): unchanged. + +Reference implementation (JavaScript): + +```javascript +function canonicalize(value) { + if (value === null || typeof value !== 'object') return value; + if (Array.isArray(value)) return value.map(canonicalize); + const sorted = {}; + for (const key of Object.keys(value).sort()) { + sorted[key] = canonicalize(value[key]); + } + return sorted; +} + +function hashRecord(record) { + const canonical = JSON.stringify({ + id: record.id, + type: record.type, + data: canonicalize(record.data), + }); + // Node.js: + // const hash = createHash('sha256').update(canonical).digest('hex'); + // Browser: + // const buf = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(canonical)); + // const hash = Array.from(new Uint8Array(buf)).map(b => b.toString(16).padStart(2, '0')).join(''); + return { hash, canonical }; +} +``` + +### Example + +Input record: + { "id": "article-1", "type": "Article", "data": { "title": "Hello", "body": "World" } } + +Canonical JSON (data keys already sorted): + {"id":"article-1","type":"Article","data":{"body":"World","title":"Hello"}} + +SHA-256 hash: + e3b7a... (64 hex characters) + +Note: if data keys were in a different order (e.g. `{"title":"Hello","body":"World"}`), +canonicalization sorts them to `{"body":"World","title":"Hello"}` — producing the same hash. + +### Schema hashing + +Schemas are also content-addressed. The hash is SHA-256 of `JSON.stringify(canonicalize(schemaBody))`. +The same canonicalization rules apply. + +--- + +## Schema Discovery + +Schemas are globally deduplicated by content hash. If two collections use the same type shape, they share the same schema row. This enables cross-collection discovery. + +GET /api/schemas → search schemas (?q=text&slug=TypeName&label=uri&schema_hash=sha256:...&limit=50&offset=0) +GET /api/schemas/:id → single schema with labels and usage info +GET /api/collections/:owner/:slug/schemas → schemas for latest version (?version=v1.2.0 for specific, ?raw=true to skip label enrichment) +POST /api/schemas/:id/labels → add label {"label": "schema.org/Person"} (requires write scope) +DELETE /api/schemas/:id/labels/:label → remove label (requires admin scope) + +When schemas are returned via the collection schemas endpoint, known labels are injected as "x-underlay-labels" on the schema body (opt-out with ?raw=true). + +--- + +## Versioning + +- Versions are identified solely by semver (e.g. "v1.0.0", "v1.2.3"). +- Semver semantics: major bump = schema change (types added/removed or schema_id changed), minor bump = records/files change, patch bump = metadata-only change. +- Each version has a content-addressed hash computed from sorted schema hashes + sorted record hashes + sorted file hashes + metadata. +- Each version has a `metadata` field: a JSON object that can contain `readme`, `license`, and other arbitrary metadata. +- Records are content-addressed: SHA-256 of canonical JSON (see "Record Hashing" section). Records are globally deduplicated. +- Versions are immutable once created. +- The provenance endpoint (GET /api/records/:hash/provenance) shows every collection and version that includes a given record. + +--- + +## Organization Management + +Organizations are managed via better-auth's organization plugin at /api/auth/organization/*. +Every user gets a default organization on signup. Users can create additional organizations. + +POST /api/auth/organization/create → create org {"name", "slug"} +GET /api/auth/organization/list → list user's organizations +PATCH /api/auth/organization/update → update org +DELETE /api/auth/organization/delete → delete org + +Member management (invite, remove, update roles) is also under /api/auth/organization/*. + +## Collection Management + +POST /api/accounts/:owner/collections → create collection {"slug", "name", "public"} +PATCH /api/collections/:owner/:slug → update {"name", "slug", "public"} +PATCH /api/collections/:owner/:slug/metadata → update version metadata {"description", "readme", "license", ...} — creates a patch version (requires write scope) +DELETE /api/collections/:owner/:slug → delete collection (requires admin scope) +GET /api/accounts/:owner/collections → list collections for an organization + +--- + +## Privacy & Visibility + +Underlay supports privacy at four levels that compose — a reader sees content only if it passes all of them: + 1. Collection (collections.public) — a private collection 404s entirely; nothing below is evaluated. + 2. Type — "private": true on the type's schema. Per-version. + 3. Field — "private": true on the property. Per-version. + 4. Record — "private": true on the negotiate MANIFEST ENTRY. Per-version. +Levels 2-4 are bound to the version, so the same content can be public in one collection's version +and hidden in another's. Private data is stored alongside public data in the same version and is +visible only to members of the owning organization. + +### Private Types +Mark an entire type as private in its schema. All records of that type are hidden from public readers. + +"schemas": { + "Article": { + "type": "object", + "properties": { "title": {"type": "string"} } + }, + "InternalNote": { + "type": "object", + "private": true, + "properties": { "note": {"type": "string"}, "articleId": {"type": "string"} } + } +} + +Public readers see only the Article type. InternalNote is completely hidden (including from the schema response). + +### Private Fields +Mark individual fields as private within a type's schema. The type itself is visible, but those fields are stripped for public readers. + +"Author": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string", "private": true}, + "phone": {"type": "string", "private": true} + } +} + +Public readers see Author records with only "name". The owner sees all fields. + +### Private Records +Mark individual records as private on their MANIFEST ENTRY in the negotiate body. The type and +schema stay visible; that specific record is hidden from non-owners. + +"manifest": [ + {"id": "article-1", "type": "Article", "hash": ""}, + {"id": "article-2", "type": "Article", "hash": "", "private": true} +] + +article-2 is only visible to members of the owning org. Public readers see article-1 only. + +Two rules you must design around: + +1. PRIVACY IS PER-VERSION AND MUST BE RE-DECLARED ON EVERY PUSH. The flag is stored on the + (version, record) edge, not on the record itself. Omitting `private` on a manifest entry + means PUBLIC — it does NOT inherit the previous version's value. A push that sends a full + manifest without the flags publishes everything it omits. (Dropping the flag is therefore + also how you deliberately un-hide a record.) Read the current flags back from + GET .../versions/:semver/manifest, which echoes `private: true` on the entries that have it. +2. REDACTION IS FORWARD-ONLY. Marking a record private in v2 hides it in v2 only. Versions are + immutable, so v1 still serves that record at /versions/v1.0.0/records. There is no + retroactive purge. Files are looser still: file access resolves across ALL ready versions, + so a file referenced publicly in v1 stays downloadable after the referencing record is + redacted in v2. + +`private` belongs ONLY on the manifest entry. A `private` key on a record BODY sent in step 4b is +parsed and silently ignored — no error, and the record ships public. + +The same content can be private in one collection and public in another: the flag lives on the +version edge, not on the globally deduplicated record body. + +### How it works +- Public hash: the digest of the public projection — private types omitted, private records + omitted, private fields stripped from the records that remain. Version metadata and the file + list are NOT filtered: both digests are computed over the same metadata and the same files. +- Version identity is BOTH digests. A push is a duplicate (409 "No changes detected") only if + `hash` AND `public_hash` match an existing version. Privacy is deliberately not folded into + `hash`, so `hash` stays independently verifiable from the content; privacy moves `public_hash` + instead. That is why re-pushing byte-identical content with a record newly marked private is a + legitimate new version — it is what makes redaction-in-place possible. Converse: flagging a + record private whose TYPE is already private changes neither digest and is correctly rejected + as a duplicate, because the flag has no observable effect. +- A privacy-only push is a PATCH bump (schema change → major, record-set change → minor, + everything else → patch). +- Public record hash: a record of a type with private fields is listed in public manifests under + the hash of its filtered projection ({"id", "type", "data"} with private fields stripped). + Record endpoints resolve either address; hashing the document you receive always reproduces + the address you requested. +- Private hash: the full digest over ALL schema and record hashes, plus files and metadata. + Served only to org members; everyone else receives public_hash in the `hash` field. Both are + prefixed — "private:<64hex>" and "public:<64hex>" — so the two forms are never confusable. +- Schema filtering: the schema returned to public readers omits private types and private fields +- Record filtering: queries by non-owners automatically exclude private records and strip private fields + +--- + +## API Key Management + +API keys are managed via better-auth's apiKey plugin. All endpoints are under /api/auth/api-key/*. + +POST /api/auth/api-key/create → create key {"name": "my-app", "metadata": {"scope": "write"}, "prefix": "ul"} + The scope in metadata is translated to permissions server-side. + Response includes the key once: {"key": "ul_abc123...", "id": "..."} +GET /api/auth/api-key/list → list keys (id, name, start, permissions, metadata, createdAt, expiresAt) +POST /api/auth/api-key/delete → revoke a key {"keyId": "..."} + +--- + +## Error Codes + +400 — Validation error (bad request body) +401 — Authentication required +403 — Insufficient scope (e.g. read key used for write) +404 — Not found (or private collection you can't access) +409 — Version conflict (re-fetch and retry with new base_version) +413 — Payload too large (file upload exceeds size limit) +422 — Missing files, schema validation failed, or records contain extra fields not in the schema +429 — Rate limited (wait and retry) + +--- + +Full documentation: https://underlay.org/docs +Protocol specification: https://underlay.org/protocol +Integration guide: https://underlay.org/docs/integration +Source code: https://github.com/knowledgefutures/underlay diff --git a/packages/web/public/logoLight.svg b/packages/web/public/logoLight.svg new file mode 100644 index 0000000..6c64b24 --- /dev/null +++ b/packages/web/public/logoLight.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/packages/web/scripts/ssr-smoke.ts b/packages/web/scripts/ssr-smoke.ts new file mode 100644 index 0000000..d296cdc --- /dev/null +++ b/packages/web/scripts/ssr-smoke.ts @@ -0,0 +1,260 @@ +/** + * SSR smoke test: the built renderPage (dist/server/entry-server.js) behind the + * v2 app on Node, with a temp SQLite file and an in-memory bucket, seeded with an + * org, a public and a private collection, and a version pushed through the push + * API. Renders pages through app.fetch, so loaders reach the API in-process the + * way they do in production. + * + * pnpm --filter @underlay/web build && pnpm --filter @underlay/web smoke + * + * The server's ports are imported from its sources (as packages/server/test/harness.ts + * builds them) because @underlay/server doesn't export them. + */ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { newSalt } from '../../core/src/index.ts' +import { MemoryBlobStore } from '../../repo/src/blob/memory.ts' +import { ed25519Signer, generateSigningKey } from '../../repo/src/index.ts' +import '../../server/src/handlers.ts' +import { createApp } from '../../server/src/app.ts' +import { MemoryCache } from '../../server/src/cache.ts' +import { openNodeDb } from '../../server/src/db/node.ts' +import * as schema from '../../server/src/db/schema.ts' +import { drainSqliteJobs, SqliteJobs } from '../../server/src/jobs.ts' +import type { Ports } from '../../server/src/ports.ts' +import { createStores } from '../../server/src/stores.ts' +// @ts-ignore: the built bundle has no declarations; its source is src/entry-server.tsx. +import { renderPage } from '../dist/server/entry-server.js' + +const dir = await mkdtemp(join(tmpdir(), 'ul-web-smoke-')) +let failures = 0 + +try { + const db = await openNodeDb(`file:${join(dir, 'db.sqlite')}`) + const cache = new MemoryCache() + const signer = await ed25519Signer(await generateSigningKey()) + const ports: Ports = { + db, + stores: createStores(db, cache, { + bucket: new MemoryBlobStore(), + repoPrefix: 'repo', + internalPrefix: 'internal', + }), + cache, + signer: async () => signer, + jobs: new SqliteJobs(db), + waitUntil: (p) => void p.catch((err) => console.error(err)), + outboundFetch: async () => new Response('ok'), + } + const app = createApp(() => ({ + ports, + config: { appUrl: 'http://smoke.test', deployment: 'smoke' }, + // A session cookie naming the user, else anonymous. A cookie (not the server + // tests' x-test-user header) because SSR loaders forward the page's Cookie. + authenticate: async (req) => { + const user = /(?:^|;\s*)test-user=([^;]+)/.exec(req.headers.get('cookie') ?? '')?.[1] + return user ? { userId: user, scope: 'session', collectionIds: null } : null + }, + renderPage, + })) + + // --- Seed: org "org" with member u1, a public and a private collection. + await db.insert(schema.organization).values({ id: 'org1', name: 'Smoke Org', slug: 'org' }) + await db.insert(schema.user).values({ id: 'u1', name: 'Ada', email: 'u1@example.org' }) + await db.insert(schema.member).values({ organizationId: 'org1', userId: 'u1', role: 'owner' }) + for (const [slug, isPublic] of [ + ['authors', true], + ['secret', false], + ] as const) { + const [c] = await db + .insert(schema.collections) + .values({ + organizationId: 'org1', + slug, + name: slug, + public: isPublic, + privateSalt: newSalt(), + }) + .returning() + await db.insert(schema.placements).values({ + collectionId: c!.id, + locationId: schema.PLATFORM_LOCATION_ID, + role: 'primary', + sets: 'public+private', + }) + } + + const call = (path: string, init: RequestInit = {}, user?: string) => { + const headers = new Headers(init.headers) + if (user) headers.set('cookie', `test-user=${user}`) + return app.fetch(new Request(`http://smoke.test${path}`, { ...init, headers })) + } + + // --- Push v1.0.0 through the push API. + const base = '/api/collections/org/authors' + const Author = { + type: 'object', + properties: { name: { type: 'string' }, born: { type: 'integer' } }, + required: ['name'], + } + let res = await call( + `${base}/push`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + schemas: { Author }, + message: 'First load of authors', + metadata: { readme: '# Authors\n\nPeople who wrote things. ' }, + }), + }, + 'u1', + ) + if (res.status !== 200) throw new Error(`push open: ${res.status} ${await res.text()}`) + const { session_id: sid } = (await res.json()) as { session_id: string } + res = await call( + `${base}/push/${sid}/records`, + { + method: 'POST', + headers: { 'content-type': 'application/x-ndjson' }, + body: [ + { id: 'ada', type: 'Author', data: { name: 'Ada Lovelace', born: 1815 } }, + { id: 'alan', type: 'Author', data: { name: 'Alan Turing', born: 1912 } }, + { id: 'kurt', type: 'Author', data: { name: 'Kurt Gödel' }, private: true }, + ] + .map((r) => JSON.stringify(r)) + .join('\n'), + }, + 'u1', + ) + if (res.status !== 200) throw new Error(`push records: ${res.status} ${await res.text()}`) + res = await call(`${base}/push/${sid}/commit`, { method: 'POST' }, 'u1') + if (res.status !== 201) throw new Error(`push commit: ${res.status} ${await res.text()}`) + const version = (await res.json()) as { semver: string } + // Post-publish jobs (the reference log behind provenance) run from the jobs table. + await drainSqliteJobs(ports) + console.log(`seeded org/authors ${version.semver}\n`) + + // --- Pages. + // A record hash and a schema id, for the provenance and schema pages. + const page = (await (await call(`${base}/versions/latest/records?type=Author`)).json()) as { + records: { id: string; hash: string }[] + } + const adaHash = page.records.find((r) => r.id === 'ada')!.hash + const schemas = (await (await call(`${base}/schemas`)).json()) as { + schemas?: { schemaId: string }[] + } + const schemaId = schemas.schemas?.[0]?.schemaId ?? '1' + + interface Check { + path: string + status: number + has?: string[] + lacks?: string[] + location?: string + user?: string + } + const checks: Check[] = [ + { + path: '/', + status: 200, + has: [ + 'A protocol for radically accessible structured knowledge', + 'authors', + 'Underlay', + ], + }, + { + path: '/explore', + status: 200, + has: ['Browse public knowledge collections', '>authors<', 'Smoke Org', 'v1.0.0'], + lacks: ['secret'], + }, + { + path: '/org/authors', + status: 200, + has: [ + 'org/authors · Underlay', + 'v1.0.0', + 'People who wrote things.', + '<script>alert(1)</script>', + 'Author', + '2 records', + 'ulv2:', + ], + lacks: [''], + }, + // Members see the private set too. + { path: '/org/authors', status: 200, has: ['3 records'], user: 'u1' }, + { + path: '/org/authors/records', + status: 200, + has: ['Records — org/authors', 'Ada Lovelace', 'Alan Turing', '1815'], + lacks: ['Kurt'], + }, + { + path: '/org/authors/records?type=Author&page=1', + status: 200, + has: ['Ada Lovelace', 'Kurt Gödel'], + user: 'u1', + }, + { + path: '/org/authors/versions', + status: 200, + has: ['Versions — org/authors', 'v1.0.0', 'First load of authors', 'ulv2:'], + }, + { path: '/org/authors/v/1.0.0/records', status: 200, has: ['Ada Lovelace'] }, + { path: '/org/authors/v/v1.0.0', status: 302, location: '/org/authors/v/1.0.0' }, + { path: '/org/authors/schemas', status: 200, has: ['Author', 'born', 'integer'] }, + { path: '/org/authors/files', status: 200, has: ['Files — org/authors'] }, + { path: '/org', status: 200, has: ['Smoke Org', 'authors'], lacks: ['secret'] }, + { path: `/records/${adaHash}`, status: 200, has: ['Ada Lovelace', 'org/authors'] }, + { path: '/records/abc123', status: 404 }, + { path: `/schemas/${schemaId}`, status: 200, has: ['born', 'org/authors'] }, + { path: '/schemas', status: 200, has: ['Schemas'] }, + { path: '/protocol', status: 200, has: ['Protocol'] }, + { path: '/docs', status: 200 }, + { path: '/org/secret', status: 404 }, + { path: '/org/secret', status: 200, has: ['secret'], user: 'u1' }, + { path: '/org/nope', status: 404 }, + { path: '/no/such/page/here', status: 404 }, + { path: '/signup', status: 302, location: '/login' }, + { path: '/dashboard', status: 302, location: '/login' }, + { path: '/dashboard', status: 200, has: ['authors', 'secret'], user: 'u1' }, + ] + + for (const c of checks) { + const r = await call(c.path, {}, c.user) + // React separates adjacent text nodes with ; match the visible text. + const body = (await r.text()).replaceAll('', '') + const problems: string[] = [] + if (r.status !== c.status) problems.push(`status ${r.status}, expected ${c.status}`) + if (c.location && r.headers.get('location') !== c.location) { + problems.push(`location ${r.headers.get('location')}, expected ${c.location}`) + } + if (c.status === 200) { + if (!r.headers.get('content-type')?.startsWith('text/html')) problems.push('not HTML') + // Every rendered page hydrates: the router data and the built client entry. + for (const s of ['window.__staticRouterHydrationData=', '/assets/entry-client-']) { + if (!body.includes(s)) problems.push(`missing ${JSON.stringify(s)}`) + } + } + for (const s of c.has ?? []) + if (!body.includes(s)) problems.push(`missing ${JSON.stringify(s)}`) + for (const s of c.lacks ?? []) if (body.includes(s)) problems.push(`has ${JSON.stringify(s)}`) + const label = `${c.path}${c.user ? ` (as ${c.user})` : ''}` + if (problems.length) { + failures++ + console.log(`FAIL ${label}\n ${problems.join('\n ')}`) + } else { + console.log(`ok ${label} ${r.status} ${body.length} bytes`) + } + } +} finally { + await rm(dir, { recursive: true, force: true }) +} + +console.log(failures ? `\n${failures} failed` : '\nall passed') +process.exit(failures ? 1 : 0) diff --git a/packages/web/src/App.tsx b/packages/web/src/App.tsx new file mode 100644 index 0000000..0da8de9 --- /dev/null +++ b/packages/web/src/App.tsx @@ -0,0 +1,43 @@ +import type { LoaderFunctionArgs, RouteObject } from 'react-router' + +import { RouteErrorBoundary } from '~/components/NotFound' +import Root from '~/components/Root' +import { apiFetch, fetchBase, ssrHeaders } from '~/lib/fetch-base' +import { buildDataRoutes } from '~/route-gen' + +const components = import.meta.glob<{ default: React.ComponentType }>('./routes/**/[!_]*.tsx') +const dataModules = import.meta.glob<{ + loader?: RouteObject['loader'] + handle?: unknown + middleware?: RouteObject['middleware'] + shouldRevalidate?: RouteObject['shouldRevalidate'] +}>('./routes/**/*.data.ts', { eager: true }) + +async function rootLoader({ request }: LoaderFunctionArgs) { + const res = await apiFetch(`${fetchBase(request.url)}/api/context`, { + headers: ssrHeaders(request), + }) + if (!res.ok) return { currentUser: null, kfAccountUrl: '', kfAuthUrl: '' } + return res.json() +} + +const NotFound = () => import('~/routes/404').then((m) => ({ Component: m.default })) + +// Unmatched paths render the 404 page with a 404 status (the static handler +// reports 200 for a matched splat route unless its loader says otherwise). +function notFoundLoader(): never { + throw new Response('Not Found', { status: 404 }) +} + +export const routes: RouteObject[] = [ + { + id: 'root', + Component: Root, + ErrorBoundary: RouteErrorBoundary, + loader: rootLoader, + children: [ + ...buildDataRoutes(components, dataModules), + { path: '*', loader: notFoundLoader, lazy: NotFound }, + ], + }, +] diff --git a/packages/web/src/components/ApiPlayground.tsx b/packages/web/src/components/ApiPlayground.tsx new file mode 100644 index 0000000..a5c2fc6 --- /dev/null +++ b/packages/web/src/components/ApiPlayground.tsx @@ -0,0 +1,299 @@ +import { useCallback, useState } from 'react' + +import { Button } from '~/components/ui' + +interface Collection { + id: string + slug: string +} + +interface ApiPlaygroundProps { + slug: string + collections: Collection[] +} + +interface ResponseState { + status: number + statusText: string + time: number + body: string +} + +interface Endpoint { + label: string + method: string + path: string + body: string + description: string +} + +function getEndpoints(slug: string, collectionSlug: string): Endpoint[] { + return [ + { + label: 'List collections', + method: 'GET', + path: `/api/accounts/${slug}/collections`, + body: '', + description: 'Returns all collections for this account.', + }, + { + label: 'Get account profile', + method: 'GET', + path: `/api/accounts/${slug}`, + body: '', + description: 'Returns public profile information.', + }, + ...(collectionSlug + ? [ + { + label: 'Get collection', + method: 'GET', + path: `/api/collections/${slug}/${collectionSlug}`, + body: '', + description: 'Returns collection metadata and latest version info.', + }, + { + label: 'List versions', + method: 'GET', + path: `/api/collections/${slug}/${collectionSlug}/versions`, + body: '', + description: 'Returns all versions for this collection.', + }, + { + label: 'Get latest version', + method: 'GET', + path: `/api/collections/${slug}/${collectionSlug}/versions/latest`, + body: '', + description: 'Returns the latest version with records and files.', + }, + { + label: 'List files', + method: 'GET', + path: `/api/collections/${slug}/${collectionSlug}/files`, + body: '', + description: 'Returns all files in the latest version.', + }, + ] + : []), + ] +} + +export function ApiPlayground({ slug, collections }: ApiPlaygroundProps) { + const [selectedCollection, setSelectedCollection] = useState(collections[0]?.slug ?? '') + const [selectedEndpoint, setSelectedEndpoint] = useState(0) + const [response, setResponse] = useState(null) + const [loading, setLoading] = useState(false) + const [copied, setCopied] = useState(false) + const [token, setToken] = useState('') + + const endpoints = getEndpoints(slug, selectedCollection) + const current = endpoints[selectedEndpoint] ?? endpoints[0] + + const sendRequest = useCallback(async () => { + if (!current) return + setLoading(true) + setResponse(null) + + const start = performance.now() + try { + const headers: Record = { 'Content-Type': 'application/json' } + if (token.trim()) { + headers['Authorization'] = `Bearer ${token.trim()}` + } + const opts: RequestInit = { + method: current.method, + headers, + credentials: token.trim() ? 'omit' : 'include', + } + if (current.body && current.method !== 'GET') { + opts.body = current.body + } + const res = await fetch(current.path, opts) + const elapsed = Math.round(performance.now() - start) + let body: string + const contentType = res.headers.get('content-type') ?? '' + if (contentType.includes('json')) { + const json = await res.json() + body = JSON.stringify(json, null, 2) + } else { + body = await res.text() + } + setResponse({ status: res.status, statusText: res.statusText, time: elapsed, body }) + } catch (err: any) { + setResponse({ status: 0, statusText: 'Network Error', time: 0, body: err.message }) + } finally { + setLoading(false) + } + }, [current, token]) + + const copyAsCurl = useCallback(() => { + if (!current) return + const keyValue = token.trim() || '' + let cmd = `curl -X ${current.method} '${window.location.origin}${current.path}'` + cmd += ` \\\n -H 'Authorization: Bearer ${keyValue}'` + if (current.body && current.method !== 'GET') { + cmd += ` \\\n -H 'Content-Type: application/json'` + cmd += ` \\\n -d '${current.body}'` + } + navigator.clipboard.writeText(cmd) + setCopied(true) + setTimeout(() => setCopied(false), 2000) + }, [current, token]) + + return ( +
+ {/* Controls bar */} +
+ {collections.length > 0 && ( +
+ + +
+ )} +
+ + setToken(e.target.value)} + placeholder="Paste key to test it (optional)" + className="bg-parchment border-rule focus:border-ink rounded-control w-52 border px-2 py-1 font-mono text-xs focus:outline-none" + /> +
+
+ +
+ {/* Left column: endpoint list */} +
+

Endpoints

+
+ {endpoints.map((ep, i) => ( + + ))} +
+ + {collections.length === 0 && ( +

+ No collections yet. Create one to see collection endpoints. +

+ )} +
+ + {/* Right column: request + response */} +
+ {current && ( + <> + {/* Request display */} +
+
+ + {current.method} + + {current.path} +
+

{current.description}

+
+ + {/* Action bar */} +
+ + + + {token.trim() ? 'Using API key' : 'Using your session'} + +
+ + )} + + {/* Response */} + {response && ( +
+
= 200 && response.status < 300 + ? 'bg-green-50 text-green-800' + : response.status >= 400 + ? 'bg-red-50 text-red-800' + : 'bg-parchment-dark' + }`} + > + + {response.status} {response.statusText} + + {response.time}ms +
+ 
+                {response.body}
+
+
+ )} + + {!response && !loading && ( +
+ Select an endpoint and hit Send to see the response. +
+ )} +
+
+
+ ) +} diff --git a/packages/web/src/components/BaseLayout.tsx b/packages/web/src/components/BaseLayout.tsx new file mode 100644 index 0000000..96abc8b --- /dev/null +++ b/packages/web/src/components/BaseLayout.tsx @@ -0,0 +1,72 @@ +import { Link } from 'react-router' + +import CreateMenu from '~/components/CreateMenu' +import UserMenu from '~/components/UserMenu' +import { useAppContext } from '~/lib/app-context' +import { UNDERLAY_UPDATES_URL } from '~/lib/kf-updates' + +export default function BaseLayout({ children }: { children: React.ReactNode }) { + const { currentUser } = useAppContext() + const isSteward = currentUser?.kfRole === 'admin' + + return ( + <> +
+ +
+ +
{children}
+ + + + ) +} diff --git a/packages/web/src/components/CollectionExplorer.tsx b/packages/web/src/components/CollectionExplorer.tsx new file mode 100644 index 0000000..04892af --- /dev/null +++ b/packages/web/src/components/CollectionExplorer.tsx @@ -0,0 +1,364 @@ +import { useEffect, useRef, useState } from 'react' +import { Link, useSearchParams } from 'react-router' + +import { formatBytesFixed, formatCount, timeAgo } from '~/lib/format' + +interface Collection { + id: string + slug: string + name: string + description?: string + tags?: string[] + ownerSlug: string + ownerName?: string + createdAt: string + updatedAt: string + /** The head version's semver ("v1.2.0"). */ + latestVersion: string | null + recordCount: number | null + fileCount: number | null + totalBytes: number | null + lastPushAt: string | null +} + +interface OwnerFacet { + slug: string + name: string | null + count: number +} + +interface TagFacet { + name: string + count: number +} + +type SortKey = 'featured' | 'updated' | 'name' | 'records' + +export interface ExploreData { + collections: Collection[] + facets: { owners: OwnerFacet[]; tags?: TagFacet[] } + featuredTags?: string[] + featuredCollections?: Collection[] +} + +/** `initial` is the explore loader's first page, so the list is in the SSR HTML. */ +export default function CollectionExplorer({ initial }: { initial?: ExploreData | null }) { + const [searchParams, setSearchParams] = useSearchParams() + const initQuery = searchParams.get('q') ?? '' + const initOwner = searchParams.get('owner') + const initTag = searchParams.get('tag') + const initSort = searchParams.get('sort') + const initSortKey: SortKey = + initSort === 'updated' || initSort === 'name' || initSort === 'records' ? initSort : 'featured' + const [query, setQuery] = useState(initQuery) + const [selectedOwner, setSelectedOwner] = useState(initOwner) + const [selectedTag, setSelectedTag] = useState(initTag) + const [sort, setSort] = useState(initSortKey) + const [collections, setCollections] = useState(initial?.collections ?? []) + const [owners, setOwners] = useState(initial?.facets.owners ?? []) + const [tagFacets, setTagFacets] = useState(initial?.facets.tags ?? []) + const [featuredTags, setFeaturedTags] = useState(initial?.featuredTags ?? []) + const [featuredCollections, setFeaturedCollections] = useState( + initial?.featuredCollections ?? [], + ) + const [loading, setLoading] = useState(!initial) + const timerRef = useRef>(undefined) + + const isFiltered = !!(query || selectedOwner || selectedTag) + + function syncUrl(q: string, owner: string | null, sortBy: SortKey, tag: string | null) { + const next = new URLSearchParams() + if (q) next.set('q', q) + if (owner) next.set('owner', owner) + if (tag) next.set('tag', tag) + if (sortBy !== 'featured') next.set('sort', sortBy) + setSearchParams(next, { replace: true }) + } + + async function load( + q = '', + owner: string | null = null, + sortBy: SortKey = sort, + tag: string | null = selectedTag, + ) { + setLoading(true) + syncUrl(q, owner, sortBy, tag) + const params = new URLSearchParams() + if (q) params.set('q', q) + if (owner) params.set('owner', owner) + if (tag) params.set('tag', tag) + params.set('sort', sortBy) + try { + const res = await fetch(`/api/collections?${params}`) + const data = await res.json() + setCollections(data.collections) + setOwners(data.facets.owners) + setTagFacets(data.facets.tags ?? []) + if (data.featuredTags) setFeaturedTags(data.featuredTags) + if (data.featuredCollections) setFeaturedCollections(data.featuredCollections) + } catch { + setCollections([]) + setOwners([]) + setTagFacets([]) + } + setLoading(false) + } + + // Mount-only by design: seed the list once from the URL-derived initial values. + // Every later fetch comes from an explicit user action (handleInput, the filter + // and sort handlers), never from this effect. The deps the rule asks for must NOT + // be added: `load` is a plain function declared in the component body, so it is a + // new reference every render, and depending on it would re-run this effect on + // each one — an endless refetch loop against /api/collections. + // + // The directive has to sit on the dependency-array line: oxlint reports this rule + // against the deps array, not the line its column points at (oxc-project/oxc#18328). + useEffect(() => { + // The loader already fetched this list for the URL's filters. + if (initial) return + load(initQuery, initOwner, initSortKey, initTag) + }, []) // oxlint-disable-line react-hooks/exhaustive-deps + + function handleInput(value: string) { + setQuery(value) + clearTimeout(timerRef.current) + timerRef.current = setTimeout(() => load(value, selectedOwner, sort, selectedTag), 300) + } + + function handleOwnerClick(ownerSlug: string | null) { + setSelectedOwner(ownerSlug) + load(query, ownerSlug, sort, selectedTag) + } + + function handleTagClick(tag: string | null) { + setSelectedTag(tag) + load(query, selectedOwner, sort, tag) + } + + function handleSortChange(value: string) { + const s = value as SortKey + setSort(s) + load(query, selectedOwner, s, selectedTag) + } + + const visibleTags = + featuredTags.length > 0 + ? featuredTags.filter((t) => tagFacets.some((f) => f.name === t)) + : tagFacets.slice(0, 12).map((f) => f.name) + + return ( +
+ {/* Sidebar facets */} + {visibleTags.length > 0 && ( + + )} + + {/* Main content */} +
+ {/* Search + sort bar (always at top, never moves) */} +
+
+ + + + + handleInput(e.target.value)} + /> +
+ +
+ + {/* Featured collections hero */} + {featuredCollections.length > 0 && !isFiltered && ( +
+

+ Featured +

+
+ {featuredCollections.map((c) => ( + +
+ {c.ownerSlug}/ + {c.slug} +
+ {c.description && ( +

+ {c.description} +

+ )} +
+ {c.recordCount != null && {formatCount(c.recordCount)} records} + {c.tags && c.tags.length > 0 && ( + + {c.tags[0]} + + )} +
+ + ))} +
+
+ )} + + {/* Mobile owner filter */} + {owners.length > 0 && ( +
+ + {owners.map((o) => ( + + ))} +
+ )} + + {loading ? ( +

Loading...

+ ) : collections.length === 0 ? ( +
+

+ {query || selectedOwner || selectedTag + ? 'No collections match your filters.' + : 'No public collections yet.'} +

+
+ ) : ( + <> +

+ {collections.length} collection{collections.length !== 1 ? 's' : ''} + {selectedTag && ` in ${selectedTag}`} + {selectedOwner && ` from ${selectedOwner}`} + {query && ` matching "${query}"`} +

+
+ {collections.map((c) => ( + +
+
+ {c.ownerSlug}/ + {c.slug} + {c.tags && c.tags.length > 0 && ( + + {c.tags.slice(0, 2).map((tag) => ( + + {tag} + + ))} + + )} +
+ {c.description && ( +

+ {c.description} +

+ )} +
+
+ {c.latestVersion && {c.latestVersion}} + {c.recordCount != null && ( + {formatCount(c.recordCount)} rec + )} + {c.totalBytes != null && c.totalBytes > 0 && ( + + {formatBytesFixed(c.totalBytes)} + + )} + {c.lastPushAt && ( + {timeAgo(c.lastPushAt)} + )} +
+ + ))} +
+ + )} +
+
+ ) +} diff --git a/packages/web/src/components/CreateMenu.tsx b/packages/web/src/components/CreateMenu.tsx new file mode 100644 index 0000000..80db050 --- /dev/null +++ b/packages/web/src/components/CreateMenu.tsx @@ -0,0 +1,47 @@ +import { useCallback, useRef, useState } from 'react' +import { Link } from 'react-router' + +import { useDismissable } from '~/lib/use-dismissable' + +export default function CreateMenu() { + const [open, setOpen] = useState(false) + const ref = useRef(null) + + useDismissable( + open, + useCallback(() => setOpen(false), []), + ref, + ) + + return ( +
+ + {open && ( +
+ setOpen(false)} + className="text-ink hover:bg-parchment-dark block px-3 py-2 text-sm transition-colors" + > + New collection + + setOpen(false)} + className="text-ink hover:bg-parchment-dark block px-3 py-2 text-sm transition-colors" + > + New organization + +
+ )} +
+ ) +} diff --git a/packages/web/src/components/DiscussionDrawer.tsx b/packages/web/src/components/DiscussionDrawer.tsx new file mode 100644 index 0000000..96fdd5f --- /dev/null +++ b/packages/web/src/components/DiscussionDrawer.tsx @@ -0,0 +1,506 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' + +import { Button, Input, Textarea } from '~/components/ui' +import { useAppContext } from '~/lib/app-context' + +interface Comment { + id: string + anchor: string + quote: string | null + quoteContext: { prefix: string; suffix: string } | null + parentId: string | null + userId: string + body: string + approvedAt: string | null + status: 'open' | 'answered' | 'decided' | 'changed' + resolutionNote: string | null + createdAt: string + editedAt: string | null + authorName: string + authorImage: string | null +} + +interface DiscussionDrawerProps { + page: string + anchor: string | null + quote?: string | null + quoteContext?: { prefix: string; suffix: string } | null + comments: Record + onClose: () => void + onRefresh: () => void + isSteward: boolean +} + +export default function DiscussionDrawer({ + page, + anchor, + quote: initialQuote, + quoteContext: initialQuoteContext, + comments, + onClose, + onRefresh, + isSteward, +}: DiscussionDrawerProps) { + const { currentUser } = useAppContext() + const [body, setBody] = useState('') + const [replyTo, setReplyTo] = useState(null) + const [replyBody, setReplyBody] = useState('') + const [submitting, setSubmitting] = useState(false) + const [resolveId, setResolveId] = useState(null) + const [resolveStatus, setResolveStatus] = useState<'answered' | 'decided' | 'changed'>('answered') + const [resolveNote, setResolveNote] = useState('') + const [showClosed, setShowClosed] = useState(false) + const drawerRef = useRef(null) + + const anchorComments = useMemo(() => (anchor ? (comments[anchor] ?? []) : []), [anchor, comments]) + const threads = anchorComments.filter((c) => !c.parentId) + const openThreads = threads.filter((t) => t.status === 'open') + const closedThreads = threads.filter((t) => t.status !== 'open') + + const getReplies = useCallback( + (parentId: string) => anchorComments.filter((c) => c.parentId === parentId), + [anchorComments], + ) + + useEffect(() => { + function handleEsc(e: KeyboardEvent) { + if (e.key === 'Escape') onClose() + } + document.addEventListener('keydown', handleEsc) + return () => document.removeEventListener('keydown', handleEsc) + }, [onClose]) + + async function handleSubmit(e: React.FormEvent) { + e.preventDefault() + if (!body.trim() || !anchor) return + setSubmitting(true) + try { + const payload: Record = { anchor, body: body.trim() } + if (initialQuote) { + payload.quote = initialQuote + if (initialQuoteContext) payload.quoteContext = initialQuoteContext + } + const res = await fetch(`/api/pages/${page}/comments`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }) + if (res.ok) { + setBody('') + onRefresh() + } + } finally { + setSubmitting(false) + } + } + + async function handleReply(e: React.FormEvent) { + e.preventDefault() + if (!replyBody.trim() || !replyTo || !anchor) return + setSubmitting(true) + try { + const res = await fetch(`/api/pages/${page}/comments`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ anchor, body: replyBody.trim(), parentId: replyTo }), + }) + if (res.ok) { + setReplyBody('') + setReplyTo(null) + onRefresh() + } + } finally { + setSubmitting(false) + } + } + + async function handleApprove(commentId: string) { + await fetch(`/api/pages/${page}/comments/${commentId}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ approve: true }), + }) + onRefresh() + } + + async function handleDecline(commentId: string) { + await fetch(`/api/pages/${page}/comments/${commentId}`, { + method: 'DELETE', + }) + onRefresh() + } + + async function handleResolve(e: React.FormEvent) { + e.preventDefault() + if (!resolveId) return + await fetch(`/api/pages/${page}/comments/${resolveId}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ status: resolveStatus, resolutionNote: resolveNote || undefined }), + }) + setResolveId(null) + setResolveNote('') + onRefresh() + } + + function formatDate(iso: string) { + return new Date(iso).toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + year: 'numeric', + }) + } + + const statusLabel: Record = { + answered: 'Answered', + decided: 'Decided', + changed: 'Changed', + } + + const statusColor: Record = { + answered: 'bg-blue-100 text-blue-800', + decided: 'bg-amber-100 text-amber-800', + changed: 'bg-green-100 text-green-800', + } + + if (!anchor) return null + + return ( +
+
+
+
+

+ Discussion: {anchor} +

+ +
+ +
+ {initialQuote && ( +
+ "{initialQuote}" +
+ )} + + {openThreads.length === 0 && closedThreads.length === 0 && ( +

+ No discussion yet.{' '} + {currentUser ? 'Start the conversation below.' : 'Log in to comment.'} +

+ )} + + {openThreads.map((thread) => ( + { + setResolveId(id) + setResolveStatus('answered') + setResolveNote('') + }} + onResolveStatusChange={setResolveStatus} + onResolveNoteChange={setResolveNote} + onResolveSubmit={handleResolve} + onResolveCancel={() => setResolveId(null)} + /> + ))} + + {closedThreads.length > 0 && ( +
+ + {showClosed && + closedThreads.map((thread) => ( + { + setResolveId(id) + setResolveStatus('answered') + setResolveNote('') + }} + onResolveStatusChange={setResolveStatus} + onResolveNoteChange={setResolveNote} + onResolveSubmit={handleResolve} + onResolveCancel={() => setResolveId(null)} + /> + ))} +
+ )} +
+ + {currentUser && ( +
+
+