From 4ddf317e4e55de52df0e41d1019a28afbc0a265f Mon Sep 17 00:00:00 2001 From: Ensky Lin Date: Tue, 12 May 2026 01:23:37 +0000 Subject: [PATCH 1/3] docs: fix typo in README ("Analaytics" -> "Analytics") Co-Authored-By: Paperclip --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dedd7bdf..22f3a180 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ There is only one "database": the Cloudflare Analytics Engine dataset, which is Right now there is no local "test" database. This means in local development: - Writes will no-op (no hits will be recorded) -- Reads will be read from the production Analaytics Engine dataset (local development shows production data) +- Reads will be read from the production Analytics Engine dataset (local development shows production data) ### Sampling From fd4106a49a8c5a3a4bd5fb8bf6a7b41da48d10da Mon Sep 17 00:00:00 2001 From: Ensky Date: Tue, 12 May 2026 10:17:31 +0800 Subject: [PATCH 2/3] fix: avoid tableFromJSON codegen for CF Workers compatibility (#3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apache-arrow's `tableFromJSON()` (via `vectorFromArray` → `makeBuilder` → `new StructBuilder` → `createIsValidFunction`) calls `new Function(...)` to generate validity checkers. Cloudflare Workers' runtime forbids runtime code generation: EvalError: Code generation from strings disallowed for this context at new Function () at createIsValidFunction ... at tableFromJSON at extractAsArrow The daily-rollup cron has thrown on every fire since deployment under this restriction, so the R2 `counterscale-daily-rollups` bucket has never received an object. Replace `tableFromJSON(records)` with a small `recordsToTable` helper that builds `Utf8`/`Float64` `Data` directly via `makeData` and assembles them into a `RecordBatch` — bypassing the Builder/codegen path entirely. Adds a vitest covering: - round-trip through `tableToIPC` / `tableFromIPC` - null/undefined handling - empty-input edge case - a guard that proxies `globalThis.Function` and asserts no `new Function(...)` call happens during table construction or IPC serialization (proves the CF Workers compatibility property at unit level) Co-authored-by: Paperclip --- .../workers/lib/__tests__/arrow.test.ts | 110 ++++++++++++++++++ packages/server/workers/lib/arrow.ts | 97 ++++++++++++++- 2 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 packages/server/workers/lib/__tests__/arrow.test.ts diff --git a/packages/server/workers/lib/__tests__/arrow.test.ts b/packages/server/workers/lib/__tests__/arrow.test.ts new file mode 100644 index 00000000..e7fbd6ea --- /dev/null +++ b/packages/server/workers/lib/__tests__/arrow.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "vitest"; +import { tableFromIPC, tableToIPC } from "apache-arrow"; + +import { recordsToTable } from "../arrow"; + +describe("recordsToTable", () => { + test("round-trips mixed string/number columns through Arrow IPC", () => { + const records = [ + { + date: "2026-05-11", + siteId: "site-a", + views: 12, + visitors: 7, + bounces: 3, + path: "/", + country: "US", + }, + { + date: "2026-05-11", + siteId: "site-a", + views: 5, + visitors: 4, + bounces: 0, + path: "/docs", + country: "TW", + }, + ]; + + const table = recordsToTable(records); + const buf = new Uint8Array(tableToIPC(table, "file")); + const decoded = tableFromIPC(buf); + + expect(decoded.numRows).toBe(2); + expect(decoded.schema.fields.map((f) => f.name)).toEqual([ + "date", + "siteId", + "views", + "visitors", + "bounces", + "path", + "country", + ]); + + const rows = decoded.toArray().map((r: unknown) => { + const row = r as Record; + return { + date: String(row.date), + siteId: String(row.siteId), + views: Number(row.views), + visitors: Number(row.visitors), + bounces: Number(row.bounces), + path: String(row.path), + country: String(row.country), + }; + }); + expect(rows).toEqual(records); + }); + + test("handles null/undefined values without crashing", () => { + const records = [ + { a: "x", b: 1 }, + { a: null, b: null }, + { a: undefined, b: undefined }, + ]; + const table = recordsToTable(records); + const buf = new Uint8Array(tableToIPC(table, "file")); + const decoded = tableFromIPC(buf); + expect(decoded.numRows).toBe(3); + }); + + test("returns empty table for empty input", () => { + const table = recordsToTable([]); + expect(table.numRows).toBe(0); + }); + + test("does not invoke `new Function()` (CF Workers codegen ban)", () => { + // Cloudflare Workers' runtime forbids `new Function(...)` and `eval`. + // apache-arrow's Builder path (used by tableFromJSON / vectorFromArray) + // triggers `new Function()` for validity-check codegen — this is the + // exact regression we are guarding against. + const realFunction = globalThis.Function; + let callCount = 0; + const FunctionProxy = new Proxy(realFunction, { + construct(target, args) { + callCount++; + return Reflect.construct(target, args); + }, + apply(target, thisArg, args) { + callCount++; + return Reflect.apply(target, thisArg, args); + }, + }); + + const records = [ + { date: "2026-05-11", siteId: "s", views: 1, path: "/" }, + ]; + + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (globalThis as any).Function = FunctionProxy; + const table = recordsToTable(records); + tableToIPC(table, "file"); + } finally { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (globalThis as any).Function = realFunction; + } + + expect(callCount).toBe(0); + }); +}); diff --git a/packages/server/workers/lib/arrow.ts b/packages/server/workers/lib/arrow.ts index 746ca262..5407981d 100644 --- a/packages/server/workers/lib/arrow.ts +++ b/packages/server/workers/lib/arrow.ts @@ -1,8 +1,95 @@ import { AnalyticsEngineAPI } from "../../app/analytics/query"; import { ColumnMappings } from "../../app/analytics/schema"; -import { tableFromJSON, tableToIPC } from "apache-arrow"; +import { + type Data, + Float64, + RecordBatch, + Schema, + Table, + Utf8, + makeData, + tableToIPC, +} from "apache-arrow"; import dayjs from "dayjs"; +type RecordValue = string | number | null | undefined; +type Record = { [key: string]: RecordValue }; + +// Build a Utf8 Data buffer directly from an array of strings. +// Bypasses apache-arrow's Builder path, which uses `new Function()` for +// validity-checking codegen and is forbidden by the Cloudflare Workers +// runtime ("Code generation from strings disallowed for this context"). +function buildUtf8Data(values: (string | null | undefined)[]): Data { + const encoder = new TextEncoder(); + const encoded: Uint8Array[] = new Array(values.length); + let totalBytes = 0; + for (let i = 0; i < values.length; i++) { + const v = values[i]; + const bytes = v == null ? new Uint8Array(0) : encoder.encode(String(v)); + encoded[i] = bytes; + totalBytes += bytes.length; + } + const valueOffsets = new Int32Array(values.length + 1); + const data = new Uint8Array(totalBytes); + let pos = 0; + for (let i = 0; i < values.length; i++) { + valueOffsets[i] = pos; + data.set(encoded[i], pos); + pos += encoded[i].length; + } + valueOffsets[values.length] = pos; + return makeData({ + type: new Utf8(), + length: values.length, + nullCount: 0, + valueOffsets, + data, + }); +} + +function buildFloat64Data(values: (number | null | undefined)[]): Data { + const buf = new Float64Array(values.length); + for (let i = 0; i < values.length; i++) { + buf[i] = values[i] ?? 0; + } + return makeData({ + type: new Float64(), + length: values.length, + nullCount: 0, + data: buf, + }); +} + +// Convert an array of homogeneous records to an Arrow Table without invoking +// the Builder/`new Function()` codegen path. Column type is inferred from the +// first non-null sample per column: `number` → Float64, otherwise → Utf8. +export function recordsToTable(records: Record[]): Table { + if (records.length === 0) { + return new Table(new Schema([])); + } + const columnNames = Object.keys(records[0]); + const children: { [name: string]: Data } = {}; + for (const name of columnNames) { + let sample: RecordValue = undefined; + for (const r of records) { + const v = r[name]; + if (v != null) { + sample = v; + break; + } + } + const values = records.map((r) => r[name]); + if (typeof sample === "number") { + children[name] = buildFloat64Data(values as (number | null)[]); + } else { + children[name] = buildUtf8Data( + values.map((v) => (v == null ? null : String(v))), + ); + } + } + return new Table(new RecordBatch(children)); +} + export async function extractAsArrow( { accountId, bearerToken }: { accountId: string; bearerToken: string }, bucket: R2Bucket, @@ -27,10 +114,10 @@ export async function extractAsArrow( ); // Convert Map to array of records for Arrow table creation - const records: any[] = []; + const records: Record[] = []; data.forEach((counts, key) => { const [date, siteId, ...columnValues] = key; - const record: any = { + const record: Record = { date, siteId, views: counts.views, @@ -46,8 +133,8 @@ export async function extractAsArrow( records.push(record); }); - // Create Arrow table from JSON records - const table = tableFromJSON(records); + // Build Arrow table without invoking codegen-based builders. + const table = recordsToTable(records); // Convert to Arrow IPC buffer const arrowBuffer = new Uint8Array(tableToIPC(table, "file")); From 5286946afb1f1e8d41cc5c0b34cae4d7b6b7d4e0 Mon Sep 17 00:00:00 2001 From: Ensky Lin Date: Tue, 11 Aug 2026 05:36:40 +0000 Subject: [PATCH 3/3] fix: preserve nulls in Arrow records Keeps nullish rollup fields distinct from empty strings and zeroes while retaining the direct builder required by Cloudflare Workers. Co-Authored-By: Paperclip --- .../workers/lib/__tests__/arrow.test.ts | 28 ++++++++++++- packages/server/workers/lib/arrow.ts | 39 ++++++++++++++++--- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/packages/server/workers/lib/__tests__/arrow.test.ts b/packages/server/workers/lib/__tests__/arrow.test.ts index e7fbd6ea..58124b73 100644 --- a/packages/server/workers/lib/__tests__/arrow.test.ts +++ b/packages/server/workers/lib/__tests__/arrow.test.ts @@ -56,7 +56,7 @@ describe("recordsToTable", () => { expect(rows).toEqual(records); }); - test("handles null/undefined values without crashing", () => { + test("round-trips null/undefined values as null", () => { const records = [ { a: "x", b: 1 }, { a: null, b: null }, @@ -65,7 +65,31 @@ describe("recordsToTable", () => { const table = recordsToTable(records); const buf = new Uint8Array(tableToIPC(table, "file")); const decoded = tableFromIPC(buf); - expect(decoded.numRows).toBe(3); + const rows = decoded.toArray().map((row: unknown) => { + const values = row as Record; + return { a: values.a, b: values.b }; + }); + + expect(rows).toEqual([ + { a: "x", b: 1 }, + { a: null, b: null }, + { a: null, b: null }, + ]); + }); + + test("only allocates validity bitmaps for columns with nulls", () => { + const table = recordsToTable([ + { nullable: null, complete: "x" }, + { nullable: "value", complete: "y" }, + ]); + const batch = table.batches[0]; + const nullable = batch.getChild("nullable")?.data[0]; + const complete = batch.getChild("complete")?.data[0]; + + expect(nullable?.nullCount).toBe(1); + expect(nullable?.nullBitmap.byteLength).toBe(8); + expect(complete?.nullCount).toBe(0); + expect(complete?.nullBitmap.byteLength).toBe(0); }); test("returns empty table for empty input", () => { diff --git a/packages/server/workers/lib/arrow.ts b/packages/server/workers/lib/arrow.ts index 5407981d..6f3912a1 100644 --- a/packages/server/workers/lib/arrow.ts +++ b/packages/server/workers/lib/arrow.ts @@ -15,6 +15,29 @@ import dayjs from "dayjs"; type RecordValue = string | number | null | undefined; type Record = { [key: string]: RecordValue }; +function buildValidityBitmap(values: RecordValue[]): { + nullCount: number; + nullBitmap?: Uint8Array; +} { + let nullCount = 0; + for (const value of values) { + if (value == null) { + nullCount++; + } + } + if (nullCount === 0) { + return { nullCount }; + } + + const nullBitmap = new Uint8Array(Math.ceil(values.length / 64) * 8); + for (let i = 0; i < values.length; i++) { + if (values[i] != null) { + nullBitmap[i >> 3] |= 1 << (i & 7); + } + } + return { nullCount, nullBitmap }; +} + // Build a Utf8 Data buffer directly from an array of strings. // Bypasses apache-arrow's Builder path, which uses `new Function()` for // validity-checking codegen and is forbidden by the Cloudflare Workers @@ -38,10 +61,12 @@ function buildUtf8Data(values: (string | null | undefined)[]): Data { pos += encoded[i].length; } valueOffsets[values.length] = pos; + const { nullCount, nullBitmap } = buildValidityBitmap(values); return makeData({ type: new Utf8(), length: values.length, - nullCount: 0, + nullCount, + nullBitmap, valueOffsets, data, }); @@ -52,17 +77,21 @@ function buildFloat64Data(values: (number | null | undefined)[]): Data for (let i = 0; i < values.length; i++) { buf[i] = values[i] ?? 0; } + const { nullCount, nullBitmap } = buildValidityBitmap(values); return makeData({ type: new Float64(), length: values.length, - nullCount: 0, + nullCount, + nullBitmap, data: buf, }); } -// Convert an array of homogeneous records to an Arrow Table without invoking -// the Builder/`new Function()` codegen path. Column type is inferred from the -// first non-null sample per column: `number` → Float64, otherwise → Utf8. +/** + * Converts homogeneous records to an Arrow Table without Builder codegen and + * preserves null or undefined values as Arrow nulls. Column type is inferred + * from the first non-null sample: `number` becomes Float64, otherwise Utf8. + */ export function recordsToTable(records: Record[]): Table { if (records.length === 0) { return new Table(new Schema([]));