From d39c8e4e99cae2518679f243b41316a69ec0c165 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 7 Oct 2026 09:47:25 +0200 Subject: [PATCH 1/2] Add a Tinybird entry whose datasources are query tables defineDatasource, defineMaterializedView, column, t, engine, node and InferRow take the call shapes of @tinybirdco/sdk, and buildProject writes the same datafiles byte for byte (checked against a 42-datasource, 44-view project). Each datasource is a ClickHouse table with its DDL, so queries take it directly and renderSchema writes it for a plain server. Adds CH.lowCardinality, CH.simpleAggregateFunction and CH.precision, which change the DDL only, and renders nullable(lowCardinality(T)) as LowCardinality(Nullable(T)). --- CHANGELOG.md | 9 + README.md | 1 + docs/README.md | 1 + docs/reference.md | 3 +- docs/tables-and-types.md | 5 + docs/tinybird.md | 40 ++++ package.json | 4 + src/ch/types.ts | 27 ++- src/clickhouse.ts | 3 + src/schema/schema.test.ts | 20 ++ src/tinybird.ts | 50 +++++ src/tinybird/datafile.ts | 143 ++++++++++++++ src/tinybird/datasource.ts | 331 +++++++++++++++++++++++++++++++++ src/tinybird/tinybird.test.ts | 183 ++++++++++++++++++ src/tinybird/types.ts | 104 +++++++++++ tests/dialect-cases.ts | 8 + tests/dialect-coverage.test.ts | 3 + tsdown.config.ts | 1 + 18 files changed, 934 insertions(+), 2 deletions(-) create mode 100644 docs/tinybird.md create mode 100644 src/tinybird.ts create mode 100644 src/tinybird/datafile.ts create mode 100644 src/tinybird/datasource.ts create mode 100644 src/tinybird/tinybird.test.ts create mode 100644 src/tinybird/types.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ebf65b..b18bfde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- New `@maple-dev/effect-orm/tinybird` entry: `defineDatasource`, `defineMaterializedView`, + `column`, `t`, `engine`, `node` and `InferRow` with the call shapes of `@tinybirdco/sdk`, and + `buildProject` writing the same datafiles byte for byte. A datasource is a ClickHouse + `table`, so the query builder takes it and `renderSchema` writes its DDL. See + [docs/tinybird.md](./docs/tinybird.md). +- `CH.lowCardinality(t)`, `CH.simpleAggregateFunction(fn, t)` and `CH.precision(t, digits)`: + storage wrappers that change the DDL and not how the column reads. `CH.nullable` of a + `LowCardinality` type renders `LowCardinality(Nullable(T))`, which ClickHouse requires. + - Fixes found migrating a large consumer onto the strict params and brands: - A param inside `when` / `whenTrue`, or one whose name is a union, is optional in the params `compile` asks for: whether it renders is only known at runtime. It used to be required, diff --git a/README.md b/README.md index 8c49f26..39ff29b 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,7 @@ regressions live in [`src/docs-examples.test.ts`](./src/docs-examples.test.ts). | `@maple-dev/effect-orm/postgres` | The same for Postgres: the builder, Postgres column types (`text`, `int8`, `timestamptz`, …) and functions, `table` with keys, indexes and foreign keys, and a `compile` for Postgres. | | `@maple-dev/effect-orm/expr` | Kitchen-sink namespace: every expression helper plus all ClickHouse functions under their raw names (`min_`, `toString_`, `toStartOfInterval`, `dynamicColumn`, …). | | `@maple-dev/effect-orm/sql` | The low-level `SqlFragment` AST (`raw`, `ident`, `compile`, …) for hand-rolling fragments. | +| `@maple-dev/effect-orm/tinybird` | Tinybird datasources and materialized views, SDK-compatible, that are also query tables; `buildProject` writes the datafiles. See [Tinybird](./docs/tinybird.md). | | `@maple-dev/effect-orm/schema` | Migration tooling over `table` values: DDL rendering, snapshots, the schema diff. Pure. | | `@maple-dev/effect-orm/kit`, `/migrate` | `effect-orm generate` and `check`; applying migrations through a driver you provide. See [Schema and migrations](./docs/migrations.md). | | `@maple-dev/effect-orm/database` | `Database` over your `SqlClient`: `run`, `execute`, `transaction` with retry. | diff --git a/docs/README.md b/docs/README.md index 5d98d99..7a61d11 100644 --- a/docs/README.md +++ b/docs/README.md @@ -84,6 +84,7 @@ Roughly in reading order. | `@maple-dev/effect-orm/benchmark` | Driver-free suite definitions, runner, report schemas, and comparisons | | `@maple-dev/effect-orm/benchmark/http` | ClickHouse HTTP transport, environment configuration, and query-log collection | | `@maple-dev/effect-orm/benchmark/cli` | `runCli(args)` for embedding the bundled `ch-bench` commands | +| `@maple-dev/effect-orm/tinybird` | Tinybird datasources and views with `@tinybirdco/sdk` call shapes; each datasource is a query table. See [Tinybird](./tinybird.md) | | `@maple-dev/effect-orm/schema` | Tooling over `table` values: `renderSchema` / `renderPgSchema`, `entitiesOf` / `pgEntitiesOf`, snapshots, and the schema diff. Pure | | `@maple-dev/effect-orm/kit` | `generate` and `check` over a migrations folder, `defineConfig`, and `runCli` for the bundled `effect-orm` command. Node or Bun | | `@maple-dev/effect-orm/migrate` | `run`, `status`, `verify`, `baseline`, and `MigrationDriver`: applies ClickHouse or Postgres migrations through a driver you provide | diff --git a/docs/reference.md b/docs/reference.md index f2d781d..6cead18 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -300,7 +300,8 @@ that cannot render throws `SchemaDefinitionDefect` while the module loads. **Constructors** — `string`, `bool`, `uint8`, `uint16`, `uint32`, `uint64`, `int32`, `int64`, `float64`, `dateTime`, `dateTime64`, `dateTimeString`, `dateTime64String`, `map`, -`array`, `nullable`, `aggregateState(fn, ...args)`, `custom(sql, schema, literalSchema?)`, +`array`, `nullable`, `lowCardinality(type)`, `simpleAggregateFunction(fn, type)`, +`precision(dateTime64Type, digits)`, `aggregateState(fn, ...args)`, `custom(sql, schema, literalSchema?)`, `brand(type, schema)` (the type narrowed by `schema`: a branded id, a literal union; see [Branded columns](./tables-and-types.md#branded-columns)), and `untyped(sql)` for a wire value passed through unvalidated. See [Tables and column types](./tables-and-types.md). diff --git a/docs/tables-and-types.md b/docs/tables-and-types.md index e9c9efc..c2d8952 100644 --- a/docs/tables-and-types.md +++ b/docs/tables-and-types.md @@ -182,6 +182,11 @@ The constructors are values, not calls (except the parameterised ones): | `CH.nullable(t)` | `Nullable(T)` | `T \| null` | value or `null` | | `CH.untyped(sql)` | whatever you name | `unknown` | unvalidated | +Three wrappers change only the DDL, never how a column reads back: +`CH.lowCardinality(t)` (`LowCardinality(T)`; inside `nullable` it renders +`LowCardinality(Nullable(T))`), `CH.simpleAggregateFunction("sum", t)` +(`SimpleAggregateFunction(sum, T)`), and `CH.precision(CH.dateTime64String, 9)` (`DateTime64(9)`). + Column types, functions, and the query builder share the one `CH` namespace, so a schema module and a query module import the same thing. diff --git a/docs/tinybird.md b/docs/tinybird.md new file mode 100644 index 0000000..561f46d --- /dev/null +++ b/docs/tinybird.md @@ -0,0 +1,40 @@ +# Tinybird datasources + +`@maple-dev/effect-orm/tinybird` defines Tinybird datasources and materialized views with the +call shapes of `@tinybirdco/sdk` (`defineDatasource`, `column`, `t`, `engine`, +`defineMaterializedView`, `node`, `InferRow`), so a project moves over by changing its import. + +What it adds over the SDK: a datasource **is** a ClickHouse `table` with its DDL. The query +builder takes it directly, `renderSchema` writes it for a plain ClickHouse server, and the +datafiles Tinybird deploys come from the same definition. + +```ts +import * as CH from "@maple-dev/effect-orm/clickhouse" +import { buildProject, column, defineDatasource, engine, t } from "@maple-dev/effect-orm/tinybird" + +export const events = defineDatasource("events", { + schema: { + OrgId: column(t.string().lowCardinality().brand(OrgId), { jsonPath: "$.org_id" }), + Timestamp: t.dateTime64(9), + Kind: t.string().lowCardinality().default("click"), + }, + engine: engine.mergeTree({ sortingKey: ["OrgId", "Timestamp"], ttl: "toDate(Timestamp) + INTERVAL 30 DAY" }), + tenantColumn: "OrgId", +}) + +CH.from(events).select("Kind").where(($) => [$.OrgId.eq(CH.param.of(events.columns.OrgId, "orgId"))]) + +const { datasources, pipes } = buildProject(await import("./datasources"), await import("./views")) +``` + +- `t.*` builds a ClickHouse column type plus its datafile modifiers. `.lowCardinality()`, + `.nullable()`, `.default(v)`, `.defaultExpr(sql)`, `.codec(c)` and `.jsonPath(p)` behave as in + the SDK; `.brand(schema)` narrows the query column and leaves the ingested row alone. +- `DateTime` and `DateTime64` columns read back as the string ClickHouse sends, which is also + what `InferRow` types them as. `InferRow` types a `Map` as a `Record`, the JSON Tinybird ingests. +- `buildProject(...modules)` writes every datasource and view a module exports, in export order. + Its output matches `@tinybirdco/sdk` 0.0.84 byte for byte for the features here: schemas with + json paths, defaults and codecs, the MergeTree family and `Null` engines, indexes, forward + queries, and materialized views. Kafka, S3, tokens, endpoints and copy pipes are not ported. +- Materialized view SQL is a string. For a view checked against its target's columns, use + `CH.materializedView` from `/clickhouse`. diff --git a/package.json b/package.json index 4f44ae7..82d854f 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,10 @@ "types": "./dist/schema.d.mts", "import": "./dist/schema.mjs" }, + "./tinybird": { + "types": "./dist/tinybird.d.mts", + "import": "./dist/tinybird.mjs" + }, "./migrate": { "types": "./dist/migrate.d.mts", "import": "./dist/migrate.mjs" diff --git a/src/ch/types.ts b/src/ch/types.ts index 8101035..543dc8d 100644 --- a/src/ch/types.ts +++ b/src/ch/types.ts @@ -325,15 +325,40 @@ export const map = , V extends CHType>(e: E): CHArray => chType("Array", `Array(${e.sql})`, Schema.Array(e.schema), undefined, e) as CHArray +const LOW_CARDINALITY = /^LowCardinality\((.+)\)$/s + export const nullable = >(t: T): CHNullable => chType( "Nullable", - `Nullable(${t.sql})`, + // ClickHouse rejects `Nullable(LowCardinality(T))`; the wrapper goes outside. + LOW_CARDINALITY.test(t.sql) ? t.sql.replace(LOW_CARDINALITY, "LowCardinality(Nullable($1))") : `Nullable(${t.sql})`, Schema.NullOr(t.schema), Schema.NullOr(t.literalSchema), t, ) as CHNullable +/** + * `LowCardinality(T)`: dictionary-encoded storage. Queries see `T` unchanged, + * so only the DDL differs. + */ +export const lowCardinality = >(t: T): T => + LOW_CARDINALITY.test(t.sql) ? t : { ...t, sql: `LowCardinality(${t.sql})` } + +/** + * `SimpleAggregateFunction(fn, T)`: stores the aggregated value itself, so it + * reads back as `T`. Only the DDL differs. + */ +export const simpleAggregateFunction = >(fn: string, t: T): T => ({ + ...t, + sql: `SimpleAggregateFunction(${fn}, ${t.sql})`, +}) + +/** A `DateTime64` with its sub-second digits: `precision(dateTime64, 9)` is `DateTime64(9)`. */ +export const precision = >( + t: T, + digits: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9, +): T => ({ ...t, sql: `DateTime64(${digits})` }) + /** * A column type of your own: a ClickHouse type name and the schema its wire * value decodes with. diff --git a/src/clickhouse.ts b/src/clickhouse.ts index 2401e47..2ab20c0 100644 --- a/src/clickhouse.ts +++ b/src/clickhouse.ts @@ -44,6 +44,9 @@ export { map, array, nullable, + lowCardinality, + simpleAggregateFunction, + precision, custom, brand, } from "./ch/types" diff --git a/src/schema/schema.test.ts b/src/schema/schema.test.ts index 9cd4dff..adf484b 100644 --- a/src/schema/schema.test.ts +++ b/src/schema/schema.test.ts @@ -73,6 +73,26 @@ describe("defineTable", () => { ) }) + it("renders storage-only wrappers without changing the column's query type", () => { + const Rollup = CH.table("rollup", { + columns: { + OrgId: CH.lowCardinality(CH.string), + Env: CH.nullable(CH.lowCardinality(CH.string)), + At: CH.precision(CH.dateTime64String, 9), + Calls: CH.simpleAggregateFunction("sum", CH.uint64), + }, + engine: CH.engine.aggregatingMergeTree(), + orderBy: ["OrgId"], + }) + const [table] = S.renderSchema(S.entitiesOf([Rollup])) + expect(table).toContain("\tOrgId LowCardinality(String),") + expect(table).toContain("\tEnv LowCardinality(Nullable(String)),") + expect(table).toContain("\tAt DateTime64(9),") + expect(table).toContain("\tCalls SimpleAggregateFunction(sum, UInt64)") + expect(Rollup.columns.OrgId._tag).toBe("String") + expect(Rollup.columns.Calls._tag).toBe("UInt64") + }) + it("rejects a MergeTree without a sorting key", () => { expect(() => CH.table("bad", { columns: { a: CH.string }, engine: CH.engine.mergeTree() }), diff --git a/src/tinybird.ts b/src/tinybird.ts new file mode 100644 index 0000000..af2536d --- /dev/null +++ b/src/tinybird.ts @@ -0,0 +1,50 @@ +// @maple-dev/effect-orm/tinybird +// +// Tinybird datasources and materialized views with the call shapes of +// `@tinybirdco/sdk`. A datasource is a ClickHouse schema table, so the query +// builder takes it as-is, and `buildProject` writes its datafiles. See +// docs/tinybird.md. + +export { + t, + getTinybirdType, + getModifiers, + isTinybirdType, + type TinybirdType, + type AnyTinybirdType, + type TypeModifiers, + type RowOf, +} from "./tinybird/types" + +export { + column, + defineDatasource, + defineMaterializedView, + engine, + node, + getColumnType, + getColumnJsonPath, + isDatasourceDefinition, + isPipeDefinition, + formatDefaultValue, + type AnyDatasource, + type ColumnDefinition, + type ColumnsOfSchema, + type Datasource, + type DatasourceIndex, + type DatasourceOptions, + type DefaultedColumnsOfSchema, + type EngineConfig, + type InferRow, + type MaterializedViewDefinition, + type NodeDefinition, + type SchemaDefinition, +} from "./tinybird/datasource" + +export { + buildProject, + generateDatasource, + generatePipe, + type Datafile, + type TinybirdProject, +} from "./tinybird/datafile" diff --git a/src/tinybird/datafile.ts b/src/tinybird/datafile.ts new file mode 100644 index 0000000..bf47151 --- /dev/null +++ b/src/tinybird/datafile.ts @@ -0,0 +1,143 @@ +// Tinybird datafiles (`.datasource`, `.pipe`), byte-compatible with the +// generator of `@tinybirdco/sdk` 0.0.84 (MIT), for the features ported here. + +import { + defaultSqlOf, + getColumnJsonPath, + getColumnType, + isDatasourceDefinition, + isPipeDefinition, + splitKey, + type AnyDatasource, + type EngineConfig, + type MaterializedViewDefinition, + type NodeDefinition, +} from "./datasource" +import { getTinybirdType } from "./types" + +export interface Datafile { + readonly name: string + readonly content: string +} + +export interface TinybirdProject { + readonly datasources: ReadonlyArray + readonly pipes: ReadonlyArray +} + +const indent = (text: string): ReadonlyArray => text.split(/\r?\n/).map((line) => ` ${line}`) + +const engineClause = (config: EngineConfig | undefined): string => { + if (config === undefined) return 'ENGINE "MergeTree"' + if (config.type === "Null") return "ENGINE Null" + const lines = [`ENGINE "${config.type}"`] + if (config.partitionKey) lines.push(`ENGINE_PARTITION_KEY "${config.partitionKey}"`) + lines.push(`ENGINE_SORTING_KEY "${normalizeKey(config.sortingKey)}"`) + if (config.primaryKey) lines.push(`ENGINE_PRIMARY_KEY "${normalizeKey(config.primaryKey)}"`) + if (config.ttl) lines.push(`ENGINE_TTL "${config.ttl}"`) + if (config.type === "ReplacingMergeTree" && config.ver) lines.push(`ENGINE_VER "${config.ver}"`) + if (config.type === "ReplacingMergeTree" && config.isDeleted) lines.push(`ENGINE_IS_DELETED "${config.isDeleted}"`) + if (config.type === "CollapsingMergeTree" || config.type === "VersionedCollapsingMergeTree") { + lines.push(`ENGINE_SIGN "${config.sign}"`) + } + if (config.type === "VersionedCollapsingMergeTree") lines.push(`ENGINE_VERSION "${config.version}"`) + if (config.type === "SummingMergeTree" && config.columns && config.columns.length > 0) { + lines.push(`ENGINE_SUMMING_COLUMNS "${config.columns.join(", ")}"`) + } + if (config.settings && Object.keys(config.settings).length > 0) { + const settings = Object.entries(config.settings) + .map(([key, value]) => (typeof value === "string" ? `${key}='${value.replace(/'/g, "\\'")}'` : `${key}=${value}`)) + .join(", ") + lines.push(`ENGINE_SETTINGS "${settings}"`) + } + return lines.join("\n") +} + +// The SDK joins an array key with ", " and keeps a string key as written. +const normalizeKey = (key: string | ReadonlyArray): string => (typeof key === "string" ? key : key.join(", ")) + +export const generateDatasource = (datasource: AnyDatasource): Datafile => { + const { options } = datasource + const parts: Array = [] + if (options.description) parts.push(`DESCRIPTION >\n ${options.description}`, "") + + const includeJsonPaths = options.jsonPaths !== false + const names = Object.keys(options.schema) + const columns = names.map((name, index) => { + const input = options.schema[name]! + const type = getColumnType(input) + const line = [` ${name} ${getTinybirdType(type)}`] + if (includeJsonPaths) line.push(`\`json:${getColumnJsonPath(input) ?? `$.${name}`}\``) + const defaultSql = defaultSqlOf(type) + if (defaultSql !== undefined) line.push(`DEFAULT ${defaultSql}`) + if (type.modifiers.codec) line.push(`CODEC(${type.modifiers.codec})`) + return line.join(" ") + (index < names.length - 1 ? "," : "") + }) + parts.push(["SCHEMA >", ...columns].join("\n"), "") + parts.push(engineClause(options.engine)) + + if (options.indexes && options.indexes.length > 0) { + parts.push( + "", + [ + "INDEXES >", + ...options.indexes.map( + (index) => ` ${index.name} ${index.expr} TYPE ${index.type} GRANULARITY ${index.granularity}`, + ), + ].join("\n"), + ) + } + + const forwardQuery = options.forwardQuery?.trim() + if (forwardQuery) parts.push("", ["FORWARD_QUERY >", ...indent(forwardQuery)].join("\n")) + + return { name: datasource._name, content: parts.join("\n") } +} + +const TEMPLATE = /\{\{[^}]+\}\}|\{%[^%]+%\}/ + +const generateNode = (node: NodeDefinition): string => { + const lines = [`NODE ${node._name}`] + if (node.description) lines.push("DESCRIPTION >", ` ${node.description}`) + lines.push("SQL >") + if (TEMPLATE.test(node.sql)) lines.push(" %") + for (const line of node.sql.trim().split("\n")) lines.push(` ${line}`) + return lines.join("\n") +} + +export const generatePipe = (pipe: MaterializedViewDefinition): Datafile => { + const { options } = pipe + const parts: Array = [] + if (options.description) parts.push(`DESCRIPTION >\n ${options.description}`, "") + options.nodes.forEach((node, index) => { + parts.push(generateNode(node)) + if (index < options.nodes.length - 1) parts.push("") + }) + const materialized = ["TYPE MATERIALIZED", `DATASOURCE ${options.materialized.datasource._name}`] + if (options.materialized.deploymentMethod === "alter") materialized.push("DEPLOYMENT_METHOD alter") + parts.push("", materialized.join("\n")) + return { name: pipe._name, content: parts.join("\n") } +} + +/** + * The datafiles of every datasource and materialized view exported by the + * given modules, in export order (a module namespace lists exports by name). + */ +export const buildProject = (...modules: ReadonlyArray>>): TinybirdProject => { + const seen = new Set() + const datasources: Array = [] + const pipes: Array = [] + for (const module of modules) { + for (const value of Object.values(module)) { + if (seen.has(value)) continue + if (isDatasourceDefinition(value)) { + seen.add(value) + datasources.push(generateDatasource(value)) + } else if (isPipeDefinition(value)) { + seen.add(value) + pipes.push(generatePipe(value)) + } + } + } + return { datasources, pipes } +} diff --git a/src/tinybird/datasource.ts b/src/tinybird/datasource.ts new file mode 100644 index 0000000..accc6e9 --- /dev/null +++ b/src/tinybird/datasource.ts @@ -0,0 +1,331 @@ +// Tinybird datasources and materialized views, with the call shapes of +// `@tinybirdco/sdk`. A datasource IS a ClickHouse schema table, so queries +// take it directly and its DDL renders for a plain ClickHouse server too. + +import * as Define from "../schema/define" +import { SchemaDefinitionDefect, type SchemaTable } from "../schema/define" +import type { EngineSpec } from "../schema/entities" +import { getTinybirdType, isTinybirdType, type AnyTinybirdType, type RowOf, type TinybirdType } from "./types" + +// Columns + +export interface ColumnDefinition { + readonly type: X + readonly jsonPath?: string +} + +export type SchemaDefinition = Record + +export const column = ( + type: X, + options: { readonly jsonPath?: string } = {}, +): ColumnDefinition => ({ type, ...options }) + +type TypeOfColumn = I extends ColumnDefinition ? X : I + +export const getColumnType = (input: AnyTinybirdType | ColumnDefinition): AnyTinybirdType => + isTinybirdType(input) ? input : input.type + +export const getColumnJsonPath = (input: AnyTinybirdType | ColumnDefinition): string | undefined => + isTinybirdType(input) ? input.modifiers.jsonPath : (input.jsonPath ?? input.type.modifiers.jsonPath) + +/** The query columns of a datasource schema. */ +export type ColumnsOfSchema = { + readonly [K in keyof S]: TypeOfColumn extends TinybirdType ? C : never +} + +/** Columns with a `DEFAULT`: an insert may leave them out. */ +export type DefaultedColumnsOfSchema = { + [K in keyof S]: TypeOfColumn extends TinybirdType ? K : never +}[keyof S] & + string + +// Engines + +interface BaseMergeTreeConfig { + readonly sortingKey: string | ReadonlyArray + readonly partitionKey?: string + readonly primaryKey?: string | ReadonlyArray + readonly ttl?: string + readonly settings?: Readonly> +} + +export type EngineConfig = + | (BaseMergeTreeConfig & { readonly type: "MergeTree" }) + | (BaseMergeTreeConfig & { readonly type: "ReplacingMergeTree"; readonly ver?: string; readonly isDeleted?: string }) + | (BaseMergeTreeConfig & { readonly type: "SummingMergeTree"; readonly columns?: ReadonlyArray }) + | (BaseMergeTreeConfig & { readonly type: "AggregatingMergeTree" }) + | (BaseMergeTreeConfig & { readonly type: "CollapsingMergeTree"; readonly sign: string }) + | (BaseMergeTreeConfig & { + readonly type: "VersionedCollapsingMergeTree" + readonly sign: string + readonly version: string + }) + | { readonly type: "Null" } + +type ConfigOf = Omit, "type"> + +export const engine = { + mergeTree: (config: ConfigOf<"MergeTree">): EngineConfig => ({ type: "MergeTree", ...config }), + replacingMergeTree: (config: ConfigOf<"ReplacingMergeTree">): EngineConfig => ({ + type: "ReplacingMergeTree", + ...config, + }), + summingMergeTree: (config: ConfigOf<"SummingMergeTree">): EngineConfig => ({ type: "SummingMergeTree", ...config }), + aggregatingMergeTree: (config: ConfigOf<"AggregatingMergeTree">): EngineConfig => ({ + type: "AggregatingMergeTree", + ...config, + }), + collapsingMergeTree: (config: ConfigOf<"CollapsingMergeTree">): EngineConfig => ({ + type: "CollapsingMergeTree", + ...config, + }), + versionedCollapsingMergeTree: (config: ConfigOf<"VersionedCollapsingMergeTree">): EngineConfig => ({ + type: "VersionedCollapsingMergeTree", + ...config, + }), + null: (): EngineConfig => ({ type: "Null" }), +} as const + +/** Split `"a, f(b, c)"` into `["a", "f(b, c)"]` at top-level commas. */ +export const splitKey = (key: string | ReadonlyArray): ReadonlyArray => { + if (typeof key !== "string") return key + const parts: Array = [] + let depth = 0 + let quoted = false + let start = 0 + for (let i = 0; i < key.length; i++) { + const ch = key[i] + if (ch === "'" && key[i - 1] !== "\\") quoted = !quoted + else if (!quoted && ch === "(") depth++ + else if (!quoted && ch === ")") depth-- + else if (!quoted && depth === 0 && ch === ",") { + parts.push(key.slice(start, i).trim()) + start = i + 1 + } + } + parts.push(key.slice(start).trim()) + return parts.filter((part) => part.length > 0) +} + +const engineSpec = (config: EngineConfig): EngineSpec => { + switch (config.type) { + case "MergeTree": + return Define.engine.mergeTree() + case "ReplacingMergeTree": + return Define.engine.replacingMergeTree({ + ...(config.ver !== undefined ? { version: config.ver } : {}), + ...(config.isDeleted !== undefined ? { isDeleted: config.isDeleted } : {}), + }) + case "SummingMergeTree": + return Define.engine.summingMergeTree(config.columns !== undefined ? { columns: config.columns } : {}) + case "AggregatingMergeTree": + return Define.engine.aggregatingMergeTree() + case "CollapsingMergeTree": + return Define.engine.collapsingMergeTree(config.sign) + case "VersionedCollapsingMergeTree": + return Define.engine.versionedCollapsingMergeTree(config.sign, config.version) + case "Null": + return Define.engine.null() + } +} + +const settingValue = (value: string | number | boolean): string | number => + typeof value === "string" ? `'${value.replace(/'/g, "\\'")}'` : typeof value === "boolean" ? (value ? 1 : 0) : value + +// Datasources + +export interface DatasourceIndex { + readonly name: string + readonly expr: string + readonly type: string + readonly granularity: number +} + +export interface DatasourceOptions { + readonly description?: string + readonly schema: S + /** Defaults to a `MergeTree` ordered by `tuple()`. */ + readonly engine?: EngineConfig + /** `false` leaves the `json:$` paths out, for a datasource only views write to. */ + readonly jsonPaths?: boolean + readonly forwardQuery?: string + readonly indexes?: ReadonlyArray + /** The column carrying row-level tenancy; see docs/tenant-scoping.md. */ + readonly tenantColumn?: keyof S & string +} + +export const DatasourceTypeId: unique symbol = Symbol.for("@maple-dev/effect-orm/tinybird/Datasource") +export type DatasourceTypeId = typeof DatasourceTypeId + +export interface Datasource + extends SchemaTable, DefaultedColumnsOfSchema, never> { + readonly [DatasourceTypeId]: DatasourceTypeId + readonly _name: Name + readonly _schema: S + readonly options: DatasourceOptions +} + +/** Any datasource, whatever its schema: what generators and views take. */ +export type AnyDatasource = Datasource + +/** A JSON row as Tinybird ingests it into the datasource. */ +export type InferRow = D extends Datasource ? { [K in keyof S]: RowOf> } : never + +export const isDatasourceDefinition = (value: unknown): value is AnyDatasource => + typeof value === "object" && value !== null && DatasourceTypeId in value + +/** A literal `DEFAULT`, written the way Tinybird datafiles write it. */ +export const formatDefaultValue = (value: unknown, type: string): string => { + if (value === null) return "NULL" + if (typeof value === "string") return `'${value.replace(/'/g, "\\'")}'` + if (typeof value === "number" || typeof value === "bigint") return String(value) + if (typeof value === "boolean") return value ? "1" : "0" + if (value instanceof Date) { + return type.startsWith("Date") && !type.includes("Time") + ? `'${value.toISOString().split("T")[0]}'` + : `'${value.toISOString().replace("T", " ").slice(0, 19)}'` + } + if (typeof value === "object") return JSON.stringify(value) + return `'${String(value).replace(/'/g, "\\'")}'` +} + +/** The `DEFAULT` expression of a column type, if it has one. */ +export const defaultSqlOf = (type: AnyTinybirdType): string | undefined => + type.modifiers.defaultExpression !== undefined + ? type.modifiers.defaultExpression + : type.modifiers.defaultValue !== undefined + ? formatDefaultValue(type.modifiers.defaultValue, getTinybirdType(type)) + : undefined + +export const defineDatasource = ( + name: Name, + options: DatasourceOptions, +): Datasource => { + for (const index of options.indexes ?? []) { + if (!Number.isInteger(index.granularity) || index.granularity <= 0) { + throw new SchemaDefinitionDefect({ + object: `${name}.${index.name}`, + message: "index granularity must be a positive integer", + }) + } + } + const config = options.engine + const columns = Object.fromEntries( + Object.entries(options.schema).map(([key, input]) => { + const type = getColumnType(input) + const defaultExpr = defaultSqlOf(type) + const spec: Define.ColumnSpec = { + _tag: "ColumnSpec", + type: type.column, + options: { + ...(defaultExpr !== undefined ? { defaultExpr } : {}), + ...(type.modifiers.codec !== undefined ? { codec: type.modifiers.codec } : {}), + }, + } + return [key, spec] + }), + ) + const keys = + config === undefined + ? { orderBy: [] } + : config.type === "Null" + ? {} + : { + orderBy: () => splitKey(config.sortingKey), + ...(config.partitionKey !== undefined ? { partitionBy: config.partitionKey } : {}), + ...(config.primaryKey !== undefined ? { primaryKey: () => splitKey(config.primaryKey!) } : {}), + ...(config.ttl !== undefined ? { ttl: config.ttl } : {}), + ...(config.settings !== undefined + ? { + settings: Object.fromEntries( + Object.entries(config.settings).map(([k, v]) => [k, settingValue(v)]), + ), + } + : {}), + } + const table = Define.defineTable(name, { + columns, + engine: config === undefined ? Define.engine.mergeTree() : engineSpec(config), + ...keys, + ...(options.indexes !== undefined && config?.type !== "Null" + ? { indexes: options.indexes.map((index) => Define.index(index.name, index.expr, index.type, index.granularity)) } + : {}), + ...(options.tenantColumn !== undefined ? { tenantColumn: options.tenantColumn } : {}), + }) + return Object.assign(table, { + [DatasourceTypeId]: DatasourceTypeId, + _name: name, + _schema: options.schema, + options, + }) as unknown as Datasource +} + +// Materialized views + +export interface NodeDefinition { + readonly _name: string + readonly sql: string + readonly description?: string +} + +const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/ + +export const node = (options: { readonly name: string; readonly sql: string; readonly description?: string }): NodeDefinition => { + if (!IDENTIFIER.test(options.name)) { + throw new SchemaDefinitionDefect({ object: options.name, message: "a node name must be a plain identifier" }) + } + return { + _name: options.name, + sql: options.sql, + ...(options.description !== undefined ? { description: options.description } : {}), + } +} + +export const PipeTypeId: unique symbol = Symbol.for("@maple-dev/effect-orm/tinybird/Pipe") +export type PipeTypeId = typeof PipeTypeId + +export interface MaterializedViewDefinition { + readonly [PipeTypeId]: PipeTypeId + readonly _name: Name + readonly _type: "pipe" + readonly options: { + readonly description?: string + readonly nodes: ReadonlyArray + readonly materialized: { readonly datasource: AnyDatasource; readonly deploymentMethod?: "alter" } + } +} + +export const isPipeDefinition = (value: unknown): value is MaterializedViewDefinition => + typeof value === "object" && value !== null && PipeTypeId in value + +export const defineMaterializedView = ( + name: Name, + options: { + readonly description?: string + readonly datasource: AnyDatasource + readonly nodes: ReadonlyArray + /** `alter` changes the view in place instead of recreating its target. */ + readonly deploymentMethod?: "alter" + }, +): MaterializedViewDefinition => { + if (!IDENTIFIER.test(name)) { + throw new SchemaDefinitionDefect({ object: name, message: "a pipe name must be a plain identifier" }) + } + if (options.nodes.length === 0) { + throw new SchemaDefinitionDefect({ object: name, message: "a materialized view needs a node" }) + } + return { + [PipeTypeId]: PipeTypeId, + _name: name, + _type: "pipe", + options: { + ...(options.description !== undefined ? { description: options.description } : {}), + nodes: options.nodes, + materialized: { + datasource: options.datasource, + ...(options.deploymentMethod !== undefined ? { deploymentMethod: options.deploymentMethod } : {}), + }, + }, + } +} diff --git a/src/tinybird/tinybird.test.ts b/src/tinybird/tinybird.test.ts new file mode 100644 index 0000000..3f6982b --- /dev/null +++ b/src/tinybird/tinybird.test.ts @@ -0,0 +1,183 @@ +import { Schema } from "effect" +import { describe, expect, expectTypeOf, it } from "vitest" +import * as CH from "../clickhouse" +import * as S from "../schema" +import * as TB from "../tinybird" +import { t } from "../tinybird" + +const OrgId = Schema.String.pipe(Schema.brand("OrgId")) + +const events = TB.defineDatasource("events", { + description: "Raw events.", + schema: { + OrgId: TB.column(t.string().lowCardinality().brand(OrgId), { jsonPath: "$.org_id" }), + Timestamp: t.dateTime64(9), + Kind: t.string().lowCardinality().default("click"), + Attributes: t.map(t.string().lowCardinality(), t.string()), + Tags: t.array(t.string()).jsonPath("$.tags[:]"), + Score: t.float64().nullable(), + Items: t.array(t.string()).defaultExpr("arrayMap(k -> lower(k), mapKeys(Attributes))"), + Body: t.string().codec("ZSTD(1)"), + }, + engine: TB.engine.mergeTree({ + partitionKey: "toDate(Timestamp)", + sortingKey: "OrgId, toStartOfHour(Timestamp), Kind", + ttl: "toDate(Timestamp) + INTERVAL 30 DAY", + }), + indexes: [{ name: "idx_kind", expr: "Kind", type: "set(100)", granularity: 4 }], + forwardQuery: "SELECT *\n FROM events", + tenantColumn: "OrgId", +}) + +const hourly = TB.defineDatasource("events_hourly", { + jsonPaths: false, + schema: { + OrgId: t.string().lowCardinality(), + Hour: t.dateTime(), + Calls: t.simpleAggregateFunction("sum", t.uint64()), + Latency: t.aggregateFunction("quantilesTDigestWeighted(0.5, 0.95), UInt64", t.uint32()), + }, + engine: TB.engine.aggregatingMergeTree({ sortingKey: ["OrgId", "Hour"], ttl: "Hour + INTERVAL 1 DAY" }), +}) + +const hourlyMv = TB.defineMaterializedView("events_hourly_mv", { + description: "Rolls events up by hour.", + datasource: hourly, + nodes: [ + TB.node({ + name: "events_hourly_node", + sql: ` + SELECT OrgId, toStartOfHour(Timestamp) AS Hour + FROM events + `, + }), + ], + deploymentMethod: "alter", +}) + +describe("datafiles", () => { + it("writes a datasource as the SDK does", () => { + expect(TB.generateDatasource(events).content).toBe( + [ + "DESCRIPTION >", + " Raw events.", + "", + "SCHEMA >", + " OrgId LowCardinality(String) `json:$.org_id`,", + " Timestamp DateTime64(9) `json:$.Timestamp`,", + " Kind LowCardinality(String) `json:$.Kind` DEFAULT 'click',", + " Attributes Map(LowCardinality(String), String) `json:$.Attributes`,", + " Tags Array(String) `json:$.tags[:]`,", + " Score Nullable(Float64) `json:$.Score`,", + " Items Array(String) `json:$.Items` DEFAULT arrayMap(k -> lower(k), mapKeys(Attributes)),", + " Body String `json:$.Body` CODEC(ZSTD(1))", + "", + 'ENGINE "MergeTree"', + 'ENGINE_PARTITION_KEY "toDate(Timestamp)"', + 'ENGINE_SORTING_KEY "OrgId, toStartOfHour(Timestamp), Kind"', + 'ENGINE_TTL "toDate(Timestamp) + INTERVAL 30 DAY"', + "", + "INDEXES >", + " idx_kind Kind TYPE set(100) GRANULARITY 4", + "", + "FORWARD_QUERY >", + " SELECT *", + " FROM events", + ].join("\n"), + ) + }) + + it("leaves json paths out and writes aggregate types", () => { + expect(TB.generateDatasource(hourly).content).toBe( + [ + "SCHEMA >", + " OrgId LowCardinality(String),", + " Hour DateTime,", + " Calls SimpleAggregateFunction(sum, UInt64),", + " Latency AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32)", + "", + 'ENGINE "AggregatingMergeTree"', + 'ENGINE_SORTING_KEY "OrgId, Hour"', + 'ENGINE_TTL "Hour + INTERVAL 1 DAY"', + ].join("\n"), + ) + }) + + it("writes Null and ReplacingMergeTree engines", () => { + const ingest = TB.defineDatasource("ingest", { schema: { OrgId: t.string() }, engine: TB.engine.null() }) + const latest = TB.defineDatasource("latest", { + schema: { OrgId: t.string(), Version: t.uint64() }, + engine: TB.engine.replacingMergeTree({ sortingKey: ["OrgId"], ver: "Version" }), + }) + expect(TB.generateDatasource(ingest).content).toBe("SCHEMA >\n OrgId String `json:$.OrgId`\n\nENGINE Null") + expect(TB.generateDatasource(latest).content).toContain('ENGINE_SORTING_KEY "OrgId"\nENGINE_VER "Version"') + }) + + it("writes a materialized view pipe", () => { + expect(TB.generatePipe(hourlyMv).content).toBe( + [ + "DESCRIPTION >", + " Rolls events up by hour.", + "", + "NODE events_hourly_node", + "SQL >", + " SELECT OrgId, toStartOfHour(Timestamp) AS Hour", + " FROM events", + "", + "TYPE MATERIALIZED", + "DATASOURCE events_hourly", + "DEPLOYMENT_METHOD alter", + ].join("\n"), + ) + }) + + it("collects a project from module namespaces in export order", () => { + const project = TB.buildProject({ b: hourly, a: events, view: hourlyMv, other: 1 }, { again: events }) + expect(project.datasources.map((d) => d.name)).toEqual(["events_hourly", "events"]) + expect(project.pipes.map((p) => p.name)).toEqual(["events_hourly_mv"]) + }) +}) + +describe("a datasource is a table", () => { + it("is queried with its branded tenant column", () => { + const { sql } = CH.compileUnsafe( + CH.from(events) + .select("Kind", "Timestamp") + .where(($) => [$.OrgId.eq(CH.param.of(events.columns.OrgId, "orgId"))]), + { orgId: OrgId.make("org_1") }, + ) + expect(sql).toContain("FROM events") + expect(events.tenantColumn).toBe("OrgId") + expectTypeOf>().toEqualTypeOf() + expectTypeOf>().toEqualTypeOf() + }) + + it("renders ClickHouse DDL", () => { + const [table] = S.renderSchema(S.entitiesOf([events])) + expect(table).toContain("\tOrgId LowCardinality(String),") + expect(table).toContain("\tKind LowCardinality(String) DEFAULT 'click',") + expect(table).toContain("\tBody String CODEC(ZSTD(1)),") + expect(table).toContain("ORDER BY (OrgId, toStartOfHour(Timestamp), Kind)") + expect(table).toContain("TTL toDate(Timestamp) + INTERVAL 30 DAY") + }) + + it("infers the ingested JSON row", () => { + type Row = TB.InferRow + expectTypeOf().toEqualTypeOf() + expectTypeOf().toEqualTypeOf() + expectTypeOf().toEqualTypeOf() + expectTypeOf().toEqualTypeOf>() + expectTypeOf().toEqualTypeOf>() + type Rollup = TB.InferRow + expectTypeOf().toEqualTypeOf() + expectTypeOf().toEqualTypeOf() + }) + + it("reads column metadata", () => { + expect(TB.getColumnJsonPath(events.options.schema.OrgId)).toBe("$.org_id") + expect(TB.getColumnJsonPath(events.options.schema.Tags)).toBe("$.tags[:]") + expect(TB.getTinybirdType(TB.getColumnType(events.options.schema.Score))).toBe("Nullable(Float64)") + expect(TB.isDatasourceDefinition(events)).toBe(true) + expect(TB.isDatasourceDefinition(hourlyMv)).toBe(false) + }) +}) diff --git a/src/tinybird/types.ts b/src/tinybird/types.ts new file mode 100644 index 0000000..4c536cb --- /dev/null +++ b/src/tinybird/types.ts @@ -0,0 +1,104 @@ +// Tinybird column types: the `t` builder of `@tinybirdco/sdk`, over the +// ClickHouse column types. Each one carries the query column (`column`), the +// datafile modifiers, and the JSON row type Tinybird ingests (`InferRow`). + +import type { Schema } from "effect" +import * as T from "../ch/types" + +export const TypeId: unique symbol = Symbol.for("@maple-dev/effect-orm/tinybird/Type") +export type TypeId = typeof TypeId + +export interface TypeModifiers { + readonly defaultValue?: unknown + readonly defaultExpression?: string + readonly codec?: string + readonly jsonPath?: string +} + +type AnyColumn = T.CHType + +/** + * A Tinybird column type. `Row` is the value in an ingested JSON row (a + * `DateTime` is its string form); `Defaulted` records a `DEFAULT`, so an + * insert may leave the column out. + */ +export interface TinybirdType { + readonly [TypeId]: TypeId + /** The query column this type declares: what the table's `columns` hold. */ + readonly column: C + readonly modifiers: TypeModifiers + readonly _row?: Row + readonly _defaulted?: Defaulted + nullable(): TinybirdType, Row | null, Defaulted> + lowCardinality(): TinybirdType + default(value: Row): TinybirdType + defaultExpr(expression: string): TinybirdType + codec(codec: string): TinybirdType + jsonPath(path: string): TinybirdType + /** Narrows the query column (`T.brand`); the ingested row keeps its wire type. */ + brand(schema: Schema.Codec>): TinybirdType>, Row, Defaulted> +} + +export type AnyTinybirdType = TinybirdType +export type RowOf = X extends TinybirdType ? Row : never + +const make = ( + column: C, + modifiers: TypeModifiers = {}, +): TinybirdType => { + const { defaultValue: _value, defaultExpression: _expression, ...rest } = modifiers + return { + [TypeId]: TypeId, + column, + modifiers, + nullable: () => make, Row | null, Defaulted>(T.nullable(column), modifiers), + lowCardinality: () => make(T.lowCardinality(column), modifiers), + default: (value) => make(column, { ...rest, defaultValue: value }), + defaultExpr: (expression) => make(column, { ...rest, defaultExpression: expression.trim() }), + codec: (codec) => make(column, { ...modifiers, codec }), + jsonPath: (path) => make(column, { ...modifiers, jsonPath: path }), + brand: (schema) => make(T.brand(column, schema), modifiers), + } +} + +export const isTinybirdType = (value: unknown): value is AnyTinybirdType => + typeof value === "object" && value !== null && TypeId in value + +type AggregateRow> = Args extends readonly [infer First, ...Array] + ? RowOf + : unknown + +/** The `t` builder of `@tinybirdco/sdk`, for the types a ClickHouse column type exists for. */ +export const t = { + string: () => make(T.string), + bool: () => make(T.bool), + uint8: () => make(T.uint8), + uint16: () => make(T.uint16), + uint32: () => make(T.uint32), + uint64: () => make(T.uint64), + int32: () => make(T.int32), + int64: () => make(T.int64), + float64: () => make(T.float64), + /** Read and ingested as the string ClickHouse writes, as `@tinybirdco/sdk` types it. */ + dateTime: () => make(T.dateTimeString), + dateTime64: (precision: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 = 3) => + make(T.precision(T.dateTime64String, precision)), + array: (element: E) => + make, Array>>(T.array(element.column)), + map: , string, any>, V extends AnyTinybirdType>( + key: K, + value: V, + ) => make, Record, RowOf>>(T.map(key.column, value.column)), + simpleAggregateFunction: (fn: string, type: E) => + make>(T.simpleAggregateFunction(fn, type.column)), + /** `fn` is written as-is, so it may carry parameters and leading argument types. */ + aggregateFunction: >(fn: string, ...types: Args) => + make, AggregateRow>( + T.aggregateState(fn, ...types.map((type) => type.column.sql)), + ), +} as const + +/** The ClickHouse type a column is written as in a datafile, e.g. `LowCardinality(String)`. */ +export const getTinybirdType = (type: AnyTinybirdType): string => type.column.sql + +export const getModifiers = (type: AnyTinybirdType): TypeModifiers => type.modifiers diff --git a/tests/dialect-cases.ts b/tests/dialect-cases.ts index 406810d..82e6c48 100644 --- a/tests/dialect-cases.ts +++ b/tests/dialect-cases.ts @@ -732,6 +732,14 @@ const typeFixtures = [ ["array", CH.array(CH.nullable(CH.int64)), "[toNullable(toInt64(42)), NULL]", [42, null]], ["map", CH.map(CH.string, CH.array(CH.uint64)), "map('key', [toUInt64(42)])", { key: [42] }], ["nullable", CH.nullable(CH.string), "CAST(NULL AS Nullable(String))", null], + ["lowCardinality", CH.lowCardinality(CH.string), "toLowCardinality('hello')", "hello"], + ["simpleAggregateFunction", CH.simpleAggregateFunction("sum", CH.uint64), "toUInt64(42)", 42], + [ + "precision", + CH.precision(CH.dateTime64String, 9), + "toDateTime64('2026-01-01 00:00:00.123456789', 9, 'UTC')", + "2026-01-01 00:00:00.123456789", + ], // A brand over UInt64 keeps the base codec, so a quoted 64-bit value still decodes. ["brand", CH.brand(CH.uint64, Schema.Number.pipe(Schema.brand("Count"))), "toUInt64(42)", 42], ] as const diff --git a/tests/dialect-coverage.test.ts b/tests/dialect-coverage.test.ts index 21b23c8..e6f2828 100644 --- a/tests/dialect-coverage.test.ts +++ b/tests/dialect-coverage.test.ts @@ -37,8 +37,11 @@ const typeConstructors = [ "float64", "int32", "int64", + "lowCardinality", "map", "nullable", + "precision", + "simpleAggregateFunction", "string", "uint8", "uint16", diff --git a/tsdown.config.ts b/tsdown.config.ts index e90eb1e..eddaa4f 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -7,6 +7,7 @@ export default defineConfig({ sql: "./src/sql/index.ts", postgres: "./src/postgres.ts", schema: "./src/schema.ts", + tinybird: "./src/tinybird.ts", migrate: "./src/migrate.ts", database: "./src/database.ts", kit: "./src/kit.ts", From 3d1c81650b4c026a4c9c1801d1db308a22630b9f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 7 Oct 2026 09:56:12 +0200 Subject: [PATCH 2/2] Give datasources of any schema one supertype AnyDatasource is an interface, not Datasource: the stored options no longer tie tenantColumn to the schema's keys, so a concrete datasource is assignable to it and its schema keeps a real column type. --- src/tinybird/datasource.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/tinybird/datasource.ts b/src/tinybird/datasource.ts index accc6e9..031ae22 100644 --- a/src/tinybird/datasource.ts +++ b/src/tinybird/datasource.ts @@ -4,6 +4,7 @@ import * as Define from "../schema/define" import { SchemaDefinitionDefect, type SchemaTable } from "../schema/define" +import type { ColumnDefs } from "../ch/types" import type { EngineSpec } from "../schema/entities" import { getTinybirdType, isTinybirdType, type AnyTinybirdType, type RowOf, type TinybirdType } from "./types" @@ -163,11 +164,20 @@ export interface Datasource + readonly options: StoredOptions +} + +type StoredOptions = Omit, "tenantColumn"> & { + readonly tenantColumn?: string } /** Any datasource, whatever its schema: what generators and views take. */ -export type AnyDatasource = Datasource +export interface AnyDatasource extends SchemaTable { + readonly [DatasourceTypeId]: DatasourceTypeId + readonly _name: string + readonly _schema: SchemaDefinition + readonly options: StoredOptions +} /** A JSON row as Tinybird ingests it into the datasource. */ export type InferRow = D extends Datasource ? { [K in keyof S]: RowOf> } : never