diff --git a/CHANGELOG.md b/CHANGELOG.md index 618ef4c1..cec4325a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,8 +43,9 @@ ALTER EXTENSION pg_clickhouse UPDATE TO '0.11'; ### ⬆️ Dependencies * Dropped support for PostgreSQL 13. -* Updated the vendored pg-clickhouse-c, which now parses `Nested`, - `SimpleAggregateFunction`, and parameterized `JSON` types ([#363]). +* Updated the vendored pg-clickhouse-c, which now parses unflattened + `Nested`, `SimpleAggregateFunction`, and parameterized `JSON` types + ([#363]). ### ⚡ Improvements @@ -72,11 +73,11 @@ ALTER EXTENSION pg_clickhouse UPDATE TO '0.11'; the column type and stores the aggregate function name, without parameters, in a column option. Because `AggregateFunction(count)` has no argument type, it imports as `bigint` ([#349], [#363]). -* `IMPORT FOREIGN SCHEMA` now imports `Nested` as `text[][]`, with one array - per nested row. It also imports parameterized `JSON` as `jsonb` by - default, which can be mapped manually to `json` or `text`, and reads - `SimpleAggregateFunction` columns. These types previously caused errors - ([#363]). +* `IMPORT FOREIGN SCHEMA` now imports `Nested` columns created with + `flatten_nested=0` as `text[][]`, with one array per nested row. It also + imports parameterized `JSON` as `jsonb` by default, which can be mapped + manually to `json` or `text`, and reads `SimpleAggregateFunction` columns. + These types previously caused errors ([#363]). * Added pushdown for PostgreSQL `sha224()`, `sha256()`, `sha384()`, and `sha512()` functions, along with supported constant-algorithm calls to the pgcrypto extension's `digest()` function. Thanks to Siva Girish Ramesh for @@ -88,6 +89,12 @@ ALTER EXTENSION pg_clickhouse UPDATE TO '0.11'; * Added pushdown for multidimensional array indexing in `WHERE` clauses (`WHERE foo[1][1] = 'abc'`) ([#373]) +### 📚 Documentation + +* Expanded the [Composite Types] section of the documentation to detail the + behaviors of Array, Tuple, Map, and Nested data type mappings, all of + which have been improved but have caveats. + ### 🐞 Bug Fixes * Fixed inserting `interval` values with HTTP driver. Values are now @@ -149,6 +156,7 @@ ALTER EXTENSION pg_clickhouse UPDATE TO '0.11'; "ClickHouse/pg_clickhouse#373 Add pushdown for multidimensional array index" [#381]: https://github.com/ClickHouse/pg_clickhouse/pull/381 "ClickHouse/pg_clickhouse#381 Remove the `fetch_size` server and table option" + [Composite Types]: doc/pg_clickhouse.md#composite-types ## [v0.10.0] — 2026-08-11 diff --git a/doc/pg_clickhouse.md b/doc/pg_clickhouse.md index 55d77c28..33002686 100644 --- a/doc/pg_clickhouse.md +++ b/doc/pg_clickhouse.md @@ -271,13 +271,14 @@ Some details to keep in mind: `NOT NULL`, whatever the ClickHouse declaration, see [State Columns and `NOT NULL`](#state-columns-and-not-null). -* `Tuple` columns import as `text[]`. `Map` and `Nested` columns import as - `text[][]`. Each conversion emits a `NOTICE`. PostgreSQL's `record` - pseudotype cannot define a table column, so each tuple becomes an array of - fields, each map becomes an array of key-value pairs, and each nested value - becomes an array of rows. To read fields as records, alter the column to an - array of a matching composite type, see [Manual Type - Mappings](#manual-type-mappings). +* `Tuple` columns import as `text[]`. `Map` and `Nested` columns columns + created with `flatten_nested=0` import as `text[][]`. Each conversion + emits a `NOTICE`. PostgreSQL's `record` pseudotype cannot define a table + column, so each tuple becomes an array of values, each map becomes an + two-dimensional array of key-value pairs, and each Nested value becomes an + two-dimensional array of rows. To read fields as records, alter the column + to an array of a matching composite type, see [Manual Type + Mappings](#manual-type-mappings) for an example. * `DateTime64` and `Time64` columns with precision greater than 6 (microseconds) also trigger a `NOTICE`, since PostgreSQL caps precision at @@ -702,7 +703,7 @@ CREATE TABLE events (id int, ts date, val int, amt float8) -- 2023 data lives on ClickHouse CREATE FOREIGN TABLE events_2023 PARTITION OF events FOR VALUES FROM ('2023-01-01') TO ('2024-01-01') - SERVER ch_svr OPTIONS (table_name 'events'); + SERVER ch_srv OPTIONS (table_name 'events'); -- 2024 data stays local CREATE TABLE events_2024 PARTITION OF events @@ -1248,46 +1249,45 @@ drops that trailing padding, where a [BYTEA] column keeps every byte. > As a rule, only use [TEXT] columns for encoded strings and use [BYTEA] > columns only for binary data, and never switch between them. -### Manual Type Mappings +### Composite Types -`IMPORT FOREIGN SCHEMA` uses general-purpose PostgreSQL types. To retain -structure or constrain values, define PostgreSQL composite or enum types and -create a foreign table manually. For example, given ClickHouse columns -`status Enum8('new' = 1, 'done' = 2)`, `point Tuple(Int32, Int32)`, -`labels Map(String, Int64)`, and `items Nested(id Int32, name String)`: +#### Array -```sql -CREATE TYPE event_status AS ENUM ('new', 'done'); -CREATE TYPE event_point AS (x integer, y integer); -CREATE TYPE event_label AS (key text, value bigint); -CREATE TYPE event_item AS (id integer, name text); +ClickHouse [Array]s map directly to PostgreSQL arrays, with equivalent +semantics. The main differences is that Postgres multidimensional arrays must +have array expressions with matching dimensions. An attempt to read a +ClickHouse array with different dimensions, such as `[[1], [2,3]]`, +triggers an error. -CREATE FOREIGN TABLE events ( - status event_status, - point event_point, - labels event_label[], - items event_item[] -) SERVER clickhouse_srv; +Array index access pushes down as appropriate, including multidimensional +index access. Examples: + +```sql +SELECT * FROM things where tags[1] = 'foo'; +SELECT * FROM things where pairs[1][2] = 'bar'; ``` -Match composite field order and types to ClickHouse declarations. Map keys and -values become first and second fields, respectively. `Nested` and `Map` become -arrays of composites, while `Tuple` becomes one composite value. +Array slices also push down using the [arraySlice] function, although +multidimensional slice access is not yet supported: -### MAP +```sql +SELECT id, vals FROM t1 WHERE vals[1:2] = ARRAY[10,20]; +SELECT id, vals FROM t1 WHERE vals[1:2][2] = 20; -- currently fails +``` + +#### Map PostgreSQL provides no type corresponding to the ClickHouse [Map] type. pg_clickhouse therefore maps [Map] columns to `text[][]`, with each key-value -pair as its own array. [IMPORT FOREIGN SCHEMA] uses this mapping. - -Insert an array of that shape, with each value parsed as its corresponding key -or value type: +pair as its own array. [IMPORT FOREIGN SCHEMA](#import-foreign-schema) uses +this mapping. One can `INSERT` maps as arrays, as well. An example: ```sql -- Create ClickHouse table with Map column. CALL clickhouse_perform('ch_srv', $$ CREATE TABLE maps ( - c1 Int32, c2 Map(String, Int64) + c1 Int32, + c2 Map(String, Int64) ) ENGINE = MergeTree ORDER BY (c1); $$); @@ -1304,24 +1304,201 @@ try=# SELECT * FROM maps; (1 row) ``` -Each nested array must contain exactly two values. Values incompatible with -corresponding ClickHouse key or value types trigger an error. +Each nested array must contain exactly two text values. Values incompatible +with corresponding ClickHouse key or value types trigger an error. + +> [!NOTE] +> Inserting a `Map` requires the `binary` driver, which derives column types +> from the CLickHouse sever. The `http` driver does not, so lacks the +> information to format and insert the appropriate value. + +#### Tuple + +Similarly, a ClickHouse [Tuple] columns map to `text[]` and supports `INSERT`s +via the binary driver. `IMPORT FOREIGN SCHEMA` emits a `NOTICE` when it makes +such a mapping. -Similarly, a `Tuple` column maps to `text[]`. To preserve records, define a -matching PostgreSQL [composite type], then use [CREATE FOREIGN TABLE] for a -`Tuple` column with that type. `IMPORT FOREIGN SCHEMA` does not create types, -so it emits a `NOTICE` and uses text arrays instead. +#### Nested -A [Nested] column becomes an array with one item per nested row. Each item -contains that row's fields. An array of a matching composite type can read -these items as records. By default, ClickHouse splits a `Nested` column into -one `Array` column per field. `IMPORT FOREIGN SCHEMA` reads these from +By default, ClickHouse splits a `Nested` column into one `Array` column per +field (`flatten_nested=1`). `IMPORT FOREIGN SCHEMA` reads these from ClickHouse's `system.columns` catalog and creates separate PostgreSQL array -columns, preserving dotted names such as `items.a` and `items.b`. +columns, preserving dotted names such as `items.a` and `items.b`. For example, +given this foreign table: -> [!NOTE] -> Inserting a `Map` requires the `binary` driver, which derives column types -> from the server. The `http` driver builds them from the PostgreSQL column. +```sql +-- Create ClickHouse table with flattened Nested column. +CALL clickhouse_perform('ch_srv', $$ + CREATE TABLE ch.nests ( + c1 Int32, + c2 Nested(id Int64, name String) + ) ENGINE = MergeTree ORDER BY (c1) +$$); + +-- Import text[][] column, then insert two pairs. +IMPORT FOREIGN SCHEMA ch LIMIT TO (nests) FROM SERVER ch_srv INTO public; +``` + +It's imported schema has three columns rather than two: + +``` + Foreign table "public.nests" + Column | Type | Collation | Nullable | Default | FDW options +---------+----------+-----------+----------+---------+------------- + c1 | integer | | not null | | + c2.id | bigint[] | | not null | | + c2.name | text[] | | not null | | +``` + +Query the nested columns by double-quoting the column names, e.g., + +```sql +SELECT ci, "c2.id" FROM nests; +``` + +And filter values in a `WHERE` clause using the usual array features, +including array subscript syntax: + +```sql +SELECT * FROM nests + WHERE "c2.id[2]" = '2' OR '12' = ANY("c2.id"); +``` + +A [Nested] column created with `flatten_nested=0` maps to an array with one +item per nested row. Each array item contains that row's values: + +```sql +-- Create ClickHouse table with unflattened Nested column. +CALL clickhouse_perform('ch_srv', $$ + CREATE TABLE nests ( + c1 Int32, + c2 Nested(id Int64, name String) + ) ENGINE = MergeTree ORDER BY (c1) SETTINGS flatten_nested=0 +$$); + +-- Import text[][] column, then insert two pairs. +IMPORT FOREIGN SCHEMA ch LIMIT TO (nests) FROM SERVER ch_srv INTO public; +``` + +Now the resulting schema is: + +``` + Foreign table "public.nests" + Column | Type | Collation | Nullable | Default | FDW options +--------+---------+-----------+----------+---------+------------- + c1 | integer | | not null | | + c2 | text[] | | not null | | +``` + +Compose nested records in a two-dimensional array with text values formatted +for each type defined by the ClickHouse Column. For +`c2 Nested(id Int64, name String)` in this example, it would be: + +```sql +INSERT INTO nests +VALUES (1, ARRAY[['42', 'Arthur'], ['99', 'Barbara']]); +``` + +Pushdown of `text[][]` columns over Nested types fails, however: + +```pgsql +SELECT * FROM nests WHERE c2[1][1] = '42` +ERROR: pg_clickhouse: DB::Exception: First argument for function 'arrayElement' must be array, got 'Tuple(serial UInt32, order_id String)' instead +``` + +This is because pg_clickhouse cannot tell a text array column over a +ClickHouse array column from one over a Nested column, so cannot rewrite it +in ClickHouse's Nested syntax. + +However, an array of a matching composite type can also map these items (see +[Manual Type Mappings](#manual-type-mappings) for details), in which case +pushdown works as long as the composite field names are identical to the +Nested field names: + +```sql +SELECT * FROM nests WHERE c2[1].id = 42; +``` + +### Manual Type Mappings + +`IMPORT FOREIGN SCHEMA` uses general-purpose PostgreSQL types. For example, +given a ClickHouse table using [Enum], [Tuple], `Map`, and unflattened +[Nested] (`flatten_nested = 0`) columns, such as: + +```sql +CALL clickhouse_perform('ch_srv', $$ + CREATE TABLE events ( + id UInt32, + status Enum8('new' = 1, 'done' = 2), + point Tuple(Int32, Int32), + labels Map(String, Int64), + items Nested(id Int32, name String) + ) ORDER BY id SETTINGS flatten_nested = 0 +$$); + +CALL clickhouse_perform('ch_srv', $$ + INSERT INTO events + VALUES(1, 'new', tuple(3, 4), {'k1': 5, 'k2': 6}, [tuple(100, 'xx')]) +$$); +``` + +To retain structure or constrain values, define PostgreSQL composite or enum +types and create a foreign table manually: + +```sql +CREATE TYPE event_status AS ENUM ('new', 'done'); +CREATE TYPE event_point AS (x integer, y integer); +CREATE TYPE event_label AS (key text, value bigint); +CREATE TYPE event_item AS (id integer, name text); + +CREATE FOREIGN TABLE events ( + id bigint, + status event_status, + point event_point, + labels event_label[], + items event_item[] +) SERVER clickhouse_srv; +``` + +> [!TIP] +> Always make the enum labels and composite type field names identical to the +> ClickHouse enum labels and `Nested` or `Tuple` field names to ensure that +> pushdown specifies the proper names. + +Match composite field order and types to ClickHouse declarations. [Map] keys +and values become first and second fields, respectively (`key` and `value` in +this case). `Nested` and `Map` become arrays of composites, while `Tuple` +becomes one composite value: + +```pgsql +SELECT * FROM events ORDER BY id; + id | status | point | labels | items +----+--------+-------+---------------------+------------------------- + 1 | new | (3,4) | {"(k1,5)","(k2,6)"} | {"(100,xx)","(101,yy)"} +(1 row) +``` + +Of course you can use the composite type field names, too, both in a `SELECT` +list: + +```pgsql +SELECT (point).x, (point).y, + (labels[1]).key, (labels[1]).value, + (items[1]).id, (items[1]).name + FROM events ORDER BY id; + x | y | key | value | id | name +---+---+-----+-------+-----+------ + 3 | 4 | k1 | 5 | 100 | xx +``` + +And, for [Nested] types in a `WHERE` clause --- as long as the field names are +identical: + +```sql +SELECT * FROM events WHERE items[1].name = 'xx'; +``` + +`INSERT` using such composites is not yet supported. ## Function and Operator Reference @@ -1981,10 +2158,16 @@ Copyright (c) 2025-2026, ClickHouse. "PostgreSQL Docs: Declaring Composite Types" [CALL]: https://www.postgresql.org/docs/current/sql-call.html "PostgreSQL Docs: CALL" + [Array] https://clickhouse.com/docs/reference/data-types/array + "ClickHouse Docs: Array(T)" [Map]: https://clickhouse.com/docs/sql-reference/data-types/map "ClickHouse Docs: Map" + [Tuple]: https://clickhouse.com/docs/reference/data-types/tuple + "ClickHouse Docs: Tuple(T1, T2, ...)" [Nested]: https://clickhouse.com/docs/sql-reference/data-types/nested-data-structures/nested "ClickHouse Docs: Nested" + [Enum]: https://clickhouse.com/docs/reference/data-types/enum + "ClickHouse Docs: Enum" [String]: https://clickhouse.com/docs/sql-reference/data-types/string "ClickHouse Docs: String" [TEXT]: https://www.postgresql.org/docs/current/datatype-character.html diff --git a/test/expected/nested.out b/test/expected/nested.out new file mode 100644 index 00000000..663fa1be --- /dev/null +++ b/test/expected/nested.out @@ -0,0 +1,322 @@ +\set ECHO errors +/****************************************************************************/ +-- By default ClickHouse flattens a Nested column. +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.visits( + visit_id UInt64, + user_id UInt64, + goals Nested( + serial UInt32, + order_id String + ) + ) + ENGINE = MergeTree ORDER BY visit_id +$$); +-- By default, a Nested column is flattened into multiple columns. +IMPORT FOREIGN SCHEMA nested_test FROM SERVER binary_nested_loopback INTO nest_bin; +EXECUTE describe('nest_bin.visits'); + attname | type | attndims | attnotnull +----------------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals.serial | bigint[] | 1 | t + goals.order_id | text[] | 1 | t +(4 rows) + +IMPORT FOREIGN SCHEMA nested_test FROM SERVER http_nested_loopback INTO nest_http; +EXECUTE describe('nest_http.visits'); + attname | type | attndims | attnotnull +----------------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals.serial | bigint[] | 1 | t + goals.order_id | text[] | 1 | t +(4 rows) + +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (1, 1, '{1,2}'::bigint[],'{xx,yy}'::text[]); +INSERT INTO nest_bin.visits +VALUES (2, 2, '{3,4}'::bigint[],'{aa,bb}'::text[]); +SELECT * FROM nest_bin.visits ORDER BY visit_id; + visit_id | user_id | goals.serial | goals.order_id +----------+---------+--------------+---------------- + 1 | 1 | {1,2} | {xx,yy} + 2 | 2 | {3,4} | {aa,bb} +(2 rows) + +SELECT * FROM nest_http.visits ORDER BY visit_id; + visit_id | user_id | goals.serial | goals.order_id +----------+---------+--------------+---------------- + 1 | 1 | {1,2} | {xx,yy} + 2 | 2 | {3,4} | {aa,bb} +(2 rows) + +-- Should pushdown array access. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE "goals.serial"[1] = 1; + QUERY PLAN +----------------------------------------------------------------------------------------- + Foreign Scan on nest_bin.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE ((("goals.serial"[1]) = 1)) +(3 rows) + +SELECT visit_id FROM nest_bin.visits WHERE "goals.serial"[1] = 1; + visit_id +---------- + 1 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE "goals.serial"[1] = 1; + QUERY PLAN +----------------------------------------------------------------------------------------- + Foreign Scan on nest_http.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE ((("goals.serial"[1]) = 1)) +(3 rows) + +SELECT visit_id FROM nest_http.visits WHERE "goals.serial"[1] = 1; + visit_id +---------- + 1 +(1 row) + +\set ECHO errors +/****************************************************************************/ +-- Recreate everything with flatten_nested=0 +DROP FOREIGN TABLE nest_bin.visits; +DROP FOREIGN TABLE nest_http.visits; +CALL clickhouse_perform('nested_admin', 'DROP TABLE nested_test.visits'); +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.visits( + visit_id UInt64, + user_id UInt64, + goals Nested( + serial UInt32, + order_id String + ) + ) + ENGINE = MergeTree ORDER BY visit_id SETTINGS flatten_nested = 0 +$$); +IMPORT FOREIGN SCHEMA nested_test FROM SERVER binary_nested_loopback INTO nest_bin; +NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "goals"; please create composite type and alter the column if needed +EXECUTE describe('nest_bin.visits'); + attname | type | attndims | attnotnull +----------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals | text[] | 2 | t +(3 rows) + +IMPORT FOREIGN SCHEMA nested_test FROM SERVER http_nested_loopback INTO nest_http; +NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "goals"; please create composite type and alter the column if needed +EXECUTE describe('nest_http.visits'); + attname | type | attndims | attnotnull +----------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals | text[] | 2 | t +(3 rows) + +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (1, 1, '{{1,xx},{2,yy}}'::text[]); +INSERT INTO nest_bin.visits +VALUES (2, 2, '{{3,aa},{4,bb}}'::text[]); +SELECT * FROM nest_bin.visits ORDER BY visit_id; + visit_id | user_id | goals +----------+---------+----------------- + 1 | 1 | {{1,xx},{2,yy}} + 2 | 2 | {{3,aa},{4,bb}} +(2 rows) + +SELECT * FROM nest_http.visits ORDER BY visit_id; + visit_id | user_id | goals +----------+---------+----------------- + 1 | 1 | {{1,xx},{2,yy}} + 2 | 2 | {{3,aa},{4,bb}} +(2 rows) + +-- Pushdown multidimensional array access fails. Failure expected: we would +-- need to know that the column is Nested and thus should be converted to +-- `tupleElement(goals[1], 1) = '1'`. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE goals[1][1] = '1'; + QUERY PLAN +------------------------------------------------------------------------------------- + Foreign Scan on nest_bin.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE (((goals[1][1]) = '1')) +(3 rows) + +SELECT visit_id FROM nest_bin.visits WHERE goals[1][1] = '1'; +ERROR: pg_clickhouse: DB::Exception: First argument for function 'arrayElement' must be array, got 'Tuple(serial UInt32, order_id String)' instead: In scope SELECT visit_id FROM nested_test.visits WHERE (((goals[1])[1]) = '1') +DETAIL: Remote Query: SELECT visit_id FROM nested_test.visits WHERE (((goals[1][1]) = '1')) +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE goals[1][1] = '1'; + QUERY PLAN +------------------------------------------------------------------------------------- + Foreign Scan on nest_http.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE (((goals[1][1]) = '1')) +(3 rows) + +SELECT visit_id FROM nest_http.visits WHERE goals[1][1] = '1'; +ERROR: pg_clickhouse: Code: 43. DB::Exception: First argument for function 'arrayElement' must be array, got 'Tuple(serial UInt32, order_id String)' instead: In scope SELECT visit_id FROM nested_test.visits WHERE (((goals[1])[1]) = '1'). (ILLEGAL_TYPE_OF_ARGUMENT) +DETAIL: Remote Query: SELECT visit_id FROM nested_test.visits WHERE (((goals[1][1]) = '1')) +CONTEXT: HTTP status code: 500 +/****************************************************************************/ +-- Create them manually with a composite type; +CREATE TYPE goal_type AS (serial bigint, order_id text); +DROP FOREIGN TABLE nest_bin.visits; +CREATE FOREIGN TABLE nest_bin.visits( + visit_id numeric(20,0) NOT NULL, + user_id numeric(20,0) NOT NULL, + goals goal_type[] NOT NULL +) SERVER binary_nested_loopback OPTIONS(table_name 'visits'); +DROP FOREIGN TABLE nest_http.visits; +CREATE FOREIGN TABLE nest_http.visits( + visit_id numeric(20,0) NOT NULL, + user_id numeric(20,0) NOT NULL, + goals goal_type[] NOT NULL +) SERVER http_nested_loopback OPTIONS(table_name 'visits'); +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (3, 3, ARRAY[row(5, 'jj'), row(6, 'zz')]::goal_type[]); +INSERT INTO nest_bin.visits +VALUES (4, 4, ARRAY[row(7, 'mm'), row(8, 'uu')]::goal_type[]); +SELECT * FROM nest_bin.visits ORDER BY visit_id; + visit_id | user_id | goals +----------+---------+--------------------- + 1 | 1 | {"(1,xx)","(2,yy)"} + 2 | 2 | {"(3,aa)","(4,bb)"} + 3 | 3 | {"(5,jj)","(6,zz)"} + 4 | 4 | {"(7,mm)","(8,uu)"} +(4 rows) + +SELECT * FROM nest_http.visits ORDER BY visit_id; + visit_id | user_id | goals +----------+---------+--------------------- + 1 | 1 | {"(1,xx)","(2,yy)"} + 2 | 2 | {"(3,aa)","(4,bb)"} + 3 | 3 | {"(5,jj)","(6,zz)"} + 4 | 4 | {"(7,mm)","(8,uu)"} +(4 rows) + +-- Querying composite field access works, unlike for multidimensional arrays, +-- but remains local for now. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE goals[1].serial = 1; + QUERY PLAN +-------------------------------------------------------------- + Foreign Scan on nest_bin.visits + Output: visit_id + Filter: (visits.goals[1].serial = 1) + Remote SQL: SELECT visit_id, goals FROM nested_test.visits +(4 rows) + +SELECT visit_id FROM nest_bin.visits WHERE goals[1].serial = 1; + visit_id +---------- + 1 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE goals[1].serial = 1; + QUERY PLAN +-------------------------------------------------------------- + Foreign Scan on nest_http.visits + Output: visit_id + Filter: (visits.goals[1].serial = 1) + Remote SQL: SELECT visit_id, goals FROM nested_test.visits +(4 rows) + +SELECT visit_id FROM nest_http.visits WHERE goals[1].serial = 1; + visit_id +---------- + 1 +(1 row) + +/****************************************************************************/ +-- Replicate the composites example from the docs. +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.events ( + id UInt32, + status Enum8('new' = 1, 'done' = 2), + point Tuple(Int32, Int32), + labels Map(String, Int64), + items Nested(id Int32, name String) + ) ORDER BY id SETTINGS flatten_nested = 0 +$$); +CALL clickhouse_perform('nested_admin', $$ + INSERT INTO nested_test.events + VALUES(1, 'new', tuple(3, 4), {'k1': 5, 'k2': 6}, [tuple(100, 'xx'), tuple(101, 'yy')]) +$$); +CREATE TYPE event_status AS ENUM ('new', 'done'); +CREATE TYPE event_point AS (x integer, y integer); +CREATE TYPE event_label AS (key text, value bigint); +CREATE TYPE event_item AS (id integer, name text); +CREATE FOREIGN TABLE nest_bin.events ( + id bigint, + status event_status, + point event_point, + labels event_label[], + items event_item[] +) SERVER binary_nested_loopback OPTIONS(table_name 'events'); +SELECT * FROM nest_bin.events ORDER BY id; + id | status | point | labels | items +----+--------+-------+---------------------+------------------------- + 1 | new | (3,4) | {"(k1,5)","(k2,6)"} | {"(100,xx)","(101,yy)"} +(1 row) + +SELECT (point).x, (point).y, + (labels[1]).key, (labels[1]).value, + (items[1]).id, (items[1]).name + FROM nest_bin.events ORDER BY id; + x | y | key | value | id | name +---+---+-----+-------+-----+------ + 3 | 4 | k1 | 5 | 100 | xx +(1 row) + +-- Composite access does not yet push down. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM nest_bin.events WHERE (point).x = 3; + QUERY PLAN +-------------------------------------------------------- + Foreign Scan on nest_bin.events + Output: id + Filter: ((events.point).x = 3) + Remote SQL: SELECT id, point FROM nested_test.events +(4 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM nest_bin.events WHERE (labels[1]).key = 'k2'; + QUERY PLAN +--------------------------------------------------------- + Foreign Scan on nest_bin.events + Output: id + Filter: (events.labels[1].key = 'k2'::text) + Remote SQL: SELECT id, labels FROM nested_test.events +(4 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM nest_bin.events WHERE (items[1]).id = 100; + QUERY PLAN +-------------------------------------------------------- + Foreign Scan on nest_bin.events + Output: id + Filter: (events.items[1].id = 100) + Remote SQL: SELECT id, items FROM nested_test.events +(4 rows) + +CALL clickhouse_perform('nested_admin', 'DROP DATABASE nested_test'); +DROP USER MAPPING FOR CURRENT_USER SERVER binary_nested_loopback; +DROP USER MAPPING FOR CURRENT_USER SERVER http_nested_loopback; +DROP SERVER binary_nested_loopback CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to foreign table nest_bin.visits +drop cascades to foreign table nest_bin.events +DROP SERVER http_nested_loopback CASCADE; +NOTICE: drop cascades to foreign table nest_http.visits diff --git a/test/expected/nested_1.out b/test/expected/nested_1.out new file mode 100644 index 00000000..0060949c --- /dev/null +++ b/test/expected/nested_1.out @@ -0,0 +1,322 @@ +\set ECHO errors +/****************************************************************************/ +-- By default ClickHouse flattens a Nested column. +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.visits( + visit_id UInt64, + user_id UInt64, + goals Nested( + serial UInt32, + order_id String + ) + ) + ENGINE = MergeTree ORDER BY visit_id +$$); +-- By default, a Nested column is flattened into multiple columns. +IMPORT FOREIGN SCHEMA nested_test FROM SERVER binary_nested_loopback INTO nest_bin; +EXECUTE describe('nest_bin.visits'); + attname | type | attndims | attnotnull +----------------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals.serial | bigint[] | 1 | t + goals.order_id | text[] | 1 | t +(4 rows) + +IMPORT FOREIGN SCHEMA nested_test FROM SERVER http_nested_loopback INTO nest_http; +EXECUTE describe('nest_http.visits'); + attname | type | attndims | attnotnull +----------------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals.serial | bigint[] | 1 | t + goals.order_id | text[] | 1 | t +(4 rows) + +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (1, 1, '{1,2}'::bigint[],'{xx,yy}'::text[]); +INSERT INTO nest_bin.visits +VALUES (2, 2, '{3,4}'::bigint[],'{aa,bb}'::text[]); +SELECT * FROM nest_bin.visits ORDER BY visit_id; + visit_id | user_id | goals.serial | goals.order_id +----------+---------+--------------+---------------- + 1 | 1 | {1,2} | {xx,yy} + 2 | 2 | {3,4} | {aa,bb} +(2 rows) + +SELECT * FROM nest_http.visits ORDER BY visit_id; + visit_id | user_id | goals.serial | goals.order_id +----------+---------+--------------+---------------- + 1 | 1 | {1,2} | {xx,yy} + 2 | 2 | {3,4} | {aa,bb} +(2 rows) + +-- Should pushdown array access. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE "goals.serial"[1] = 1; + QUERY PLAN +----------------------------------------------------------------------------------------- + Foreign Scan on nest_bin.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE ((("goals.serial"[1]) = 1)) +(3 rows) + +SELECT visit_id FROM nest_bin.visits WHERE "goals.serial"[1] = 1; + visit_id +---------- + 1 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE "goals.serial"[1] = 1; + QUERY PLAN +----------------------------------------------------------------------------------------- + Foreign Scan on nest_http.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE ((("goals.serial"[1]) = 1)) +(3 rows) + +SELECT visit_id FROM nest_http.visits WHERE "goals.serial"[1] = 1; + visit_id +---------- + 1 +(1 row) + +\set ECHO errors +/****************************************************************************/ +-- Recreate everything with flatten_nested=0 +DROP FOREIGN TABLE nest_bin.visits; +DROP FOREIGN TABLE nest_http.visits; +CALL clickhouse_perform('nested_admin', 'DROP TABLE nested_test.visits'); +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.visits( + visit_id UInt64, + user_id UInt64, + goals Nested( + serial UInt32, + order_id String + ) + ) + ENGINE = MergeTree ORDER BY visit_id SETTINGS flatten_nested = 0 +$$); +IMPORT FOREIGN SCHEMA nested_test FROM SERVER binary_nested_loopback INTO nest_bin; +NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "goals"; please create composite type and alter the column if needed +EXECUTE describe('nest_bin.visits'); + attname | type | attndims | attnotnull +----------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals | text[] | 2 | t +(3 rows) + +IMPORT FOREIGN SCHEMA nested_test FROM SERVER http_nested_loopback INTO nest_http; +NOTICE: pg_clickhouse: ClickHouse type was translated to type for column "goals"; please create composite type and alter the column if needed +EXECUTE describe('nest_http.visits'); + attname | type | attndims | attnotnull +----------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals | text[] | 2 | t +(3 rows) + +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (1, 1, '{{1,xx},{2,yy}}'::text[]); +INSERT INTO nest_bin.visits +VALUES (2, 2, '{{3,aa},{4,bb}}'::text[]); +SELECT * FROM nest_bin.visits ORDER BY visit_id; + visit_id | user_id | goals +----------+---------+----------------- + 1 | 1 | {{1,xx},{2,yy}} + 2 | 2 | {{3,aa},{4,bb}} +(2 rows) + +SELECT * FROM nest_http.visits ORDER BY visit_id; + visit_id | user_id | goals +----------+---------+----------------- + 1 | 1 | {{1,xx},{2,yy}} + 2 | 2 | {{3,aa},{4,bb}} +(2 rows) + +-- Pushdown multidimensional array access fails. Failure expected: we would +-- need to know that the column is Nested and thus should be converted to +-- `tupleElement(goals[1], 1) = '1'`. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE goals[1][1] = '1'; + QUERY PLAN +------------------------------------------------------------------------------------- + Foreign Scan on nest_bin.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE (((goals[1][1]) = '1')) +(3 rows) + +SELECT visit_id FROM nest_bin.visits WHERE goals[1][1] = '1'; +ERROR: pg_clickhouse: DB::Exception: First argument for function 'arrayElement' must be array, got 'Tuple(serial UInt32, order_id String)' instead: In scope SELECT visit_id FROM nested_test.visits WHERE ((goals[1])[1]) = '1' +DETAIL: Remote Query: SELECT visit_id FROM nested_test.visits WHERE (((goals[1][1]) = '1')) +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE goals[1][1] = '1'; + QUERY PLAN +------------------------------------------------------------------------------------- + Foreign Scan on nest_http.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE (((goals[1][1]) = '1')) +(3 rows) + +SELECT visit_id FROM nest_http.visits WHERE goals[1][1] = '1'; +ERROR: pg_clickhouse: Code: 43. DB::Exception: First argument for function 'arrayElement' must be array, got 'Tuple(serial UInt32, order_id String)' instead: In scope SELECT visit_id FROM nested_test.visits WHERE ((goals[1])[1]) = '1'. (ILLEGAL_TYPE_OF_ARGUMENT) +DETAIL: Remote Query: SELECT visit_id FROM nested_test.visits WHERE (((goals[1][1]) = '1')) +CONTEXT: HTTP status code: 500 +/****************************************************************************/ +-- Create them manually with a composite type; +CREATE TYPE goal_type AS (serial bigint, order_id text); +DROP FOREIGN TABLE nest_bin.visits; +CREATE FOREIGN TABLE nest_bin.visits( + visit_id numeric(20,0) NOT NULL, + user_id numeric(20,0) NOT NULL, + goals goal_type[] NOT NULL +) SERVER binary_nested_loopback OPTIONS(table_name 'visits'); +DROP FOREIGN TABLE nest_http.visits; +CREATE FOREIGN TABLE nest_http.visits( + visit_id numeric(20,0) NOT NULL, + user_id numeric(20,0) NOT NULL, + goals goal_type[] NOT NULL +) SERVER http_nested_loopback OPTIONS(table_name 'visits'); +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (3, 3, ARRAY[row(5, 'jj'), row(6, 'zz')]::goal_type[]); +INSERT INTO nest_bin.visits +VALUES (4, 4, ARRAY[row(7, 'mm'), row(8, 'uu')]::goal_type[]); +SELECT * FROM nest_bin.visits ORDER BY visit_id; + visit_id | user_id | goals +----------+---------+--------------------- + 1 | 1 | {"(1,xx)","(2,yy)"} + 2 | 2 | {"(3,aa)","(4,bb)"} + 3 | 3 | {"(5,jj)","(6,zz)"} + 4 | 4 | {"(7,mm)","(8,uu)"} +(4 rows) + +SELECT * FROM nest_http.visits ORDER BY visit_id; + visit_id | user_id | goals +----------+---------+--------------------- + 1 | 1 | {"(1,xx)","(2,yy)"} + 2 | 2 | {"(3,aa)","(4,bb)"} + 3 | 3 | {"(5,jj)","(6,zz)"} + 4 | 4 | {"(7,mm)","(8,uu)"} +(4 rows) + +-- Querying composite field access works, unlike for multidimensional arrays, +-- but remains local for now. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE goals[1].serial = 1; + QUERY PLAN +-------------------------------------------------------------- + Foreign Scan on nest_bin.visits + Output: visit_id + Filter: (visits.goals[1].serial = 1) + Remote SQL: SELECT visit_id, goals FROM nested_test.visits +(4 rows) + +SELECT visit_id FROM nest_bin.visits WHERE goals[1].serial = 1; + visit_id +---------- + 1 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE goals[1].serial = 1; + QUERY PLAN +-------------------------------------------------------------- + Foreign Scan on nest_http.visits + Output: visit_id + Filter: (visits.goals[1].serial = 1) + Remote SQL: SELECT visit_id, goals FROM nested_test.visits +(4 rows) + +SELECT visit_id FROM nest_http.visits WHERE goals[1].serial = 1; + visit_id +---------- + 1 +(1 row) + +/****************************************************************************/ +-- Replicate the composites example from the docs. +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.events ( + id UInt32, + status Enum8('new' = 1, 'done' = 2), + point Tuple(Int32, Int32), + labels Map(String, Int64), + items Nested(id Int32, name String) + ) ORDER BY id SETTINGS flatten_nested = 0 +$$); +CALL clickhouse_perform('nested_admin', $$ + INSERT INTO nested_test.events + VALUES(1, 'new', tuple(3, 4), {'k1': 5, 'k2': 6}, [tuple(100, 'xx'), tuple(101, 'yy')]) +$$); +CREATE TYPE event_status AS ENUM ('new', 'done'); +CREATE TYPE event_point AS (x integer, y integer); +CREATE TYPE event_label AS (key text, value bigint); +CREATE TYPE event_item AS (id integer, name text); +CREATE FOREIGN TABLE nest_bin.events ( + id bigint, + status event_status, + point event_point, + labels event_label[], + items event_item[] +) SERVER binary_nested_loopback OPTIONS(table_name 'events'); +SELECT * FROM nest_bin.events ORDER BY id; + id | status | point | labels | items +----+--------+-------+---------------------+------------------------- + 1 | new | (3,4) | {"(k1,5)","(k2,6)"} | {"(100,xx)","(101,yy)"} +(1 row) + +SELECT (point).x, (point).y, + (labels[1]).key, (labels[1]).value, + (items[1]).id, (items[1]).name + FROM nest_bin.events ORDER BY id; + x | y | key | value | id | name +---+---+-----+-------+-----+------ + 3 | 4 | k1 | 5 | 100 | xx +(1 row) + +-- Composite access does not yet push down. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM nest_bin.events WHERE (point).x = 3; + QUERY PLAN +-------------------------------------------------------- + Foreign Scan on nest_bin.events + Output: id + Filter: ((events.point).x = 3) + Remote SQL: SELECT id, point FROM nested_test.events +(4 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM nest_bin.events WHERE (labels[1]).key = 'k2'; + QUERY PLAN +--------------------------------------------------------- + Foreign Scan on nest_bin.events + Output: id + Filter: (events.labels[1].key = 'k2'::text) + Remote SQL: SELECT id, labels FROM nested_test.events +(4 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM nest_bin.events WHERE (items[1]).id = 100; + QUERY PLAN +-------------------------------------------------------- + Foreign Scan on nest_bin.events + Output: id + Filter: (events.items[1].id = 100) + Remote SQL: SELECT id, items FROM nested_test.events +(4 rows) + +CALL clickhouse_perform('nested_admin', 'DROP DATABASE nested_test'); +DROP USER MAPPING FOR CURRENT_USER SERVER binary_nested_loopback; +DROP USER MAPPING FOR CURRENT_USER SERVER http_nested_loopback; +DROP SERVER binary_nested_loopback CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to foreign table nest_bin.visits +drop cascades to foreign table nest_bin.events +DROP SERVER http_nested_loopback CASCADE; +NOTICE: drop cascades to foreign table nest_http.visits diff --git a/test/expected/nested_2.out b/test/expected/nested_2.out new file mode 100644 index 00000000..c61a83f2 --- /dev/null +++ b/test/expected/nested_2.out @@ -0,0 +1,87 @@ +\set ECHO errors +/****************************************************************************/ +-- By default ClickHouse flattens a Nested column. +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.visits( + visit_id UInt64, + user_id UInt64, + goals Nested( + serial UInt32, + order_id String + ) + ) + ENGINE = MergeTree ORDER BY visit_id +$$); +-- By default, a Nested column is flattened into multiple columns. +IMPORT FOREIGN SCHEMA nested_test FROM SERVER binary_nested_loopback INTO nest_bin; +EXECUTE describe('nest_bin.visits'); + attname | type | attndims | attnotnull +----------------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals.serial | bigint[] | 1 | t + goals.order_id | text[] | 1 | t +(4 rows) + +IMPORT FOREIGN SCHEMA nested_test FROM SERVER http_nested_loopback INTO nest_http; +EXECUTE describe('nest_http.visits'); + attname | type | attndims | attnotnull +----------------+---------------+----------+------------ + visit_id | numeric(20,0) | 0 | t + user_id | numeric(20,0) | 0 | t + goals.serial | bigint[] | 1 | t + goals.order_id | text[] | 1 | t +(4 rows) + +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (1, 1, '{1,2}'::bigint[],'{xx,yy}'::text[]); +INSERT INTO nest_bin.visits +VALUES (2, 2, '{3,4}'::bigint[],'{aa,bb}'::text[]); +SELECT * FROM nest_bin.visits ORDER BY visit_id; + visit_id | user_id | goals.serial | goals.order_id +----------+---------+--------------+---------------- + 1 | 1 | {1,2} | {xx,yy} + 2 | 2 | {3,4} | {aa,bb} +(2 rows) + +SELECT * FROM nest_http.visits ORDER BY visit_id; + visit_id | user_id | goals.serial | goals.order_id +----------+---------+--------------+---------------- + 1 | 1 | {1,2} | {xx,yy} + 2 | 2 | {3,4} | {aa,bb} +(2 rows) + +-- Should pushdown array access. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE "goals.serial"[1] = 1; + QUERY PLAN +----------------------------------------------------------------------------------------- + Foreign Scan on nest_bin.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE ((("goals.serial"[1]) = 1)) +(3 rows) + +SELECT visit_id FROM nest_bin.visits WHERE "goals.serial"[1] = 1; + visit_id +---------- + 1 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE "goals.serial"[1] = 1; + QUERY PLAN +----------------------------------------------------------------------------------------- + Foreign Scan on nest_http.visits + Output: visit_id + Remote SQL: SELECT visit_id FROM nested_test.visits WHERE ((("goals.serial"[1]) = 1)) +(3 rows) + +SELECT visit_id FROM nest_http.visits WHERE "goals.serial"[1] = 1; + visit_id +---------- + 1 +(1 row) + +\set ECHO errors +SKIP: flatten_nested not available prior to ClickHouse 25.1 diff --git a/test/expected/result_map.txt b/test/expected/result_map.txt index a1e266e9..dd1065ca 100644 --- a/test/expected/result_map.txt +++ b/test/expected/result_map.txt @@ -248,6 +248,19 @@ json_coldef.sql 25.3+ | json_coldef.out 23.3-25.2 | json_coldef_1.out +nested.sql +----------- + + Postgres | File +----------|------------ + 14-19 | nested.out + + ClickHouse | File +------------|-------------- + 26.8+ | nested.out + 25.1-26.3 | nested_1.out + 23+ | nested_2.out + param.sql --------- diff --git a/test/sql/nested.sql b/test/sql/nested.sql new file mode 100644 index 00000000..3a595f74 --- /dev/null +++ b/test/sql/nested.sql @@ -0,0 +1,203 @@ +\set ECHO errors +SET datestyle = 'ISO'; +CREATE SERVER binary_nested_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'nested_test', driver 'binary'); +CREATE SERVER http_nested_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'nested_test', driver 'http'); +CREATE USER MAPPING FOR CURRENT_USER SERVER binary_nested_loopback; +CREATE USER MAPPING FOR CURRENT_USER SERVER http_nested_loopback; + +CREATE SERVER nested_admin FOREIGN DATA WRAPPER clickhouse_fdw; +CREATE USER MAPPING FOR CURRENT_USER SERVER nested_admin; + +CALL clickhouse_perform('nested_admin', 'DROP DATABASE IF EXISTS nested_test'); +CALL clickhouse_perform('nested_admin', 'CREATE DATABASE nested_test'); + +CREATE SCHEMA nest_bin; +CREATE SCHEMA nest_http; + +PREPARE describe(regclass) AS +SELECT attname, format_type(atttypid, atttypmod) AS type, attndims, attnotnull + FROM pg_attribute + WHERE attrelid = $1 AND attnum > 0 + ORDER BY attnum; + +\set ECHO all + +/****************************************************************************/ +-- By default ClickHouse flattens a Nested column. +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.visits( + visit_id UInt64, + user_id UInt64, + goals Nested( + serial UInt32, + order_id String + ) + ) + ENGINE = MergeTree ORDER BY visit_id +$$); + +-- By default, a Nested column is flattened into multiple columns. +IMPORT FOREIGN SCHEMA nested_test FROM SERVER binary_nested_loopback INTO nest_bin; +EXECUTE describe('nest_bin.visits'); +IMPORT FOREIGN SCHEMA nested_test FROM SERVER http_nested_loopback INTO nest_http; +EXECUTE describe('nest_http.visits'); + +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (1, 1, '{1,2}'::bigint[],'{xx,yy}'::text[]); + +INSERT INTO nest_bin.visits +VALUES (2, 2, '{3,4}'::bigint[],'{aa,bb}'::text[]); + +SELECT * FROM nest_bin.visits ORDER BY visit_id; +SELECT * FROM nest_http.visits ORDER BY visit_id; + +-- Should pushdown array access. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE "goals.serial"[1] = 1; +SELECT visit_id FROM nest_bin.visits WHERE "goals.serial"[1] = 1; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE "goals.serial"[1] = 1; +SELECT visit_id FROM nest_http.visits WHERE "goals.serial"[1] = 1; + +\set ECHO errors +/****************************************************************************/ +-- Is flatten_nested available? +SELECT clickhouse_server_version('nested_admin') AS ch_version \gset +SELECT (split_part(:'ch_version', '.', 1)::int, + split_part(:'ch_version', '.', 2)::int) < (25, 1) AS no_ch251 \gset +\if :no_ch251 +\echo 'SKIP: flatten_nested not available prior to ClickHouse 25.1' +\quit +\endif +\set ECHO all + +/****************************************************************************/ +-- Recreate everything with flatten_nested=0 +DROP FOREIGN TABLE nest_bin.visits; +DROP FOREIGN TABLE nest_http.visits; + +CALL clickhouse_perform('nested_admin', 'DROP TABLE nested_test.visits'); +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.visits( + visit_id UInt64, + user_id UInt64, + goals Nested( + serial UInt32, + order_id String + ) + ) + ENGINE = MergeTree ORDER BY visit_id SETTINGS flatten_nested = 0 +$$); + +IMPORT FOREIGN SCHEMA nested_test FROM SERVER binary_nested_loopback INTO nest_bin; +EXECUTE describe('nest_bin.visits'); +IMPORT FOREIGN SCHEMA nested_test FROM SERVER http_nested_loopback INTO nest_http; +EXECUTE describe('nest_http.visits'); + +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (1, 1, '{{1,xx},{2,yy}}'::text[]); + +INSERT INTO nest_bin.visits +VALUES (2, 2, '{{3,aa},{4,bb}}'::text[]); + +SELECT * FROM nest_bin.visits ORDER BY visit_id; +SELECT * FROM nest_http.visits ORDER BY visit_id; + +-- Pushdown multidimensional array access fails. Failure expected: we would +-- need to know that the column is Nested and thus should be converted to +-- `tupleElement(goals[1], 1) = '1'`. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE goals[1][1] = '1'; +SELECT visit_id FROM nest_bin.visits WHERE goals[1][1] = '1'; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE goals[1][1] = '1'; +SELECT visit_id FROM nest_http.visits WHERE goals[1][1] = '1'; + +/****************************************************************************/ +-- Create them manually with a composite type; +CREATE TYPE goal_type AS (serial bigint, order_id text); +DROP FOREIGN TABLE nest_bin.visits; + +CREATE FOREIGN TABLE nest_bin.visits( + visit_id numeric(20,0) NOT NULL, + user_id numeric(20,0) NOT NULL, + goals goal_type[] NOT NULL +) SERVER binary_nested_loopback OPTIONS(table_name 'visits'); + +DROP FOREIGN TABLE nest_http.visits; +CREATE FOREIGN TABLE nest_http.visits( + visit_id numeric(20,0) NOT NULL, + user_id numeric(20,0) NOT NULL, + goals goal_type[] NOT NULL +) SERVER http_nested_loopback OPTIONS(table_name 'visits'); + +-- Insert values. +INSERT INTO nest_bin.visits +VALUES (3, 3, ARRAY[row(5, 'jj'), row(6, 'zz')]::goal_type[]); + +INSERT INTO nest_bin.visits +VALUES (4, 4, ARRAY[row(7, 'mm'), row(8, 'uu')]::goal_type[]); + +SELECT * FROM nest_bin.visits ORDER BY visit_id; +SELECT * FROM nest_http.visits ORDER BY visit_id; + +-- Querying composite field access works, unlike for multidimensional arrays, +-- but remains local for now. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_bin.visits WHERE goals[1].serial = 1; +SELECT visit_id FROM nest_bin.visits WHERE goals[1].serial = 1; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT visit_id FROM nest_http.visits WHERE goals[1].serial = 1; +SELECT visit_id FROM nest_http.visits WHERE goals[1].serial = 1; + +/****************************************************************************/ +-- Replicate the composites example from the docs. +CALL clickhouse_perform('nested_admin', $$ + CREATE TABLE nested_test.events ( + id UInt32, + status Enum8('new' = 1, 'done' = 2), + point Tuple(Int32, Int32), + labels Map(String, Int64), + items Nested(id Int32, name String) + ) ORDER BY id SETTINGS flatten_nested = 0 +$$); + +CALL clickhouse_perform('nested_admin', $$ + INSERT INTO nested_test.events + VALUES(1, 'new', tuple(3, 4), {'k1': 5, 'k2': 6}, [tuple(100, 'xx'), tuple(101, 'yy')]) +$$); + +CREATE TYPE event_status AS ENUM ('new', 'done'); +CREATE TYPE event_point AS (x integer, y integer); +CREATE TYPE event_label AS (key text, value bigint); +CREATE TYPE event_item AS (id integer, name text); + +CREATE FOREIGN TABLE nest_bin.events ( + id bigint, + status event_status, + point event_point, + labels event_label[], + items event_item[] +) SERVER binary_nested_loopback OPTIONS(table_name 'events'); + +SELECT * FROM nest_bin.events ORDER BY id; +SELECT (point).x, (point).y, + (labels[1]).key, (labels[1]).value, + (items[1]).id, (items[1]).name + FROM nest_bin.events ORDER BY id; + +-- Composite access does not yet push down. +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM nest_bin.events WHERE (point).x = 3; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM nest_bin.events WHERE (labels[1]).key = 'k2'; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM nest_bin.events WHERE (items[1]).id = 100; + +CALL clickhouse_perform('nested_admin', 'DROP DATABASE nested_test'); +DROP USER MAPPING FOR CURRENT_USER SERVER binary_nested_loopback; +DROP USER MAPPING FOR CURRENT_USER SERVER http_nested_loopback; +DROP SERVER binary_nested_loopback CASCADE; +DROP SERVER http_nested_loopback CASCADE;