Skip to content

Querying system tables takes too long during exports #1920

@DimensionWieldr

Description

@DimensionWieldr

Describe the bug
Querying system.replicated_partition_exports takes very long when many exports are queued to an iceberg table on ice-rest-catalog.

To Reproduce
Steps to reproduce the behavior:

  1. Create source table and populate it with random values.
CREATE TABLE default.source1 ON CLUSTER `test-export`
(
    event_month Int32 MATERIALIZED toInt32(toYYYYMM(toDateTime(event_time))),
    id Int64,
    event_time DateTime64(6),
    user_id Int32,
    category String,
    value Float64,
    payload String
)
ENGINE = ReplicatedMergeTree(
    '/clickhouse/{installation}/{cluster}/tables/default/source1',
    '{replica}'
)
PARTITION BY event_month
ORDER BY (event_time, id)
SETTINGS index_granularity = 8192;

INSERT INTO default.source1 (id, event_time, user_id, category, value, payload)
SELECT
    n AS id,
    toDateTime64(
        toStartOfMonth(toDateTime('2015-01-01 00:00:00'))
        + toIntervalMonth(intDiv(n, 200000))
        + toIntervalSecond(n % 200000),
        6
    ) AS event_time,
    toInt32((n * 48271) % 2147483647),
    concat('c', toString(n % 10)),
    (n % 1000000) / 1000.,
    lower(hex(SHA256(toString(n))))
FROM (SELECT number AS n FROM numbers(10000000))
SETTINGS max_insert_threads = 8;
  1. Create DataLakeCatalog database and iceberg table.
CREATE DATABASE IF NOT EXISTS ice ON CLUSTER `test-export`
ENGINE = DataLakeCatalog('http://ice-rest-catalog:5000/')
SETTINGS
    catalog_type = 'rest',
    auth_header = 'Authorization: Bearer <YOUR_TOKEN>',
    warehouse = 's3://protvjoe-1lfyj7kp-iceberg';

CREATE TABLE ice.`default.dest5`
(
    event_month Int32,
    id Int64,
    event_time DateTime64(6),
    user_id Int32,
    category String,
    value Float64,
    payload String
)
ENGINE = Iceberg('s3://protvjoe-1lfyj7kp-iceberg/default/dest5')
PARTITION BY event_month;
  1. Export all 50 partitions (10 million total rows) at once. Send these all at once:
ALTER TABLE default.source1
EXPORT PARTITION ID '201501'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201502'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201503'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201504'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201505'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201506'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201507'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201508'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201509'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201510'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201511'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201512'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201601'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201602'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201603'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201604'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201605'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201606'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201607'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201608'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201609'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201610'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201611'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201612'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201701'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201702'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201703'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201704'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201705'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201706'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201707'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201708'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201709'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201710'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201711'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201712'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201801'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201802'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201803'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201804'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201805'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201806'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201807'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201808'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201809'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201810'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201811'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201812'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201901'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;

ALTER TABLE default.source1
EXPORT PARTITION ID '201902'
TO TABLE ice.`default.dest5`
SETTINGS allow_experimental_export_merge_tree_part = 1;
  1. Query system.replicated_partition_exports for status of exports. This step takes really long. This is the issue.
SELECT
    partition_id,
    status,
    parts_to_do,
    exception_count,
    source_replica,
    create_time
FROM system.replicated_partition_exports
WHERE source_database = 'default'
  AND source_table = 'source1'
  AND destination_database = 'ice'
  AND destination_table = 'default.dest5'
ORDER BY partition_id;

Expected behavior
Query shouldn't take so long.

Key information

  • 26.3.10.20001 Antalya

Metadata

Metadata

Assignees

No one assigned

    Labels

    Type

    No type
    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions