mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-23 08:02:02 +00:00
c25d6cd624
* Limit log frequence for "Skipping send data over distributed table" message
After SYSTEM STOP DISTRIBUTED SENDS it will constantly print this
message.
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
* Rename directory monitor concept into async INSERT
Rename the following query settings (with preserving backward
compatiblity, by keeping old name as an alias):
- distributed_directory_monitor_sleep_time_ms -> distributed_async_insert_sleep_time_ms
- distributed_directory_monitor_max_sleep_time_ms -> distributed_async_insert_max_sleep_time_ms
- distributed_directory_monitor_batch -> distributed_async_insert_batch_inserts
- distributed_directory_monitor_split_batch_on_failure -> distributed_async_insert_split_batch_on_failure
Rename the following table settings (with preserving backward
compatiblity, by keeping old name as an alias):
- monitor_batch_inserts -> async_insert_batch
- monitor_split_batch_on_failure -> async_insert_split_batch_on_failure
- directory_monitor_sleep_time_ms -> async_insert_sleep_time_ms
- directory_monitor_max_sleep_time_ms -> async_insert_max_sleep_time_ms
And also update all the references:
$ gg -e directory_monitor_ -e monitor_ tests docs | cut -d: -f1 | sort -u | xargs sed -e 's/distributed_directory_monitor_sleep_time_ms/distributed_async_insert_sleep_time_ms/g' -e 's/distributed_directory_monitor_max_sleep_time_ms/distributed_async_insert_max_sleep_time_ms/g' -e 's/distributed_directory_monitor_batch_inserts/distributed_async_insert_batch/g' -e 's/distributed_directory_monitor_split_batch_on_failure/distributed_async_insert_split_batch_on_failure/g' -e 's/monitor_batch_inserts/async_insert_batch/g' -e 's/monitor_split_batch_on_failure/async_insert_split_batch_on_failure/g' -e 's/monitor_sleep_time_ms/async_insert_sleep_time_ms/g' -e 's/monitor_max_sleep_time_ms/async_insert_max_sleep_time_ms/g' -i
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
* Rename async_insert for Distributed into background_insert
This will avoid amigibuity between general async INSERT's and INSERT
into Distributed, which are indeed background, so new term express it
even better.
Mostly done with:
$ git di HEAD^ --name-only | xargs sed -i -e 's/distributed_async_insert/distributed_background_insert/g' -e 's/async_insert_batch/background_insert_batch/g' -e 's/async_insert_split_batch_on_failure/background_insert_split_batch_on_failure/g' -e 's/async_insert_sleep_time_ms/background_insert_sleep_time_ms/g' -e 's/async_insert_max_sleep_time_ms/background_insert_max_sleep_time_ms/g'
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
* Mark 02417_opentelemetry_insert_on_distributed_table as long
CI: https://s3.amazonaws.com/clickhouse-test-reports/55978/7a6abb03a0b507e29e999cb7e04f246a119c6f28/stateless_tests_flaky_check__asan_.html
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---------
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
110 lines
4.7 KiB
XML
110 lines
4.7 KiB
XML
<clickhouse>
|
|
<remote_servers>
|
|
<source_cluster>
|
|
<shard>
|
|
<internal_replication>false</internal_replication>
|
|
<replica>
|
|
<host>s0_0_0</host>
|
|
<port>9000</port>
|
|
</replica>
|
|
</shard>
|
|
</source_cluster>
|
|
|
|
<destination_cluster>
|
|
<shard>
|
|
<internal_replication>false</internal_replication>
|
|
<replica>
|
|
<host>s1_1_0</host>
|
|
<port>9000</port>
|
|
</replica>
|
|
</shard>
|
|
</destination_cluster>
|
|
</remote_servers>
|
|
|
|
<!-- How many simultaneously active workers are possible. If you run more workers superfluous workers will sleep. -->
|
|
<max_workers>2</max_workers>
|
|
|
|
<!-- Setting used to fetch (pull) data from source cluster tables -->
|
|
<settings_pull>
|
|
<readonly>1</readonly>
|
|
</settings_pull>
|
|
|
|
<!-- Setting used to insert (push) data to destination cluster tables -->
|
|
<settings_push>
|
|
<readonly>0</readonly>
|
|
</settings_push>
|
|
|
|
<!-- Common setting for fetch (pull) and insert (push) operations. Also, copier process context uses it.
|
|
They are overlaid by <settings_pull/> and <settings_push/> respectively. -->
|
|
<settings>
|
|
<connect_timeout>3</connect_timeout>
|
|
<!-- Sync insert is set forcibly, leave it here just in case. -->
|
|
<distributed_foreground_insert>1</distributed_foreground_insert>
|
|
</settings>
|
|
|
|
<!-- Copying tasks description.
|
|
You could specify several table task in the same task description (in the same ZooKeeper node), they will be performed
|
|
sequentially.
|
|
-->
|
|
<tables>
|
|
<!-- A table task, copies one table. -->
|
|
<table_hits>
|
|
<!-- Source cluster name (from <remote_servers/> section) and tables in it that should be copied -->
|
|
<cluster_pull>source_cluster</cluster_pull>
|
|
<database_pull>default</database_pull>
|
|
<table_pull>ontime</table_pull>
|
|
<!-- <table_pull>onetime</table_pull> -->
|
|
|
|
<!-- Destination cluster name and tables in which the data should be inserted -->
|
|
<cluster_push>destination_cluster</cluster_push>
|
|
<database_push>default</database_push>
|
|
<table_push>ontime22</table_push>
|
|
<!-- <table_pull>onetime</table_pull> -->
|
|
|
|
<!-- Engine of destination tables.
|
|
If destination tables have not be created, workers create them using columns definition from source tables and engine
|
|
definition from here.
|
|
|
|
NOTE: If the first worker starts insert data and detects that destination partition is not empty then the partition will
|
|
be dropped and refilled, take it into account if you already have some data in destination tables. You could directly
|
|
specify partitions that should be copied in <enabled_partitions/>, they should be in quoted format like partition column of
|
|
system.parts table.
|
|
-->
|
|
|
|
|
|
<engine>
|
|
ENGINE = MergeTree() PARTITION BY Year ORDER BY (Year, FlightDate) SETTINGS index_granularity=8192
|
|
</engine>
|
|
|
|
<!-- Sharding key used to insert data to destination cluster -->
|
|
|
|
<sharding_key>jumpConsistentHash(intHash64(Year), 2)</sharding_key>
|
|
|
|
<!-- Optional expression that filter data while pull them from source servers -->
|
|
<!-- <where_condition>CounterID != 0</where_condition> -->
|
|
|
|
<!-- This section specifies partitions that should be copied, other partition will be ignored.
|
|
Partition names should have the same format as
|
|
partition column of system.parts table (i.e. a quoted text).
|
|
Since partition key of source and destination cluster could be different,
|
|
these partition names specify destination partitions.
|
|
|
|
NOTE: In spite of this section is optional (if it is not specified, all partitions will be copied),
|
|
it is strictly recommended to specify them explicitly.
|
|
If you already have some ready partitions on destination cluster they
|
|
will be removed at the start of the copying since they will be interpeted
|
|
as unfinished data from the previous copying!!!
|
|
-->
|
|
<enabled_partitions>
|
|
<partition>2017</partition>
|
|
</enabled_partitions>
|
|
</table_hits>
|
|
|
|
<!-- Next table to copy. It is not copied until previous table is copying. -->
|
|
<!-- </table_visits>
|
|
|
|
</table_visits>
|
|
-->
|
|
</tables>
|
|
</clickhouse>
|