mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Merge branch 'master' into 42648_Support_scalar_subqueries_cache
This commit is contained in:
commit
bb4f251448
@ -1203,12 +1203,14 @@ SELECT * FROM json_each_row_nested
|
|||||||
- [input_format_json_read_bools_as_numbers](/docs/en/operations/settings/settings.md/#input_format_json_read_bools_as_numbers) - allow to parse bools as numbers in JSON input formats. Default value - `true`.
|
- [input_format_json_read_bools_as_numbers](/docs/en/operations/settings/settings.md/#input_format_json_read_bools_as_numbers) - allow to parse bools as numbers in JSON input formats. Default value - `true`.
|
||||||
- [input_format_json_read_numbers_as_strings](/docs/en/operations/settings/settings.md/#input_format_json_read_numbers_as_strings) - allow to parse numbers as strings in JSON input formats. Default value - `false`.
|
- [input_format_json_read_numbers_as_strings](/docs/en/operations/settings/settings.md/#input_format_json_read_numbers_as_strings) - allow to parse numbers as strings in JSON input formats. Default value - `false`.
|
||||||
- [input_format_json_read_objects_as_strings](/docs/en/operations/settings/settings.md/#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `false`.
|
- [input_format_json_read_objects_as_strings](/docs/en/operations/settings/settings.md/#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `false`.
|
||||||
|
- [input_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings.md/#input_format_json_named_tuples_as_objects) - parse named tuple columns as JSON objects. Default value - `true`.
|
||||||
|
- [input_format_json_defaults_for_missing_elements_in_named_tuple](/docs/en/operations/settings/settings.md/#input_format_json_defaults_for_missing_elements_in_named_tuple) - insert default values for missing elements in JSON object while parsing named tuple. Default value - `true`.
|
||||||
- [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`.
|
- [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`.
|
||||||
- [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`.
|
- [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`.
|
||||||
- [output_format_json_quote_denormals](/docs/en/operations/settings/settings.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`.
|
- [output_format_json_quote_denormals](/docs/en/operations/settings/settings.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`.
|
||||||
- [output_format_json_quote_decimals](/docs/en/operations/settings/settings.md/#output_format_json_quote_decimals) - controls quoting of decimals in JSON output format. Default value - `false`.
|
- [output_format_json_quote_decimals](/docs/en/operations/settings/settings.md/#output_format_json_quote_decimals) - controls quoting of decimals in JSON output format. Default value - `false`.
|
||||||
- [output_format_json_escape_forward_slashes](/docs/en/operations/settings/settings.md/#output_format_json_escape_forward_slashes) - controls escaping forward slashes for string outputs in JSON output format. Default value - `true`.
|
- [output_format_json_escape_forward_slashes](/docs/en/operations/settings/settings.md/#output_format_json_escape_forward_slashes) - controls escaping forward slashes for string outputs in JSON output format. Default value - `true`.
|
||||||
- [output_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings.md/#output_format_json_named_tuples_as_objects) - serialize named tuple columns as JSON objects. Default value - `false`.
|
- [output_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings.md/#output_format_json_named_tuples_as_objects) - serialize named tuple columns as JSON objects. Default value - `true`.
|
||||||
- [output_format_json_array_of_rows](/docs/en/operations/settings/settings.md/#output_format_json_array_of_rows) - output a JSON array of all rows in JSONEachRow(Compact) format. Default value - `false`.
|
- [output_format_json_array_of_rows](/docs/en/operations/settings/settings.md/#output_format_json_array_of_rows) - output a JSON array of all rows in JSONEachRow(Compact) format. Default value - `false`.
|
||||||
- [output_format_json_validate_utf8](/docs/en/operations/settings/settings.md/#output_format_json_validate_utf8) - enables validation of UTF-8 sequences in JSON output formats (note that it doesn't impact formats JSON/JSONCompact/JSONColumnsWithMetadata, they always validate utf8). Default value - `false`.
|
- [output_format_json_validate_utf8](/docs/en/operations/settings/settings.md/#output_format_json_validate_utf8) - enables validation of UTF-8 sequences in JSON output formats (note that it doesn't impact formats JSON/JSONCompact/JSONColumnsWithMetadata, they always validate utf8). Default value - `false`.
|
||||||
|
|
||||||
|
@ -266,7 +266,7 @@ Default value: 0.
|
|||||||
|
|
||||||
Limits the size in bytes of the hash table used when joining tables.
|
Limits the size in bytes of the hash table used when joining tables.
|
||||||
|
|
||||||
This settings applies to [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) operations and [Join table engine](../../engines/table-engines/special/join.md).
|
This setting applies to [SELECT … JOIN](../../sql-reference/statements/select/join.md#select-join) operations and [Join table engine](../../engines/table-engines/special/join.md).
|
||||||
|
|
||||||
If the query contains joins, ClickHouse checks this setting for every intermediate result.
|
If the query contains joins, ClickHouse checks this setting for every intermediate result.
|
||||||
|
|
||||||
|
@ -402,40 +402,62 @@ Default value: `ALL`.
|
|||||||
|
|
||||||
## join_algorithm {#settings-join_algorithm}
|
## join_algorithm {#settings-join_algorithm}
|
||||||
|
|
||||||
Specifies [JOIN](../../sql-reference/statements/select/join.md) algorithm.
|
Specifies which [JOIN](../../sql-reference/statements/select/join.md) algorithm is used.
|
||||||
|
|
||||||
Several algorithms can be specified, and an available one would be chosen for a particular query based on kind/strictness and table engine.
|
Several algorithms can be specified, and an available one would be chosen for a particular query based on kind/strictness and table engine.
|
||||||
|
|
||||||
Possible values:
|
Possible values:
|
||||||
|
|
||||||
- `default` — `hash` or `direct`, if possible (same as `direct,hash`)
|
### `default`
|
||||||
|
|
||||||
- `hash` — [Hash join algorithm](https://en.wikipedia.org/wiki/Hash_join) is used. The most generic implementation that supports all combinations of kind and strictness and multiple join keys that are combined with `OR` in the `JOIN ON` section.
|
This is the equivalent of `hash` or `direct`, if possible (same as `direct,hash`)
|
||||||
|
|
||||||
- `parallel_hash` - a variation of `hash` join that splits the data into buckets and builds several hashtables instead of one concurrently to speed up this process.
|
### `grace_hash`
|
||||||
|
|
||||||
|
[Grace hash join](https://en.wikipedia.org/wiki/Hash_join#Grace_hash_join) is used. Grace hash provides an algorithm option that provides performant complex joins while limiting memory use.
|
||||||
|
|
||||||
|
The first phase of a grace join reads the right table and splits it into N buckets depending on the hash value of key columns (initially, N is `grace_hash_join_initial_buckets`). This is done in a way to ensure that each bucket can be processed independently. Rows from the first bucket are added to an in-memory hash table while the others are saved to disk. If the hash table grows beyond the memory limit (e.g., as set by [`max_bytes_in_join`](/docs/en/operations/settings/query-complexity.md/#settings-max_bytes_in_join)), the number of buckets is increased and the assigned bucket for each row. Any rows which don’t belong to the current bucket are flushed and reassigned.
|
||||||
|
|
||||||
|
### `hash`
|
||||||
|
|
||||||
|
[Hash join algorithm](https://en.wikipedia.org/wiki/Hash_join) is used. The most generic implementation that supports all combinations of kind and strictness and multiple join keys that are combined with `OR` in the `JOIN ON` section.
|
||||||
|
|
||||||
|
### `parallel_hash`
|
||||||
|
|
||||||
|
A variation of `hash` join that splits the data into buckets and builds several hashtables instead of one concurrently to speed up this process.
|
||||||
|
|
||||||
When using the `hash` algorithm, the right part of `JOIN` is uploaded into RAM.
|
When using the `hash` algorithm, the right part of `JOIN` is uploaded into RAM.
|
||||||
|
|
||||||
- `partial_merge` — a variation of the [sort-merge algorithm](https://en.wikipedia.org/wiki/Sort-merge_join), where only the right table is fully sorted.
|
### `partial_merge`
|
||||||
|
|
||||||
|
A variation of the [sort-merge algorithm](https://en.wikipedia.org/wiki/Sort-merge_join), where only the right table is fully sorted.
|
||||||
|
|
||||||
The `RIGHT JOIN` and `FULL JOIN` are supported only with `ALL` strictness (`SEMI`, `ANTI`, `ANY`, and `ASOF` are not supported).
|
The `RIGHT JOIN` and `FULL JOIN` are supported only with `ALL` strictness (`SEMI`, `ANTI`, `ANY`, and `ASOF` are not supported).
|
||||||
|
|
||||||
When using `partial_merge` algorithm, ClickHouse sorts the data and dumps it to the disk. The `partial_merge` algorithm in ClickHouse differs slightly from the classic realization. First, ClickHouse sorts the right table by joining keys in blocks and creates a min-max index for sorted blocks. Then it sorts parts of the left table by `join key` and joins them over the right table. The min-max index is also used to skip unneeded right table blocks.
|
When using the `partial_merge` algorithm, ClickHouse sorts the data and dumps it to the disk. The `partial_merge` algorithm in ClickHouse differs slightly from the classic realization. First, ClickHouse sorts the right table by joining keys in blocks and creates a min-max index for sorted blocks. Then it sorts parts of the left table by the `join key` and joins them over the right table. The min-max index is also used to skip unneeded right table blocks.
|
||||||
|
|
||||||
- `direct` - can be applied when the right storage supports key-value requests.
|
### `direct`
|
||||||
|
|
||||||
|
This algorithm can be applied when the storage for the right table supports key-value requests.
|
||||||
|
|
||||||
The `direct` algorithm performs a lookup in the right table using rows from the left table as keys. It's supported only by special storage such as [Dictionary](../../engines/table-engines/special/dictionary.md/#dictionary) or [EmbeddedRocksDB](../../engines/table-engines/integrations/embedded-rocksdb.md) and only the `LEFT` and `INNER` JOINs.
|
The `direct` algorithm performs a lookup in the right table using rows from the left table as keys. It's supported only by special storage such as [Dictionary](../../engines/table-engines/special/dictionary.md/#dictionary) or [EmbeddedRocksDB](../../engines/table-engines/integrations/embedded-rocksdb.md) and only the `LEFT` and `INNER` JOINs.
|
||||||
|
|
||||||
- `auto` — try `hash` join and switch on the fly to another algorithm if the memory limit is violated.
|
### `auto`
|
||||||
|
|
||||||
- `full_sorting_merge` — [Sort-merge algorithm](https://en.wikipedia.org/wiki/Sort-merge_join) with full sorting joined tables before joining.
|
When set to `auto`, `hash` join is tried first, and the algorithm is switched on the fly to another algorithm if the memory limit is violated.
|
||||||
|
|
||||||
- `prefer_partial_merge` — ClickHouse always tries to use `partial_merge` join if possible, otherwise, it uses `hash`. *Deprecated*, same as `partial_merge,hash`.
|
### `full_sorting_merge`
|
||||||
|
|
||||||
|
[Sort-merge algorithm](https://en.wikipedia.org/wiki/Sort-merge_join) with full sorting joined tables before joining.
|
||||||
|
|
||||||
|
### `prefer_partial_merge`
|
||||||
|
|
||||||
|
ClickHouse always tries to use `partial_merge` join if possible, otherwise, it uses `hash`. *Deprecated*, same as `partial_merge,hash`.
|
||||||
|
|
||||||
|
|
||||||
## join_any_take_last_row {#settings-join_any_take_last_row}
|
## join_any_take_last_row {#settings-join_any_take_last_row}
|
||||||
|
|
||||||
Changes behaviour of join operations with `ANY` strictness.
|
Changes the behaviour of join operations with `ANY` strictness.
|
||||||
|
|
||||||
:::warning
|
:::warning
|
||||||
This setting applies only for `JOIN` operations with [Join](../../engines/table-engines/special/join.md) engine tables.
|
This setting applies only for `JOIN` operations with [Join](../../engines/table-engines/special/join.md) engine tables.
|
||||||
@ -498,7 +520,7 @@ Default value: 65536.
|
|||||||
|
|
||||||
Limits the number of files allowed for parallel sorting in MergeJoin operations when they are executed on disk.
|
Limits the number of files allowed for parallel sorting in MergeJoin operations when they are executed on disk.
|
||||||
|
|
||||||
The bigger the value of the setting, the more RAM used and the less disk I/O needed.
|
The bigger the value of the setting, the more RAM is used and the less disk I/O is needed.
|
||||||
|
|
||||||
Possible values:
|
Possible values:
|
||||||
|
|
||||||
@ -514,12 +536,12 @@ Enables legacy ClickHouse server behaviour in `ANY INNER|LEFT JOIN` operations.
|
|||||||
Use this setting only for backward compatibility if your use cases depend on legacy `JOIN` behaviour.
|
Use this setting only for backward compatibility if your use cases depend on legacy `JOIN` behaviour.
|
||||||
:::
|
:::
|
||||||
|
|
||||||
When the legacy behaviour enabled:
|
When the legacy behaviour is enabled:
|
||||||
|
|
||||||
- Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are not equal because ClickHouse uses the logic with many-to-one left-to-right table keys mapping.
|
- Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are not equal because ClickHouse uses the logic with many-to-one left-to-right table keys mapping.
|
||||||
- Results of `ANY INNER JOIN` operations contain all rows from the left table like the `SEMI LEFT JOIN` operations do.
|
- Results of `ANY INNER JOIN` operations contain all rows from the left table like the `SEMI LEFT JOIN` operations do.
|
||||||
|
|
||||||
When the legacy behaviour disabled:
|
When the legacy behaviour is disabled:
|
||||||
|
|
||||||
- Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are equal because ClickHouse uses the logic which provides one-to-many keys mapping in `ANY RIGHT JOIN` operations.
|
- Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are equal because ClickHouse uses the logic which provides one-to-many keys mapping in `ANY RIGHT JOIN` operations.
|
||||||
- Results of `ANY INNER JOIN` operations contain one row per key from both the left and right tables.
|
- Results of `ANY INNER JOIN` operations contain one row per key from both the left and right tables.
|
||||||
@ -572,7 +594,7 @@ Default value: `163840`.
|
|||||||
|
|
||||||
## merge_tree_min_rows_for_concurrent_read_for_remote_filesystem {#merge-tree-min-rows-for-concurrent-read-for-remote-filesystem}
|
## merge_tree_min_rows_for_concurrent_read_for_remote_filesystem {#merge-tree-min-rows-for-concurrent-read-for-remote-filesystem}
|
||||||
|
|
||||||
The minimum number of lines to read from one file before [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) engine can parallelize reading, when reading from remote filesystem.
|
The minimum number of lines to read from one file before the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) engine can parallelize reading, when reading from remote filesystem.
|
||||||
|
|
||||||
Possible values:
|
Possible values:
|
||||||
|
|
||||||
@ -706,7 +728,7 @@ log_queries=1
|
|||||||
|
|
||||||
## log_queries_min_query_duration_ms {#settings-log-queries-min-query-duration-ms}
|
## log_queries_min_query_duration_ms {#settings-log-queries-min-query-duration-ms}
|
||||||
|
|
||||||
If enabled (non-zero), queries faster then the value of this setting will not be logged (you can think about this as a `long_query_time` for [MySQL Slow Query Log](https://dev.mysql.com/doc/refman/5.7/en/slow-query-log.html)), and this basically means that you will not find them in the following tables:
|
If enabled (non-zero), queries faster than the value of this setting will not be logged (you can think about this as a `long_query_time` for [MySQL Slow Query Log](https://dev.mysql.com/doc/refman/5.7/en/slow-query-log.html)), and this basically means that you will not find them in the following tables:
|
||||||
|
|
||||||
- `system.query_log`
|
- `system.query_log`
|
||||||
- `system.query_thread_log`
|
- `system.query_thread_log`
|
||||||
@ -741,7 +763,7 @@ log_queries_min_type='EXCEPTION_WHILE_PROCESSING'
|
|||||||
|
|
||||||
Setting up query threads logging.
|
Setting up query threads logging.
|
||||||
|
|
||||||
Query threads log into [system.query_thread_log](../../operations/system-tables/query_thread_log.md) table. This setting have effect only when [log_queries](#settings-log-queries) is true. Queries’ threads run by ClickHouse with this setup are logged according to the rules in the [query_thread_log](../../operations/server-configuration-parameters/settings.md/#server_configuration_parameters-query_thread_log) server configuration parameter.
|
Query threads log into the [system.query_thread_log](../../operations/system-tables/query_thread_log.md) table. This setting has effect only when [log_queries](#settings-log-queries) is true. Queries’ threads run by ClickHouse with this setup are logged according to the rules in the [query_thread_log](../../operations/server-configuration-parameters/settings.md/#server_configuration_parameters-query_thread_log) server configuration parameter.
|
||||||
|
|
||||||
Possible values:
|
Possible values:
|
||||||
|
|
||||||
@ -760,7 +782,7 @@ log_query_threads=1
|
|||||||
|
|
||||||
Setting up query views logging.
|
Setting up query views logging.
|
||||||
|
|
||||||
When a query run by ClickHouse with this setup on has associated views (materialized or live views), they are logged in the [query_views_log](../../operations/server-configuration-parameters/settings.md/#server_configuration_parameters-query_views_log) server configuration parameter.
|
When a query run by ClickHouse with this setting enabled has associated views (materialized or live views), they are logged in the [query_views_log](../../operations/server-configuration-parameters/settings.md/#server_configuration_parameters-query_views_log) server configuration parameter.
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
|
|
||||||
@ -787,7 +809,7 @@ It can be used to improve the readability of server logs. Additionally, it helps
|
|||||||
|
|
||||||
Possible values:
|
Possible values:
|
||||||
|
|
||||||
- Any string no longer than [max_query_size](#settings-max_query_size). If length is exceeded, the server throws an exception.
|
- Any string no longer than [max_query_size](#settings-max_query_size). If the max_query_size is exceeded, the server throws an exception.
|
||||||
|
|
||||||
Default value: empty string.
|
Default value: empty string.
|
||||||
|
|
||||||
@ -821,11 +843,11 @@ The setting also does not have a purpose when using INSERT SELECT, since data is
|
|||||||
|
|
||||||
Default value: 1,048,576.
|
Default value: 1,048,576.
|
||||||
|
|
||||||
The default is slightly more than `max_block_size`. The reason for this is because certain table engines (`*MergeTree`) form a data part on the disk for each inserted block, which is a fairly large entity. Similarly, `*MergeTree` tables sort data during insertion, and a large enough block size allow sorting more data in RAM.
|
The default is slightly more than `max_block_size`. The reason for this is that certain table engines (`*MergeTree`) form a data part on the disk for each inserted block, which is a fairly large entity. Similarly, `*MergeTree` tables sort data during insertion, and a large enough block size allow sorting more data in RAM.
|
||||||
|
|
||||||
## min_insert_block_size_rows {#min-insert-block-size-rows}
|
## min_insert_block_size_rows {#min-insert-block-size-rows}
|
||||||
|
|
||||||
Sets the minimum number of rows in the block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones.
|
Sets the minimum number of rows in the block that can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones.
|
||||||
|
|
||||||
Possible values:
|
Possible values:
|
||||||
|
|
||||||
@ -891,7 +913,7 @@ Higher values will lead to higher memory usage.
|
|||||||
|
|
||||||
## max_compress_block_size {#max-compress-block-size}
|
## max_compress_block_size {#max-compress-block-size}
|
||||||
|
|
||||||
The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). Specifying smaller block size generally leads to slightly reduced compression ratio, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced.
|
The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). Specifying a smaller block size generally leads to slightly reduced compression ratio, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced.
|
||||||
|
|
||||||
:::warning
|
:::warning
|
||||||
This is an expert-level setting, and you shouldn't change it if you're just getting started with ClickHouse.
|
This is an expert-level setting, and you shouldn't change it if you're just getting started with ClickHouse.
|
||||||
@ -935,7 +957,7 @@ Default value: 1000.
|
|||||||
|
|
||||||
## interactive_delay {#interactive-delay}
|
## interactive_delay {#interactive-delay}
|
||||||
|
|
||||||
The interval in microseconds for checking whether request execution has been cancelled and sending the progress.
|
The interval in microseconds for checking whether request execution has been canceled and sending the progress.
|
||||||
|
|
||||||
Default value: 100,000 (checks for cancelling and sends the progress ten times per second).
|
Default value: 100,000 (checks for cancelling and sends the progress ten times per second).
|
||||||
|
|
||||||
@ -4122,7 +4144,20 @@ Enabled by default.
|
|||||||
|
|
||||||
Serialize named tuple columns as JSON objects.
|
Serialize named tuple columns as JSON objects.
|
||||||
|
|
||||||
Disabled by default.
|
Enabled by default.
|
||||||
|
|
||||||
|
### input_format_json_named_tuples_as_objects {#input_format_json_named_tuples_as_objects}
|
||||||
|
|
||||||
|
Parse named tuple columns as JSON objects.
|
||||||
|
|
||||||
|
Enabled by default.
|
||||||
|
|
||||||
|
### input_format_json_defaults_for_missing_elements_in_named_tuple {#input_format_json_defaults_for_missing_elements_in_named_tuple}
|
||||||
|
|
||||||
|
Insert default values for missing elements in JSON object while parsing named tuple.
|
||||||
|
This setting works only when setting `input_format_json_named_tuples_as_objects` is enabled.
|
||||||
|
|
||||||
|
Enabled by default.
|
||||||
|
|
||||||
### output_format_json_array_of_rows {#output_format_json_array_of_rows}
|
### output_format_json_array_of_rows {#output_format_json_array_of_rows}
|
||||||
|
|
||||||
|
@ -0,0 +1,68 @@
|
|||||||
|
---
|
||||||
|
slug: /en/sql-reference/aggregate-functions/reference/quantileInterpolatedWeighted
|
||||||
|
sidebar_position: 203
|
||||||
|
---
|
||||||
|
|
||||||
|
# quantileInterpolatedWeighted
|
||||||
|
|
||||||
|
Computes [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence using linear interpolation, taking into account the weight of each element.
|
||||||
|
|
||||||
|
To get the interpolated value, all the passed values are combined into an array, which are then sorted by their corresponding weights. Quantile interpolation is then performed using the [weighted percentile method](https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method) by building a cumulative distribution based on weights and then a linear interpolation is performed using the weights and the values to compute the quantiles.
|
||||||
|
|
||||||
|
When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function.
|
||||||
|
|
||||||
|
**Syntax**
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
quantileInterpolatedWeighted(level)(expr, weight)
|
||||||
|
```
|
||||||
|
|
||||||
|
Alias: `medianInterpolatedWeighted`.
|
||||||
|
|
||||||
|
**Arguments**
|
||||||
|
|
||||||
|
- `level` — Level of quantile. Optional parameter. Constant floating-point number from 0 to 1. We recommend using a `level` value in the range of `[0.01, 0.99]`. Default value: 0.5. At `level=0.5` the function calculates [median](https://en.wikipedia.org/wiki/Median).
|
||||||
|
- `expr` — Expression over the column values resulting in numeric [data types](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) or [DateTime](../../../sql-reference/data-types/datetime.md).
|
||||||
|
- `weight` — Column with weights of sequence members. Weight is a number of value occurrences.
|
||||||
|
|
||||||
|
**Returned value**
|
||||||
|
|
||||||
|
- Quantile of the specified level.
|
||||||
|
|
||||||
|
Type:
|
||||||
|
|
||||||
|
- [Float64](../../../sql-reference/data-types/float.md) for numeric data type input.
|
||||||
|
- [Date](../../../sql-reference/data-types/date.md) if input values have the `Date` type.
|
||||||
|
- [DateTime](../../../sql-reference/data-types/datetime.md) if input values have the `DateTime` type.
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
Input table:
|
||||||
|
|
||||||
|
``` text
|
||||||
|
┌─n─┬─val─┐
|
||||||
|
│ 0 │ 3 │
|
||||||
|
│ 1 │ 2 │
|
||||||
|
│ 2 │ 1 │
|
||||||
|
│ 5 │ 4 │
|
||||||
|
└───┴─────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
SELECT quantileInterpolatedWeighted(n, val) FROM t
|
||||||
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
``` text
|
||||||
|
┌─quantileInterpolatedWeighted(n, val)─┐
|
||||||
|
│ 1 │
|
||||||
|
└──────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**See Also**
|
||||||
|
|
||||||
|
- [median](../../../sql-reference/aggregate-functions/reference/median.md#median)
|
||||||
|
- [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles)
|
@ -9,7 +9,7 @@ sidebar_position: 201
|
|||||||
|
|
||||||
Syntax: `quantiles(level1, level2, …)(x)`
|
Syntax: `quantiles(level1, level2, …)(x)`
|
||||||
|
|
||||||
All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`, `quantilesBFloat16`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.
|
All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantileInterpolatedWeighted`, `quantilesTDigest`, `quantilesBFloat16`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.
|
||||||
|
|
||||||
## quantilesExactExclusive
|
## quantilesExactExclusive
|
||||||
|
|
||||||
|
@ -140,6 +140,7 @@ namespace CurrentMetrics
|
|||||||
namespace ProfileEvents
|
namespace ProfileEvents
|
||||||
{
|
{
|
||||||
extern const Event MainConfigLoads;
|
extern const Event MainConfigLoads;
|
||||||
|
extern const Event ServerStartupMilliseconds;
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace fs = std::filesystem;
|
namespace fs = std::filesystem;
|
||||||
@ -652,6 +653,8 @@ static void sanityChecks(Server & server)
|
|||||||
int Server::main(const std::vector<std::string> & /*args*/)
|
int Server::main(const std::vector<std::string> & /*args*/)
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
Stopwatch startup_watch;
|
||||||
|
|
||||||
Poco::Logger * log = &logger();
|
Poco::Logger * log = &logger();
|
||||||
|
|
||||||
UseSSL use_ssl;
|
UseSSL use_ssl;
|
||||||
@ -1822,6 +1825,9 @@ try
|
|||||||
LOG_INFO(log, "Ready for connections.");
|
LOG_INFO(log, "Ready for connections.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
startup_watch.stop();
|
||||||
|
ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds());
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
global_context->startClusterDiscovery();
|
global_context->startClusterDiscovery();
|
||||||
|
@ -167,6 +167,7 @@ enum class AccessType
|
|||||||
M(SYSTEM_SYNC_REPLICA, "SYNC REPLICA", TABLE, SYSTEM) \
|
M(SYSTEM_SYNC_REPLICA, "SYNC REPLICA", TABLE, SYSTEM) \
|
||||||
M(SYSTEM_RESTART_REPLICA, "RESTART REPLICA", TABLE, SYSTEM) \
|
M(SYSTEM_RESTART_REPLICA, "RESTART REPLICA", TABLE, SYSTEM) \
|
||||||
M(SYSTEM_RESTORE_REPLICA, "RESTORE REPLICA", TABLE, SYSTEM) \
|
M(SYSTEM_RESTORE_REPLICA, "RESTORE REPLICA", TABLE, SYSTEM) \
|
||||||
|
M(SYSTEM_WAIT_LOADING_PARTS, "WAIT LOADING PARTS", TABLE, SYSTEM) \
|
||||||
M(SYSTEM_SYNC_DATABASE_REPLICA, "SYNC DATABASE REPLICA", DATABASE, SYSTEM) \
|
M(SYSTEM_SYNC_DATABASE_REPLICA, "SYNC DATABASE REPLICA", DATABASE, SYSTEM) \
|
||||||
M(SYSTEM_SYNC_TRANSACTION_LOG, "SYNC TRANSACTION LOG", GLOBAL, SYSTEM) \
|
M(SYSTEM_SYNC_TRANSACTION_LOG, "SYNC TRANSACTION LOG", GLOBAL, SYSTEM) \
|
||||||
M(SYSTEM_FLUSH_DISTRIBUTED, "FLUSH DISTRIBUTED", TABLE, SYSTEM_FLUSH) \
|
M(SYSTEM_FLUSH_DISTRIBUTED, "FLUSH DISTRIBUTED", TABLE, SYSTEM_FLUSH) \
|
||||||
|
@ -53,7 +53,7 @@ TEST(AccessRights, Union)
|
|||||||
"SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
|
"SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
|
||||||
"SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
|
"SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
|
||||||
"SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
|
"SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
|
||||||
"SYSTEM RESTORE REPLICA, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
|
"SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -207,7 +207,7 @@ private:
|
|||||||
{
|
{
|
||||||
// Fuse points if their text representations differ only in last digit
|
// Fuse points if their text representations differ only in last digit
|
||||||
auto min_diff = 10 * (points[left].mean + points[right].mean) * std::numeric_limits<Mean>::epsilon();
|
auto min_diff = 10 * (points[left].mean + points[right].mean) * std::numeric_limits<Mean>::epsilon();
|
||||||
if (points[left].mean + min_diff >= points[right].mean)
|
if (points[left].mean + std::fabs(min_diff) >= points[right].mean)
|
||||||
{
|
{
|
||||||
points[left] = points[left] + points[right];
|
points[left] = points[left] + points[right];
|
||||||
}
|
}
|
||||||
|
@ -232,6 +232,9 @@ struct NameQuantilesExactInclusive { static constexpr auto name = "quantilesExac
|
|||||||
struct NameQuantileExactWeighted { static constexpr auto name = "quantileExactWeighted"; };
|
struct NameQuantileExactWeighted { static constexpr auto name = "quantileExactWeighted"; };
|
||||||
struct NameQuantilesExactWeighted { static constexpr auto name = "quantilesExactWeighted"; };
|
struct NameQuantilesExactWeighted { static constexpr auto name = "quantilesExactWeighted"; };
|
||||||
|
|
||||||
|
struct NameQuantileInterpolatedWeighted { static constexpr auto name = "quantileInterpolatedWeighted"; };
|
||||||
|
struct NameQuantilesInterpolatedWeighted { static constexpr auto name = "quantilesInterpolatedWeighted"; };
|
||||||
|
|
||||||
struct NameQuantileTiming { static constexpr auto name = "quantileTiming"; };
|
struct NameQuantileTiming { static constexpr auto name = "quantileTiming"; };
|
||||||
struct NameQuantileTimingWeighted { static constexpr auto name = "quantileTimingWeighted"; };
|
struct NameQuantileTimingWeighted { static constexpr auto name = "quantileTimingWeighted"; };
|
||||||
struct NameQuantilesTiming { static constexpr auto name = "quantilesTiming"; };
|
struct NameQuantilesTiming { static constexpr auto name = "quantilesTiming"; };
|
||||||
|
@ -0,0 +1,70 @@
|
|||||||
|
#include <AggregateFunctions/AggregateFunctionQuantile.h>
|
||||||
|
#include <AggregateFunctions/QuantileInterpolatedWeighted.h>
|
||||||
|
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||||
|
#include <AggregateFunctions/Helpers.h>
|
||||||
|
#include <DataTypes/DataTypeDate.h>
|
||||||
|
#include <DataTypes/DataTypeDateTime.h>
|
||||||
|
#include <Core/Field.h>
|
||||||
|
|
||||||
|
|
||||||
|
namespace DB
|
||||||
|
{
|
||||||
|
struct Settings;
|
||||||
|
|
||||||
|
namespace ErrorCodes
|
||||||
|
{
|
||||||
|
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
|
||||||
|
template <typename Value, bool _> using FuncQuantileInterpolatedWeighted = AggregateFunctionQuantile<Value, QuantileInterpolatedWeighted<Value>, NameQuantileInterpolatedWeighted, true, void, false>;
|
||||||
|
template <typename Value, bool _> using FuncQuantilesInterpolatedWeighted = AggregateFunctionQuantile<Value, QuantileInterpolatedWeighted<Value>, NameQuantilesInterpolatedWeighted, true, void, true>;
|
||||||
|
|
||||||
|
template <template <typename, bool> class Function>
|
||||||
|
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||||
|
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||||
|
{
|
||||||
|
/// Second argument type check doesn't depend on the type of the first one.
|
||||||
|
Function<void, true>::assertSecondArg(argument_types);
|
||||||
|
|
||||||
|
const DataTypePtr & argument_type = argument_types[0];
|
||||||
|
WhichDataType which(argument_type);
|
||||||
|
|
||||||
|
#define DISPATCH(TYPE) \
|
||||||
|
if (which.idx == TypeIndex::TYPE) return std::make_shared<Function<TYPE, true>>(argument_types, params);
|
||||||
|
FOR_BASIC_NUMERIC_TYPES(DISPATCH)
|
||||||
|
#undef DISPATCH
|
||||||
|
if (which.idx == TypeIndex::Date) return std::make_shared<Function<DataTypeDate::FieldType, false>>(argument_types, params);
|
||||||
|
if (which.idx == TypeIndex::DateTime) return std::make_shared<Function<DataTypeDateTime::FieldType, false>>(argument_types, params);
|
||||||
|
|
||||||
|
if (which.idx == TypeIndex::Decimal32) return std::make_shared<Function<Decimal32, false>>(argument_types, params);
|
||||||
|
if (which.idx == TypeIndex::Decimal64) return std::make_shared<Function<Decimal64, false>>(argument_types, params);
|
||||||
|
if (which.idx == TypeIndex::Decimal128) return std::make_shared<Function<Decimal128, false>>(argument_types, params);
|
||||||
|
if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false>>(argument_types, params);
|
||||||
|
if (which.idx == TypeIndex::DateTime64) return std::make_shared<Function<DateTime64, false>>(argument_types, params);
|
||||||
|
|
||||||
|
if (which.idx == TypeIndex::Int128) return std::make_shared<Function<Int128, true>>(argument_types, params);
|
||||||
|
if (which.idx == TypeIndex::UInt128) return std::make_shared<Function<UInt128, true>>(argument_types, params);
|
||||||
|
if (which.idx == TypeIndex::Int256) return std::make_shared<Function<Int256, true>>(argument_types, params);
|
||||||
|
if (which.idx == TypeIndex::UInt256) return std::make_shared<Function<UInt256, true>>(argument_types, params);
|
||||||
|
|
||||||
|
throw Exception("Illegal type " + argument_type->getName() + " of argument for aggregate function " + name,
|
||||||
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void registerAggregateFunctionsQuantileInterpolatedWeighted(AggregateFunctionFactory & factory)
|
||||||
|
{
|
||||||
|
/// For aggregate functions returning array we cannot return NULL on empty set.
|
||||||
|
AggregateFunctionProperties properties = { .returns_default_when_only_null = true };
|
||||||
|
|
||||||
|
factory.registerFunction(NameQuantileInterpolatedWeighted::name, createAggregateFunctionQuantile<FuncQuantileInterpolatedWeighted>);
|
||||||
|
factory.registerFunction(NameQuantilesInterpolatedWeighted::name, { createAggregateFunctionQuantile<FuncQuantilesInterpolatedWeighted>, properties });
|
||||||
|
|
||||||
|
/// 'median' is an alias for 'quantile'
|
||||||
|
factory.registerAlias("medianInterpolatedWeighted", NameQuantileInterpolatedWeighted::name);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
308
src/AggregateFunctions/QuantileInterpolatedWeighted.h
Normal file
308
src/AggregateFunctions/QuantileInterpolatedWeighted.h
Normal file
@ -0,0 +1,308 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <base/sort.h>
|
||||||
|
|
||||||
|
#include <Common/HashTable/HashMap.h>
|
||||||
|
#include <Common/NaNUtils.h>
|
||||||
|
|
||||||
|
|
||||||
|
namespace DB
|
||||||
|
{
|
||||||
|
struct Settings;
|
||||||
|
|
||||||
|
namespace ErrorCodes
|
||||||
|
{
|
||||||
|
extern const int NOT_IMPLEMENTED;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Approximates Quantile by:
|
||||||
|
* - sorting input values and weights
|
||||||
|
* - building a cumulative distribution based on weights
|
||||||
|
* - performing linear interpolation between the weights and values
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
template <typename Value>
|
||||||
|
struct QuantileInterpolatedWeighted
|
||||||
|
{
|
||||||
|
struct Int128Hash
|
||||||
|
{
|
||||||
|
size_t operator()(Int128 x) const
|
||||||
|
{
|
||||||
|
return CityHash_v1_0_2::Hash128to64({x >> 64, x & 0xffffffffffffffffll});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
using Weight = UInt64;
|
||||||
|
using UnderlyingType = NativeType<Value>;
|
||||||
|
using Hasher = std::conditional_t<std::is_same_v<Value, Decimal128>, Int128Hash, HashCRC32<UnderlyingType>>;
|
||||||
|
|
||||||
|
/// When creating, the hash table must be small.
|
||||||
|
using Map = HashMapWithStackMemory<UnderlyingType, Weight, Hasher, 4>;
|
||||||
|
|
||||||
|
Map map;
|
||||||
|
|
||||||
|
void add(const Value & x)
|
||||||
|
{
|
||||||
|
/// We must skip NaNs as they are not compatible with comparison sorting.
|
||||||
|
if (!isNaN(x))
|
||||||
|
++map[x];
|
||||||
|
}
|
||||||
|
|
||||||
|
void add(const Value & x, Weight weight)
|
||||||
|
{
|
||||||
|
if (!isNaN(x))
|
||||||
|
map[x] += weight;
|
||||||
|
}
|
||||||
|
|
||||||
|
void merge(const QuantileInterpolatedWeighted & rhs)
|
||||||
|
{
|
||||||
|
for (const auto & pair : rhs.map)
|
||||||
|
map[pair.getKey()] += pair.getMapped();
|
||||||
|
}
|
||||||
|
|
||||||
|
void serialize(WriteBuffer & buf) const
|
||||||
|
{
|
||||||
|
map.write(buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
void deserialize(ReadBuffer & buf)
|
||||||
|
{
|
||||||
|
typename Map::Reader reader(buf);
|
||||||
|
while (reader.next())
|
||||||
|
{
|
||||||
|
const auto & pair = reader.get();
|
||||||
|
map[pair.first] = pair.second;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Value get(Float64 level) const
|
||||||
|
{
|
||||||
|
return getImpl<Value>(level);
|
||||||
|
}
|
||||||
|
|
||||||
|
void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result) const
|
||||||
|
{
|
||||||
|
getManyImpl<Value>(levels, indices, size, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same, but in the case of an empty state, NaN is returned.
|
||||||
|
Float64 getFloat(Float64) const
|
||||||
|
{
|
||||||
|
throw Exception("Method getFloat is not implemented for QuantileInterpolatedWeighted", ErrorCodes::NOT_IMPLEMENTED);
|
||||||
|
}
|
||||||
|
|
||||||
|
void getManyFloat(const Float64 *, const size_t *, size_t, Float64 *) const
|
||||||
|
{
|
||||||
|
throw Exception("Method getManyFloat is not implemented for QuantileInterpolatedWeighted", ErrorCodes::NOT_IMPLEMENTED);
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
using Pair = typename std::pair<UnderlyingType, Float64>;
|
||||||
|
|
||||||
|
/// Get the value of the `level` quantile. The level must be between 0 and 1.
|
||||||
|
template <typename T>
|
||||||
|
T getImpl(Float64 level) const
|
||||||
|
{
|
||||||
|
size_t size = map.size();
|
||||||
|
|
||||||
|
if (0 == size)
|
||||||
|
return std::numeric_limits<Value>::quiet_NaN();
|
||||||
|
|
||||||
|
/// Maintain a vector of pair of values and weights for easier sorting and for building
|
||||||
|
/// a cumulative distribution using the provided weights.
|
||||||
|
std::vector<Pair> value_weight_pairs;
|
||||||
|
value_weight_pairs.reserve(size);
|
||||||
|
|
||||||
|
/// Note: weight provided must be a 64-bit integer
|
||||||
|
/// Float64 is used as accumulator here to get approximate results.
|
||||||
|
/// But weight used in the internal array is stored as Float64 as we
|
||||||
|
/// do some quantile estimation operation which involves division and
|
||||||
|
/// require Float64 level of precision.
|
||||||
|
|
||||||
|
Float64 sum_weight = 0;
|
||||||
|
for (const auto & pair : map)
|
||||||
|
{
|
||||||
|
sum_weight += pair.getMapped();
|
||||||
|
auto value = pair.getKey();
|
||||||
|
auto weight = pair.getMapped();
|
||||||
|
value_weight_pairs.push_back({value, weight});
|
||||||
|
}
|
||||||
|
|
||||||
|
::sort(value_weight_pairs.begin(), value_weight_pairs.end(), [](const Pair & a, const Pair & b) { return a.first < b.first; });
|
||||||
|
|
||||||
|
Float64 accumulated = 0;
|
||||||
|
|
||||||
|
/// vector for populating and storing the cumulative sum using the provided weights.
|
||||||
|
/// example: [0,1,2,3,4,5] -> [0,1,3,6,10,15]
|
||||||
|
std::vector<Float64> weights_cum_sum;
|
||||||
|
weights_cum_sum.reserve(size);
|
||||||
|
|
||||||
|
for (size_t idx = 0; idx < size; ++idx)
|
||||||
|
{
|
||||||
|
accumulated += value_weight_pairs[idx].second;
|
||||||
|
weights_cum_sum.push_back(accumulated);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The following estimation of quantile is general and the idea is:
|
||||||
|
/// https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method
|
||||||
|
|
||||||
|
/// calculates a simple cumulative distribution based on weights
|
||||||
|
if (sum_weight != 0)
|
||||||
|
{
|
||||||
|
for (size_t idx = 0; idx < size; ++idx)
|
||||||
|
value_weight_pairs[idx].second = (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// perform linear interpolation
|
||||||
|
size_t idx = 0;
|
||||||
|
if (size >= 2)
|
||||||
|
{
|
||||||
|
if (level >= value_weight_pairs[size - 2].second)
|
||||||
|
{
|
||||||
|
idx = size - 2;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
size_t start = 0, end = size - 1;
|
||||||
|
while (start <= end)
|
||||||
|
{
|
||||||
|
size_t mid = start + (end - start) / 2;
|
||||||
|
if (mid > size)
|
||||||
|
break;
|
||||||
|
if (level > value_weight_pairs[mid + 1].second)
|
||||||
|
start = mid + 1;
|
||||||
|
else
|
||||||
|
{
|
||||||
|
idx = mid;
|
||||||
|
end = mid - 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t l = idx;
|
||||||
|
size_t u = idx + 1 < size ? idx + 1 : idx;
|
||||||
|
|
||||||
|
Float64 xl = value_weight_pairs[l].second, xr = value_weight_pairs[u].second;
|
||||||
|
UnderlyingType yl = value_weight_pairs[l].first, yr = value_weight_pairs[u].first;
|
||||||
|
|
||||||
|
if (level < xl)
|
||||||
|
yr = yl;
|
||||||
|
if (level > xr)
|
||||||
|
yl = yr;
|
||||||
|
|
||||||
|
return static_cast<T>(interpolate(level, xl, xr, yl, yr));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get the `size` values of `levels` quantiles. Write `size` results starting with `result` address.
|
||||||
|
/// indices - an array of index levels such that the corresponding elements will go in ascending order.
|
||||||
|
template <typename T>
|
||||||
|
void getManyImpl(const Float64 * levels, const size_t * indices, size_t num_levels, Value * result) const
|
||||||
|
{
|
||||||
|
size_t size = map.size();
|
||||||
|
|
||||||
|
if (0 == size)
|
||||||
|
{
|
||||||
|
for (size_t i = 0; i < num_levels; ++i)
|
||||||
|
result[i] = Value();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<Pair> value_weight_pairs;
|
||||||
|
value_weight_pairs.reserve(size);
|
||||||
|
|
||||||
|
Float64 sum_weight = 0;
|
||||||
|
for (const auto & pair : map)
|
||||||
|
{
|
||||||
|
sum_weight += pair.getMapped();
|
||||||
|
auto value = pair.getKey();
|
||||||
|
auto weight = pair.getMapped();
|
||||||
|
value_weight_pairs.push_back({value, weight});
|
||||||
|
}
|
||||||
|
|
||||||
|
::sort(value_weight_pairs.begin(), value_weight_pairs.end(), [](const Pair & a, const Pair & b) { return a.first < b.first; });
|
||||||
|
|
||||||
|
Float64 accumulated = 0;
|
||||||
|
|
||||||
|
/// vector for populating and storing the cumulative sum using the provided weights.
|
||||||
|
/// example: [0,1,2,3,4,5] -> [0,1,3,6,10,15]
|
||||||
|
std::vector<Float64> weights_cum_sum;
|
||||||
|
weights_cum_sum.reserve(size);
|
||||||
|
|
||||||
|
for (size_t idx = 0; idx < size; ++idx)
|
||||||
|
{
|
||||||
|
accumulated += value_weight_pairs[idx].second;
|
||||||
|
weights_cum_sum.emplace_back(accumulated);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/// The following estimation of quantile is general and the idea is:
|
||||||
|
/// https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method
|
||||||
|
|
||||||
|
/// calculates a simple cumulative distribution based on weights
|
||||||
|
if (sum_weight != 0)
|
||||||
|
{
|
||||||
|
for (size_t idx = 0; idx < size; ++idx)
|
||||||
|
value_weight_pairs[idx].second = (weights_cum_sum[idx] - 0.5 * value_weight_pairs[idx].second) / sum_weight;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (size_t level_index = 0; level_index < num_levels; ++level_index)
|
||||||
|
{
|
||||||
|
/// perform linear interpolation for every level
|
||||||
|
auto level = levels[indices[level_index]];
|
||||||
|
|
||||||
|
size_t idx = 0;
|
||||||
|
if (size >= 2)
|
||||||
|
{
|
||||||
|
if (level >= value_weight_pairs[size - 2].second)
|
||||||
|
{
|
||||||
|
idx = size - 2;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
size_t start = 0, end = size - 1;
|
||||||
|
while (start <= end)
|
||||||
|
{
|
||||||
|
size_t mid = start + (end - start) / 2;
|
||||||
|
if (mid > size)
|
||||||
|
break;
|
||||||
|
if (level > value_weight_pairs[mid + 1].second)
|
||||||
|
start = mid + 1;
|
||||||
|
else
|
||||||
|
{
|
||||||
|
idx = mid;
|
||||||
|
end = mid - 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t l = idx;
|
||||||
|
size_t u = idx + 1 < size ? idx + 1 : idx;
|
||||||
|
|
||||||
|
Float64 xl = value_weight_pairs[l].second, xr = value_weight_pairs[u].second;
|
||||||
|
UnderlyingType yl = value_weight_pairs[l].first, yr = value_weight_pairs[u].first;
|
||||||
|
|
||||||
|
if (level < xl)
|
||||||
|
yr = yl;
|
||||||
|
if (level > xr)
|
||||||
|
yl = yr;
|
||||||
|
|
||||||
|
result[indices[level_index]] = static_cast<T>(interpolate(level, xl, xr, yl, yr));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// This ignores overflows or NaN's that might arise during add, sub and mul operations and doesn't aim to provide exact
|
||||||
|
/// results since `the quantileInterpolatedWeighted` function itself relies mainly on approximation.
|
||||||
|
UnderlyingType NO_SANITIZE_UNDEFINED interpolate(Float64 level, Float64 xl, Float64 xr, UnderlyingType yl, UnderlyingType yr) const
|
||||||
|
{
|
||||||
|
UnderlyingType dy = yr - yl;
|
||||||
|
Float64 dx = xr - xl;
|
||||||
|
dx = dx == 0 ? 1 : dx; /// to handle NaN behavior that might arise during integer division below.
|
||||||
|
|
||||||
|
/// yl + (dy / dx) * (level - xl)
|
||||||
|
return static_cast<UnderlyingType>(yl + (dy / dx) * (level - xl));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
@ -21,6 +21,7 @@ void registerAggregateFunctionsQuantile(AggregateFunctionFactory &);
|
|||||||
void registerAggregateFunctionsQuantileDeterministic(AggregateFunctionFactory &);
|
void registerAggregateFunctionsQuantileDeterministic(AggregateFunctionFactory &);
|
||||||
void registerAggregateFunctionsQuantileExact(AggregateFunctionFactory &);
|
void registerAggregateFunctionsQuantileExact(AggregateFunctionFactory &);
|
||||||
void registerAggregateFunctionsQuantileExactWeighted(AggregateFunctionFactory &);
|
void registerAggregateFunctionsQuantileExactWeighted(AggregateFunctionFactory &);
|
||||||
|
void registerAggregateFunctionsQuantileInterpolatedWeighted(AggregateFunctionFactory &);
|
||||||
void registerAggregateFunctionsQuantileExactLow(AggregateFunctionFactory &);
|
void registerAggregateFunctionsQuantileExactLow(AggregateFunctionFactory &);
|
||||||
void registerAggregateFunctionsQuantileExactHigh(AggregateFunctionFactory &);
|
void registerAggregateFunctionsQuantileExactHigh(AggregateFunctionFactory &);
|
||||||
void registerAggregateFunctionsQuantileExactInclusive(AggregateFunctionFactory &);
|
void registerAggregateFunctionsQuantileExactInclusive(AggregateFunctionFactory &);
|
||||||
@ -106,6 +107,7 @@ void registerAggregateFunctions()
|
|||||||
registerAggregateFunctionsQuantileDeterministic(factory);
|
registerAggregateFunctionsQuantileDeterministic(factory);
|
||||||
registerAggregateFunctionsQuantileExact(factory);
|
registerAggregateFunctionsQuantileExact(factory);
|
||||||
registerAggregateFunctionsQuantileExactWeighted(factory);
|
registerAggregateFunctionsQuantileExactWeighted(factory);
|
||||||
|
registerAggregateFunctionsQuantileInterpolatedWeighted(factory);
|
||||||
registerAggregateFunctionsQuantileExactLow(factory);
|
registerAggregateFunctionsQuantileExactLow(factory);
|
||||||
registerAggregateFunctionsQuantileExactHigh(factory);
|
registerAggregateFunctionsQuantileExactHigh(factory);
|
||||||
registerAggregateFunctionsQuantileExactInclusive(factory);
|
registerAggregateFunctionsQuantileExactInclusive(factory);
|
||||||
|
@ -11,6 +11,7 @@
|
|||||||
#include <Parsers/ASTQualifiedAsterisk.h>
|
#include <Parsers/ASTQualifiedAsterisk.h>
|
||||||
#include <Parsers/ASTColumnsMatcher.h>
|
#include <Parsers/ASTColumnsMatcher.h>
|
||||||
#include <Parsers/ASTExpressionList.h>
|
#include <Parsers/ASTExpressionList.h>
|
||||||
|
#include <Parsers/ASTColumnsTransformers.h>
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
@ -206,19 +207,43 @@ QueryTreeNodePtr MatcherNode::cloneImpl() const
|
|||||||
ASTPtr MatcherNode::toASTImpl() const
|
ASTPtr MatcherNode::toASTImpl() const
|
||||||
{
|
{
|
||||||
ASTPtr result;
|
ASTPtr result;
|
||||||
|
ASTPtr transformers;
|
||||||
|
|
||||||
|
if (!children.empty())
|
||||||
|
{
|
||||||
|
transformers = std::make_shared<ASTColumnsTransformerList>();
|
||||||
|
|
||||||
|
for (const auto & child : children)
|
||||||
|
transformers->children.push_back(child->toAST());
|
||||||
|
}
|
||||||
|
|
||||||
if (matcher_type == MatcherNodeType::ASTERISK)
|
if (matcher_type == MatcherNodeType::ASTERISK)
|
||||||
{
|
{
|
||||||
if (qualified_identifier.empty())
|
if (qualified_identifier.empty())
|
||||||
{
|
{
|
||||||
result = std::make_shared<ASTAsterisk>();
|
auto asterisk = std::make_shared<ASTAsterisk>();
|
||||||
|
|
||||||
|
if (transformers)
|
||||||
|
{
|
||||||
|
asterisk->transformers = std::move(transformers);
|
||||||
|
asterisk->children.push_back(asterisk->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
|
result = asterisk;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
auto qualified_asterisk = std::make_shared<ASTQualifiedAsterisk>();
|
auto qualified_asterisk = std::make_shared<ASTQualifiedAsterisk>();
|
||||||
|
|
||||||
auto identifier_parts = qualified_identifier.getParts();
|
auto identifier_parts = qualified_identifier.getParts();
|
||||||
qualified_asterisk->children.push_back(std::make_shared<ASTIdentifier>(std::move(identifier_parts)));
|
qualified_asterisk->qualifier = std::make_shared<ASTIdentifier>(std::move(identifier_parts));
|
||||||
|
qualified_asterisk->children.push_back(qualified_asterisk->qualifier);
|
||||||
|
|
||||||
|
if (transformers)
|
||||||
|
{
|
||||||
|
qualified_asterisk->transformers = std::move(transformers);
|
||||||
|
qualified_asterisk->children.push_back(qualified_asterisk->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
result = qualified_asterisk;
|
result = qualified_asterisk;
|
||||||
}
|
}
|
||||||
@ -229,6 +254,13 @@ ASTPtr MatcherNode::toASTImpl() const
|
|||||||
{
|
{
|
||||||
auto regexp_matcher = std::make_shared<ASTColumnsRegexpMatcher>();
|
auto regexp_matcher = std::make_shared<ASTColumnsRegexpMatcher>();
|
||||||
regexp_matcher->setPattern(columns_matcher->pattern());
|
regexp_matcher->setPattern(columns_matcher->pattern());
|
||||||
|
|
||||||
|
if (transformers)
|
||||||
|
{
|
||||||
|
regexp_matcher->transformers = std::move(transformers);
|
||||||
|
regexp_matcher->children.push_back(regexp_matcher->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
result = regexp_matcher;
|
result = regexp_matcher;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@ -237,7 +269,14 @@ ASTPtr MatcherNode::toASTImpl() const
|
|||||||
regexp_matcher->setPattern(columns_matcher->pattern());
|
regexp_matcher->setPattern(columns_matcher->pattern());
|
||||||
|
|
||||||
auto identifier_parts = qualified_identifier.getParts();
|
auto identifier_parts = qualified_identifier.getParts();
|
||||||
regexp_matcher->children.push_back(std::make_shared<ASTIdentifier>(std::move(identifier_parts)));
|
regexp_matcher->qualifier = std::make_shared<ASTIdentifier>(std::move(identifier_parts));
|
||||||
|
regexp_matcher->children.push_back(regexp_matcher->qualifier);
|
||||||
|
|
||||||
|
if (transformers)
|
||||||
|
{
|
||||||
|
regexp_matcher->transformers = std::move(transformers);
|
||||||
|
regexp_matcher->children.push_back(regexp_matcher->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
result = regexp_matcher;
|
result = regexp_matcher;
|
||||||
}
|
}
|
||||||
@ -257,23 +296,36 @@ ASTPtr MatcherNode::toASTImpl() const
|
|||||||
{
|
{
|
||||||
auto columns_list_matcher = std::make_shared<ASTColumnsListMatcher>();
|
auto columns_list_matcher = std::make_shared<ASTColumnsListMatcher>();
|
||||||
columns_list_matcher->column_list = std::move(column_list);
|
columns_list_matcher->column_list = std::move(column_list);
|
||||||
|
columns_list_matcher->children.push_back(columns_list_matcher->column_list);
|
||||||
|
|
||||||
|
if (transformers)
|
||||||
|
{
|
||||||
|
columns_list_matcher->transformers = std::move(transformers);
|
||||||
|
columns_list_matcher->children.push_back(columns_list_matcher->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
result = columns_list_matcher;
|
result = columns_list_matcher;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
auto columns_list_matcher = std::make_shared<ASTQualifiedColumnsListMatcher>();
|
auto columns_list_matcher = std::make_shared<ASTQualifiedColumnsListMatcher>();
|
||||||
columns_list_matcher->column_list = std::move(column_list);
|
|
||||||
|
|
||||||
auto identifier_parts = qualified_identifier.getParts();
|
auto identifier_parts = qualified_identifier.getParts();
|
||||||
columns_list_matcher->children.push_back(std::make_shared<ASTIdentifier>(std::move(identifier_parts)));
|
columns_list_matcher->qualifier = std::make_shared<ASTIdentifier>(std::move(identifier_parts));
|
||||||
|
columns_list_matcher->column_list = std::move(column_list);
|
||||||
|
columns_list_matcher->children.push_back(columns_list_matcher->qualifier);
|
||||||
|
columns_list_matcher->children.push_back(columns_list_matcher->column_list);
|
||||||
|
|
||||||
|
if (transformers)
|
||||||
|
{
|
||||||
|
columns_list_matcher->transformers = std::move(transformers);
|
||||||
|
columns_list_matcher->children.push_back(columns_list_matcher->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
result = columns_list_matcher;
|
result = columns_list_matcher;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const auto & child : children)
|
|
||||||
result->children.push_back(child->toAST());
|
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -111,7 +111,7 @@ private:
|
|||||||
|
|
||||||
QueryTreeNodePtr buildJoinTree(const ASTPtr & tables_in_select_query, const ContextPtr & context) const;
|
QueryTreeNodePtr buildJoinTree(const ASTPtr & tables_in_select_query, const ContextPtr & context) const;
|
||||||
|
|
||||||
ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index, const ContextPtr & context) const;
|
ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression, const ContextPtr & context) const;
|
||||||
|
|
||||||
ASTPtr query;
|
ASTPtr query;
|
||||||
QueryTreeNodePtr query_tree_node;
|
QueryTreeNodePtr query_tree_node;
|
||||||
@ -439,13 +439,13 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co
|
|||||||
}
|
}
|
||||||
else if (const auto * asterisk = expression->as<ASTAsterisk>())
|
else if (const auto * asterisk = expression->as<ASTAsterisk>())
|
||||||
{
|
{
|
||||||
auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/, context);
|
auto column_transformers = buildColumnTransformers(asterisk->transformers, context);
|
||||||
result = std::make_shared<MatcherNode>(std::move(column_transformers));
|
result = std::make_shared<MatcherNode>(std::move(column_transformers));
|
||||||
}
|
}
|
||||||
else if (const auto * qualified_asterisk = expression->as<ASTQualifiedAsterisk>())
|
else if (const auto * qualified_asterisk = expression->as<ASTQualifiedAsterisk>())
|
||||||
{
|
{
|
||||||
auto & qualified_identifier = qualified_asterisk->children.at(0)->as<ASTTableIdentifier &>();
|
auto & qualified_identifier = qualified_asterisk->qualifier->as<ASTIdentifier &>();
|
||||||
auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/, context);
|
auto column_transformers = buildColumnTransformers(qualified_asterisk->transformers, context);
|
||||||
result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), std::move(column_transformers));
|
result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), std::move(column_transformers));
|
||||||
}
|
}
|
||||||
else if (const auto * ast_literal = expression->as<ASTLiteral>())
|
else if (const auto * ast_literal = expression->as<ASTLiteral>())
|
||||||
@ -543,7 +543,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co
|
|||||||
}
|
}
|
||||||
else if (const auto * columns_regexp_matcher = expression->as<ASTColumnsRegexpMatcher>())
|
else if (const auto * columns_regexp_matcher = expression->as<ASTColumnsRegexpMatcher>())
|
||||||
{
|
{
|
||||||
auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/, context);
|
auto column_transformers = buildColumnTransformers(columns_regexp_matcher->transformers, context);
|
||||||
result = std::make_shared<MatcherNode>(columns_regexp_matcher->getMatcher(), std::move(column_transformers));
|
result = std::make_shared<MatcherNode>(columns_regexp_matcher->getMatcher(), std::move(column_transformers));
|
||||||
}
|
}
|
||||||
else if (const auto * columns_list_matcher = expression->as<ASTColumnsListMatcher>())
|
else if (const auto * columns_list_matcher = expression->as<ASTColumnsListMatcher>())
|
||||||
@ -557,18 +557,18 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co
|
|||||||
column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts});
|
column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts});
|
||||||
}
|
}
|
||||||
|
|
||||||
auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/, context);
|
auto column_transformers = buildColumnTransformers(columns_list_matcher->transformers, context);
|
||||||
result = std::make_shared<MatcherNode>(std::move(column_list_identifiers), std::move(column_transformers));
|
result = std::make_shared<MatcherNode>(std::move(column_list_identifiers), std::move(column_transformers));
|
||||||
}
|
}
|
||||||
else if (const auto * qualified_columns_regexp_matcher = expression->as<ASTQualifiedColumnsRegexpMatcher>())
|
else if (const auto * qualified_columns_regexp_matcher = expression->as<ASTQualifiedColumnsRegexpMatcher>())
|
||||||
{
|
{
|
||||||
auto & qualified_identifier = qualified_columns_regexp_matcher->children.at(0)->as<ASTTableIdentifier &>();
|
auto & qualified_identifier = qualified_columns_regexp_matcher->qualifier->as<ASTIdentifier &>();
|
||||||
auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/, context);
|
auto column_transformers = buildColumnTransformers(qualified_columns_regexp_matcher->transformers, context);
|
||||||
result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getMatcher(), std::move(column_transformers));
|
result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getMatcher(), std::move(column_transformers));
|
||||||
}
|
}
|
||||||
else if (const auto * qualified_columns_list_matcher = expression->as<ASTQualifiedColumnsListMatcher>())
|
else if (const auto * qualified_columns_list_matcher = expression->as<ASTQualifiedColumnsListMatcher>())
|
||||||
{
|
{
|
||||||
auto & qualified_identifier = qualified_columns_list_matcher->children.at(0)->as<ASTTableIdentifier &>();
|
auto & qualified_identifier = qualified_columns_list_matcher->qualifier->as<ASTIdentifier &>();
|
||||||
|
|
||||||
Identifiers column_list_identifiers;
|
Identifiers column_list_identifiers;
|
||||||
column_list_identifiers.reserve(qualified_columns_list_matcher->column_list->children.size());
|
column_list_identifiers.reserve(qualified_columns_list_matcher->column_list->children.size());
|
||||||
@ -579,7 +579,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co
|
|||||||
column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts});
|
column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts});
|
||||||
}
|
}
|
||||||
|
|
||||||
auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/, context);
|
auto column_transformers = buildColumnTransformers(qualified_columns_list_matcher->transformers, context);
|
||||||
result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), std::move(column_list_identifiers), std::move(column_transformers));
|
result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), std::move(column_list_identifiers), std::move(column_transformers));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@ -833,15 +833,15 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index, const ContextPtr & context) const
|
ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & matcher_expression, const ContextPtr & context) const
|
||||||
{
|
{
|
||||||
ColumnTransformersNodes column_transformers;
|
ColumnTransformersNodes column_transformers;
|
||||||
size_t children_size = matcher_expression->children.size();
|
|
||||||
|
|
||||||
for (; start_child_index < children_size; ++start_child_index)
|
if (!matcher_expression)
|
||||||
|
return column_transformers;
|
||||||
|
|
||||||
|
for (const auto & child : matcher_expression->children)
|
||||||
{
|
{
|
||||||
const auto & child = matcher_expression->children[start_child_index];
|
|
||||||
|
|
||||||
if (auto * apply_transformer = child->as<ASTColumnsApplyTransformer>())
|
if (auto * apply_transformer = child->as<ASTColumnsApplyTransformer>())
|
||||||
{
|
{
|
||||||
if (apply_transformer->lambda)
|
if (apply_transformer->lambda)
|
||||||
|
@ -449,7 +449,8 @@ The server successfully detected this situation and will download merged part fr
|
|||||||
M(OverflowBreak, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'break' and the result is incomplete.") \
|
M(OverflowBreak, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'break' and the result is incomplete.") \
|
||||||
M(OverflowThrow, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'throw' and exception was thrown.") \
|
M(OverflowThrow, "Number of times, data processing was cancelled by query complexity limitation with setting '*_overflow_mode' = 'throw' and exception was thrown.") \
|
||||||
M(OverflowAny, "Number of times approximate GROUP BY was in effect: when aggregation was performed only on top of first 'max_rows_to_group_by' unique keys and other keys were ignored due to 'group_by_overflow_mode' = 'any'.") \
|
M(OverflowAny, "Number of times approximate GROUP BY was in effect: when aggregation was performed only on top of first 'max_rows_to_group_by' unique keys and other keys were ignored due to 'group_by_overflow_mode' = 'any'.") \
|
||||||
|
\
|
||||||
|
M(ServerStartupMilliseconds, "Time elapsed from starting server to listening to sockets in milliseconds")\
|
||||||
|
|
||||||
namespace ProfileEvents
|
namespace ProfileEvents
|
||||||
{
|
{
|
||||||
|
@ -28,7 +28,6 @@ protected:
|
|||||||
|
|
||||||
bool isCompression() const override { return false; }
|
bool isCompression() const override { return false; }
|
||||||
bool isGenericCompression() const override { return false; }
|
bool isGenericCompression() const override { return false; }
|
||||||
bool isDelta() const override { return true; }
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
UInt8 delta_bytes_size;
|
UInt8 delta_bytes_size;
|
||||||
|
@ -133,7 +133,6 @@ protected:
|
|||||||
|
|
||||||
bool isCompression() const override { return true; }
|
bool isCompression() const override { return true; }
|
||||||
bool isGenericCompression() const override { return false; }
|
bool isGenericCompression() const override { return false; }
|
||||||
bool isDelta() const override { return true; }
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
UInt8 data_bytes_size;
|
UInt8 data_bytes_size;
|
||||||
|
@ -39,7 +39,6 @@ protected:
|
|||||||
|
|
||||||
bool isCompression() const override { return true; }
|
bool isCompression() const override { return true; }
|
||||||
bool isGenericCompression() const override { return false; }
|
bool isGenericCompression() const override { return false; }
|
||||||
bool isFloatingPointTimeSeries() const override { return true; }
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
static constexpr UInt32 HEADER_SIZE = 2;
|
static constexpr UInt32 HEADER_SIZE = 2;
|
||||||
|
@ -123,7 +123,6 @@ protected:
|
|||||||
|
|
||||||
bool isCompression() const override { return true; }
|
bool isCompression() const override { return true; }
|
||||||
bool isGenericCompression() const override { return false; }
|
bool isGenericCompression() const override { return false; }
|
||||||
bool isFloatingPointTimeSeries() const override { return true; }
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
UInt8 data_bytes_size;
|
UInt8 data_bytes_size;
|
||||||
@ -445,19 +444,14 @@ void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 sourc
|
|||||||
void registerCodecGorilla(CompressionCodecFactory & factory)
|
void registerCodecGorilla(CompressionCodecFactory & factory)
|
||||||
{
|
{
|
||||||
UInt8 method_code = static_cast<UInt8>(CompressionMethodByte::Gorilla);
|
UInt8 method_code = static_cast<UInt8>(CompressionMethodByte::Gorilla);
|
||||||
auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
|
factory.registerCompressionCodecWithType("Gorilla", method_code,
|
||||||
|
[&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
|
||||||
{
|
{
|
||||||
if (arguments)
|
if (arguments)
|
||||||
throw Exception("Codec Gorilla does not accept any arguments", ErrorCodes::BAD_ARGUMENTS);
|
throw Exception("Codec Gorilla does not accept any arguments", ErrorCodes::BAD_ARGUMENTS);
|
||||||
|
|
||||||
if (column_type != nullptr)
|
|
||||||
if (!WhichDataType(*column_type).isFloat())
|
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Gorilla codec is not applicable for {} because the data type is not float",
|
|
||||||
column_type->getName());
|
|
||||||
|
|
||||||
UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0;
|
UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0;
|
||||||
return std::make_shared<CompressionCodecGorilla>(data_bytes_size);
|
return std::make_shared<CompressionCodecGorilla>(data_bytes_size);
|
||||||
};
|
});
|
||||||
factory.registerCompressionCodecWithType("Gorilla", method_code, codec_builder);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -59,17 +59,15 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
|||||||
{
|
{
|
||||||
ASTPtr codecs_descriptions = std::make_shared<ASTExpressionList>();
|
ASTPtr codecs_descriptions = std::make_shared<ASTExpressionList>();
|
||||||
|
|
||||||
bool with_compressing_codec = false;
|
bool is_compression = false;
|
||||||
bool with_none_codec = false;
|
bool has_none = false;
|
||||||
std::optional<size_t> generic_compression_codec_pos;
|
std::optional<size_t> generic_compression_codec_pos;
|
||||||
std::optional<size_t> first_delta_codec_pos;
|
std::set<size_t> encryption_codecs;
|
||||||
std::optional<size_t> last_floating_point_time_series_codec_pos;
|
|
||||||
std::set<size_t> encryption_codecs_pos;
|
|
||||||
|
|
||||||
bool can_substitute_codec_arguments = true;
|
bool can_substitute_codec_arguments = true;
|
||||||
for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i)
|
for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i)
|
||||||
{
|
{
|
||||||
const ASTPtr & inner_codec_ast = func->arguments->children[i];
|
const auto & inner_codec_ast = func->arguments->children[i];
|
||||||
String codec_family_name;
|
String codec_family_name;
|
||||||
ASTPtr codec_arguments;
|
ASTPtr codec_arguments;
|
||||||
if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
|
if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
|
||||||
@ -85,7 +83,8 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
|||||||
else
|
else
|
||||||
throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
|
throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
|
||||||
|
|
||||||
/// Replace "Default" codec by configured default codec which may depend on different settings and data properties at runtime.
|
/// Default codec replaced with current default codec which may depend on different
|
||||||
|
/// settings (and properties of data) in runtime.
|
||||||
CompressionCodecPtr result_codec;
|
CompressionCodecPtr result_codec;
|
||||||
if (codec_family_name == DEFAULT_CODEC_NAME)
|
if (codec_family_name == DEFAULT_CODEC_NAME)
|
||||||
{
|
{
|
||||||
@ -137,27 +136,21 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
|||||||
codecs_descriptions->children.emplace_back(result_codec->getCodecDesc());
|
codecs_descriptions->children.emplace_back(result_codec->getCodecDesc());
|
||||||
}
|
}
|
||||||
|
|
||||||
with_compressing_codec |= result_codec->isCompression();
|
is_compression |= result_codec->isCompression();
|
||||||
with_none_codec |= result_codec->isNone();
|
has_none |= result_codec->isNone();
|
||||||
|
|
||||||
if (!generic_compression_codec_pos && result_codec->isGenericCompression())
|
if (!generic_compression_codec_pos && result_codec->isGenericCompression())
|
||||||
generic_compression_codec_pos = i;
|
generic_compression_codec_pos = i;
|
||||||
|
|
||||||
if (result_codec->isEncryption())
|
if (result_codec->isEncryption())
|
||||||
encryption_codecs_pos.insert(i);
|
encryption_codecs.insert(i);
|
||||||
|
|
||||||
if (result_codec->isDelta() && !first_delta_codec_pos.has_value())
|
|
||||||
first_delta_codec_pos = i;
|
|
||||||
|
|
||||||
if (result_codec->isFloatingPointTimeSeries())
|
|
||||||
last_floating_point_time_series_codec_pos = i;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
String codec_description = queryToString(codecs_descriptions);
|
String codec_description = queryToString(codecs_descriptions);
|
||||||
|
|
||||||
if (sanity_check)
|
if (sanity_check)
|
||||||
{
|
{
|
||||||
if (codecs_descriptions->children.size() > 1 && with_none_codec)
|
if (codecs_descriptions->children.size() > 1 && has_none)
|
||||||
throw Exception(
|
throw Exception(
|
||||||
"It does not make sense to have codec NONE along with other compression codecs: " + codec_description
|
"It does not make sense to have codec NONE along with other compression codecs: " + codec_description
|
||||||
+ ". (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).",
|
+ ". (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).",
|
||||||
@ -166,7 +159,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
|||||||
/// Allow to explicitly specify single NONE codec if user don't want any compression.
|
/// Allow to explicitly specify single NONE codec if user don't want any compression.
|
||||||
/// But applying other transformations solely without compression (e.g. Delta) does not make sense.
|
/// But applying other transformations solely without compression (e.g. Delta) does not make sense.
|
||||||
/// It's okay to apply encryption codecs solely without anything else.
|
/// It's okay to apply encryption codecs solely without anything else.
|
||||||
if (!with_compressing_codec && !with_none_codec && encryption_codecs_pos.size() != codecs_descriptions->children.size())
|
if (!is_compression && !has_none && encryption_codecs.size() != codecs_descriptions->children.size())
|
||||||
throw Exception(
|
throw Exception(
|
||||||
"Compression codec " + codec_description
|
"Compression codec " + codec_description
|
||||||
+ " does not compress anything."
|
+ " does not compress anything."
|
||||||
@ -178,8 +171,8 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
|||||||
|
|
||||||
/// It does not make sense to apply any non-encryption codecs
|
/// It does not make sense to apply any non-encryption codecs
|
||||||
/// after encryption one.
|
/// after encryption one.
|
||||||
if (!encryption_codecs_pos.empty() &&
|
if (!encryption_codecs.empty() &&
|
||||||
*encryption_codecs_pos.begin() != codecs_descriptions->children.size() - encryption_codecs_pos.size())
|
*encryption_codecs.begin() != codecs_descriptions->children.size() - encryption_codecs.size())
|
||||||
throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
|
throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
|
||||||
" because it does not make sense to apply any non-post-processing codecs after"
|
" because it does not make sense to apply any non-post-processing codecs after"
|
||||||
" post-processing ones. (Note: you can enable setting 'allow_suspicious_codecs'"
|
" post-processing ones. (Note: you can enable setting 'allow_suspicious_codecs'"
|
||||||
@ -188,18 +181,11 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
|||||||
/// It does not make sense to apply any transformations after generic compression algorithm
|
/// It does not make sense to apply any transformations after generic compression algorithm
|
||||||
/// So, generic compression can be only one and only at the end.
|
/// So, generic compression can be only one and only at the end.
|
||||||
if (generic_compression_codec_pos &&
|
if (generic_compression_codec_pos &&
|
||||||
*generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs_pos.size())
|
*generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs.size())
|
||||||
throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
|
throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
|
||||||
" because it does not make sense to apply any transformations after generic compression algorithm."
|
" because it does not make sense to apply any transformations after generic compression algorithm."
|
||||||
" (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
|
" (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
|
||||||
|
|
||||||
/// Floating point time series codecs usually have implicit delta compression (or something equivalent), so it does not make
|
|
||||||
/// sense to run delta compression manually. Another reason for blocking such combination is occasional data corruption (#45195).
|
|
||||||
if (first_delta_codec_pos.has_value() && last_floating_point_time_series_codec_pos.has_value()
|
|
||||||
&& (*first_delta_codec_pos < last_floating_point_time_series_codec_pos))
|
|
||||||
throw Exception("The combination of compression codecs " + codec_description + " is meaningless,"
|
|
||||||
" because it does not make sense to apply delta transformations before floating point time series codecs."
|
|
||||||
" (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// For columns with nested types like Tuple(UInt32, UInt64) we
|
/// For columns with nested types like Tuple(UInt32, UInt64) we
|
||||||
|
@ -113,12 +113,6 @@ public:
|
|||||||
/// If it does nothing.
|
/// If it does nothing.
|
||||||
virtual bool isNone() const { return false; }
|
virtual bool isNone() const { return false; }
|
||||||
|
|
||||||
/// If the only purpose of the codec is to delta (or double-delta) the data.
|
|
||||||
virtual bool isDelta() const { return false; }
|
|
||||||
|
|
||||||
/// If the codec is specialized for floating point time series.
|
|
||||||
virtual bool isFloatingPointTimeSeries() const { return false; }
|
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
/// This is used for fuzz testing
|
/// This is used for fuzz testing
|
||||||
friend int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size);
|
friend int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size);
|
||||||
|
@ -529,13 +529,6 @@ public:
|
|||||||
|
|
||||||
TEST_P(CodecTest, TranscodingWithDataType)
|
TEST_P(CodecTest, TranscodingWithDataType)
|
||||||
{
|
{
|
||||||
/// Gorilla can only be applied to floating point columns
|
|
||||||
bool codec_is_gorilla = std::get<0>(GetParam()).codec_statement.find("Gorilla") != std::string::npos;
|
|
||||||
WhichDataType which(std::get<1>(GetParam()).data_type.get());
|
|
||||||
bool data_is_float = which.isFloat();
|
|
||||||
if (codec_is_gorilla && !data_is_float)
|
|
||||||
GTEST_SKIP() << "Skipping Gorilla-compressed integer column";
|
|
||||||
|
|
||||||
const auto codec = makeCodec(CODEC_WITH_DATA_TYPE);
|
const auto codec = makeCodec(CODEC_WITH_DATA_TYPE);
|
||||||
testTranscoding(*codec);
|
testTranscoding(*codec);
|
||||||
}
|
}
|
||||||
@ -1211,20 +1204,68 @@ auto DDperformanceTestSequence()
|
|||||||
}
|
}
|
||||||
|
|
||||||
// prime numbers in ascending order with some random repetitions hit all the cases of Gorilla.
|
// prime numbers in ascending order with some random repetitions hit all the cases of Gorilla.
|
||||||
// auto PrimesWithMultiplierGenerator = [](int multiplier = 1)
|
auto PrimesWithMultiplierGenerator = [](int multiplier = 1)
|
||||||
// {
|
{
|
||||||
// return [multiplier](auto i)
|
return [multiplier](auto i)
|
||||||
// {
|
{
|
||||||
// static const int vals[] = {
|
static const int vals[] = {
|
||||||
// 2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43,
|
2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43,
|
||||||
// 47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103,
|
47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103,
|
||||||
// 107, 107, 109, 113, 113, 127, 127, 127
|
107, 107, 109, 113, 113, 127, 127, 127
|
||||||
// };
|
};
|
||||||
// static const size_t count = sizeof(vals)/sizeof(vals[0]);
|
static const size_t count = sizeof(vals)/sizeof(vals[0]);
|
||||||
//
|
|
||||||
// return static_cast<UInt64>(vals[i % count]) * multiplier;
|
return static_cast<UInt64>(vals[i % count]) * multiplier;
|
||||||
// };
|
};
|
||||||
// };
|
};
|
||||||
|
|
||||||
|
template <typename ValueType>
|
||||||
|
auto GCompatibilityTestSequence()
|
||||||
|
{
|
||||||
|
// Also multiply result by some factor to test large values on types that can hold those.
|
||||||
|
return generateSeq<ValueType>(G(PrimesWithMultiplierGenerator(intExp10(sizeof(ValueType)))), 0, 42);
|
||||||
|
}
|
||||||
|
|
||||||
|
INSTANTIATE_TEST_SUITE_P(Gorilla,
|
||||||
|
CodecTestCompatibility,
|
||||||
|
::testing::Combine(
|
||||||
|
::testing::Values(Codec("Gorilla")),
|
||||||
|
::testing::ValuesIn(std::initializer_list<std::tuple<CodecTestSequence, std::string>>{
|
||||||
|
{
|
||||||
|
GCompatibilityTestSequence<Int8>(),
|
||||||
|
BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a")
|
||||||
|
},
|
||||||
|
{
|
||||||
|
GCompatibilityTestSequence<UInt8>(),
|
||||||
|
BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a")
|
||||||
|
},
|
||||||
|
{
|
||||||
|
GCompatibilityTestSequence<Int16>(),
|
||||||
|
BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4")
|
||||||
|
},
|
||||||
|
{
|
||||||
|
GCompatibilityTestSequence<UInt16>(),
|
||||||
|
BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4")
|
||||||
|
},
|
||||||
|
{
|
||||||
|
GCompatibilityTestSequence<Int32>(),
|
||||||
|
BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4")
|
||||||
|
},
|
||||||
|
{
|
||||||
|
GCompatibilityTestSequence<UInt32>(),
|
||||||
|
BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4")
|
||||||
|
},
|
||||||
|
{
|
||||||
|
GCompatibilityTestSequence<Int64>(),
|
||||||
|
BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10")
|
||||||
|
},
|
||||||
|
{
|
||||||
|
GCompatibilityTestSequence<UInt64>(),
|
||||||
|
BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10")
|
||||||
|
},
|
||||||
|
})
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
// These 'tests' try to measure performance of encoding and decoding and hence only make sense to be run locally,
|
// These 'tests' try to measure performance of encoding and decoding and hence only make sense to be run locally,
|
||||||
// also they require pretty big data to run against and generating this data slows down startup of unit test process.
|
// also they require pretty big data to run against and generating this data slows down startup of unit test process.
|
||||||
|
@ -773,6 +773,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
|
|||||||
M(Bool, input_format_json_validate_types_from_metadata, true, "For JSON/JSONCompact/JSONColumnsWithMetadata input formats this controls whether format parser should check if data types from input metadata match data types of the corresponding columns from the table", 0) \
|
M(Bool, input_format_json_validate_types_from_metadata, true, "For JSON/JSONCompact/JSONColumnsWithMetadata input formats this controls whether format parser should check if data types from input metadata match data types of the corresponding columns from the table", 0) \
|
||||||
M(Bool, input_format_json_read_numbers_as_strings, false, "Allow to parse numbers as strings in JSON input formats", 0) \
|
M(Bool, input_format_json_read_numbers_as_strings, false, "Allow to parse numbers as strings in JSON input formats", 0) \
|
||||||
M(Bool, input_format_json_read_objects_as_strings, true, "Allow to parse JSON objects as strings in JSON input formats", 0) \
|
M(Bool, input_format_json_read_objects_as_strings, true, "Allow to parse JSON objects as strings in JSON input formats", 0) \
|
||||||
|
M(Bool, input_format_json_named_tuples_as_objects, true, "Deserialize named tuple columns as JSON objects", 0) \
|
||||||
|
M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \
|
||||||
M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
|
M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
|
||||||
M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
|
M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
|
||||||
M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
|
M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
|
||||||
|
@ -80,7 +80,8 @@ namespace SettingsChangesHistory
|
|||||||
/// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972)
|
/// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972)
|
||||||
static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> settings_changes_history =
|
static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> settings_changes_history =
|
||||||
{
|
{
|
||||||
{"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}}},
|
{"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
|
||||||
|
{"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}}},
|
||||||
{"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
|
{"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
|
||||||
{"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
|
{"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
|
||||||
{"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
|
{"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
|
||||||
|
@ -16,6 +16,7 @@ namespace ErrorCodes
|
|||||||
{
|
{
|
||||||
extern const int SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH;
|
extern const int SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH;
|
||||||
extern const int NOT_FOUND_COLUMN_IN_BLOCK;
|
extern const int NOT_FOUND_COLUMN_IN_BLOCK;
|
||||||
|
extern const int INCORRECT_DATA;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -154,7 +155,7 @@ void SerializationTuple::deserializeText(IColumn & column, ReadBuffer & istr, co
|
|||||||
|
|
||||||
void SerializationTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
void SerializationTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||||
{
|
{
|
||||||
if (settings.json.named_tuples_as_objects
|
if (settings.json.write_named_tuples_as_objects
|
||||||
&& have_explicit_names)
|
&& have_explicit_names)
|
||||||
{
|
{
|
||||||
writeChar('{', ostr);
|
writeChar('{', ostr);
|
||||||
@ -185,7 +186,7 @@ void SerializationTuple::serializeTextJSON(const IColumn & column, size_t row_nu
|
|||||||
|
|
||||||
void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||||
{
|
{
|
||||||
if (settings.json.named_tuples_as_objects
|
if (settings.json.read_named_tuples_as_objects
|
||||||
&& have_explicit_names)
|
&& have_explicit_names)
|
||||||
{
|
{
|
||||||
skipWhitespaceIfAny(istr);
|
skipWhitespaceIfAny(istr);
|
||||||
@ -194,12 +195,15 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr
|
|||||||
|
|
||||||
addElementSafe(elems.size(), column, [&]
|
addElementSafe(elems.size(), column, [&]
|
||||||
{
|
{
|
||||||
// Require all elements but in arbitrary order.
|
std::vector<UInt8> seen_elements(elems.size(), 0);
|
||||||
for (size_t i = 0; i < elems.size(); ++i)
|
size_t i = 0;
|
||||||
|
while (!istr.eof() && *istr.position() != '}')
|
||||||
{
|
{
|
||||||
|
if (i == elems.size())
|
||||||
|
throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected number of elements in named tuple. Expected no more than {}", elems.size());
|
||||||
|
|
||||||
if (i > 0)
|
if (i > 0)
|
||||||
{
|
{
|
||||||
skipWhitespaceIfAny(istr);
|
|
||||||
assertChar(',', istr);
|
assertChar(',', istr);
|
||||||
skipWhitespaceIfAny(istr);
|
skipWhitespaceIfAny(istr);
|
||||||
}
|
}
|
||||||
@ -211,12 +215,35 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr
|
|||||||
skipWhitespaceIfAny(istr);
|
skipWhitespaceIfAny(istr);
|
||||||
|
|
||||||
const size_t element_pos = getPositionByName(name);
|
const size_t element_pos = getPositionByName(name);
|
||||||
|
seen_elements[element_pos] = 1;
|
||||||
auto & element_column = extractElementColumn(column, element_pos);
|
auto & element_column = extractElementColumn(column, element_pos);
|
||||||
elems[element_pos]->deserializeTextJSON(element_column, istr, settings);
|
elems[element_pos]->deserializeTextJSON(element_column, istr, settings);
|
||||||
|
|
||||||
|
skipWhitespaceIfAny(istr);
|
||||||
|
++i;
|
||||||
}
|
}
|
||||||
|
|
||||||
skipWhitespaceIfAny(istr);
|
|
||||||
assertChar('}', istr);
|
assertChar('}', istr);
|
||||||
|
|
||||||
|
/// Check if we have missing elements.
|
||||||
|
if (i != elems.size())
|
||||||
|
{
|
||||||
|
for (size_t element_pos = 0; element_pos != seen_elements.size(); ++element_pos)
|
||||||
|
{
|
||||||
|
if (seen_elements[element_pos])
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (!settings.json.defaults_for_missing_elements_in_named_tuple)
|
||||||
|
throw Exception(
|
||||||
|
ErrorCodes::INCORRECT_DATA,
|
||||||
|
"JSON object doesn't contain tuple element {}. If you want to insert defaults in case of missing elements, "
|
||||||
|
"enable setting input_format_json_defaults_for_missing_elements_in_named_tuple",
|
||||||
|
elems[element_pos]->getElementName());
|
||||||
|
|
||||||
|
auto & element_column = extractElementColumn(column, element_pos);
|
||||||
|
element_column.insertDefault();
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -2,6 +2,7 @@
|
|||||||
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
|
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
|
||||||
#include <Interpreters/Cluster.h>
|
#include <Interpreters/Cluster.h>
|
||||||
#include <Interpreters/Context.h>
|
#include <Interpreters/Context.h>
|
||||||
|
#include <Interpreters/misc.h>
|
||||||
#include <Interpreters/InDepthNodeVisitor.h>
|
#include <Interpreters/InDepthNodeVisitor.h>
|
||||||
#include <Interpreters/evaluateConstantExpression.h>
|
#include <Interpreters/evaluateConstantExpression.h>
|
||||||
#include <Interpreters/getClusterName.h>
|
#include <Interpreters/getClusterName.h>
|
||||||
@ -175,7 +176,7 @@ namespace
|
|||||||
/// Finds dependencies of a function.
|
/// Finds dependencies of a function.
|
||||||
void visitFunction(const ASTFunction & function)
|
void visitFunction(const ASTFunction & function)
|
||||||
{
|
{
|
||||||
if (function.name == "joinGet" || function.name == "dictHas" || function.name == "dictIsIn" || function.name.starts_with("dictGet"))
|
if (functionIsJoinGet(function.name) || functionIsDictGet(function.name))
|
||||||
{
|
{
|
||||||
/// dictGet('dict_name', attr_names, id_expr)
|
/// dictGet('dict_name', attr_names, id_expr)
|
||||||
/// dictHas('dict_name', id_expr)
|
/// dictHas('dict_name', id_expr)
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
#include <Databases/DDLLoadingDependencyVisitor.h>
|
#include <Databases/DDLLoadingDependencyVisitor.h>
|
||||||
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
|
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
|
||||||
#include <Interpreters/Context.h>
|
#include <Interpreters/Context.h>
|
||||||
|
#include <Interpreters/misc.h>
|
||||||
#include <Parsers/ASTCreateQuery.h>
|
#include <Parsers/ASTCreateQuery.h>
|
||||||
#include <Parsers/ASTFunction.h>
|
#include <Parsers/ASTFunction.h>
|
||||||
#include <Parsers/ASTIdentifier.h>
|
#include <Parsers/ASTIdentifier.h>
|
||||||
@ -52,23 +53,41 @@ bool DDLMatcherBase::needChildVisit(const ASTPtr & node, const ASTPtr & child)
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
ssize_t DDLMatcherBase::getPositionOfTableNameArgument(const ASTFunction & function)
|
ssize_t DDLMatcherBase::getPositionOfTableNameArgumentToEvaluate(const ASTFunction & function)
|
||||||
{
|
{
|
||||||
if (function.name == "joinGet" ||
|
if (functionIsJoinGet(function.name) || functionIsDictGet(function.name))
|
||||||
function.name == "dictHas" ||
|
|
||||||
function.name == "dictIsIn" ||
|
|
||||||
function.name.starts_with("dictGet"))
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
if (Poco::toLower(function.name) == "in")
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
ssize_t DDLMatcherBase::getPositionOfTableNameArgumentToVisit(const ASTFunction & function)
|
||||||
|
{
|
||||||
|
ssize_t maybe_res = getPositionOfTableNameArgumentToEvaluate(function);
|
||||||
|
if (0 <= maybe_res)
|
||||||
|
return maybe_res;
|
||||||
|
|
||||||
|
if (functionIsInOrGlobalInOperator(function.name))
|
||||||
|
{
|
||||||
|
if (function.children.empty())
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
const auto * args = function.children[0]->as<ASTExpressionList>();
|
||||||
|
if (!args || args->children.size() != 2)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
if (args->children[1]->as<ASTFunction>())
|
||||||
|
return -1;
|
||||||
|
|
||||||
return 1;
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
void DDLLoadingDependencyVisitor::visit(const ASTFunction & function, Data & data)
|
void DDLLoadingDependencyVisitor::visit(const ASTFunction & function, Data & data)
|
||||||
{
|
{
|
||||||
ssize_t table_name_arg_idx = getPositionOfTableNameArgument(function);
|
ssize_t table_name_arg_idx = getPositionOfTableNameArgumentToVisit(function);
|
||||||
if (table_name_arg_idx < 0)
|
if (table_name_arg_idx < 0)
|
||||||
return;
|
return;
|
||||||
extractTableNameFromArgument(function, data, table_name_arg_idx);
|
extractTableNameFromArgument(function, data, table_name_arg_idx);
|
||||||
|
@ -23,7 +23,8 @@ class DDLMatcherBase
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
static bool needChildVisit(const ASTPtr & node, const ASTPtr & child);
|
static bool needChildVisit(const ASTPtr & node, const ASTPtr & child);
|
||||||
static ssize_t getPositionOfTableNameArgument(const ASTFunction & function);
|
static ssize_t getPositionOfTableNameArgumentToVisit(const ASTFunction & function);
|
||||||
|
static ssize_t getPositionOfTableNameArgumentToEvaluate(const ASTFunction & function);
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Visits ASTCreateQuery and extracts the names of all tables which should be loaded before a specified table.
|
/// Visits ASTCreateQuery and extracts the names of all tables which should be loaded before a specified table.
|
||||||
|
@ -23,7 +23,7 @@ void NormalizeAndEvaluateConstants::visit(const ASTFunction & function, Data & d
|
|||||||
{
|
{
|
||||||
/// Replace expressions like "dictGet(currentDatabase() || '.dict', 'value', toUInt32(1))"
|
/// Replace expressions like "dictGet(currentDatabase() || '.dict', 'value', toUInt32(1))"
|
||||||
/// with "dictGet('db_name.dict', 'value', toUInt32(1))"
|
/// with "dictGet('db_name.dict', 'value', toUInt32(1))"
|
||||||
ssize_t table_name_arg_idx = getPositionOfTableNameArgument(function);
|
ssize_t table_name_arg_idx = getPositionOfTableNameArgumentToEvaluate(function);
|
||||||
if (table_name_arg_idx < 0)
|
if (table_name_arg_idx < 0)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
@ -90,7 +90,9 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
|
|||||||
format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio;
|
format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio;
|
||||||
format_settings.json.array_of_rows = settings.output_format_json_array_of_rows;
|
format_settings.json.array_of_rows = settings.output_format_json_array_of_rows;
|
||||||
format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes;
|
format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes;
|
||||||
format_settings.json.named_tuples_as_objects = settings.output_format_json_named_tuples_as_objects;
|
format_settings.json.write_named_tuples_as_objects = settings.output_format_json_named_tuples_as_objects;
|
||||||
|
format_settings.json.read_named_tuples_as_objects = settings.input_format_json_named_tuples_as_objects;
|
||||||
|
format_settings.json.defaults_for_missing_elements_in_named_tuple = settings.input_format_json_defaults_for_missing_elements_in_named_tuple;
|
||||||
format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
|
format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
|
||||||
format_settings.json.quote_64bit_floats = settings.output_format_json_quote_64bit_floats;
|
format_settings.json.quote_64bit_floats = settings.output_format_json_quote_64bit_floats;
|
||||||
format_settings.json.quote_denormals = settings.output_format_json_quote_denormals;
|
format_settings.json.quote_denormals = settings.output_format_json_quote_denormals;
|
||||||
|
@ -153,7 +153,9 @@ struct FormatSettings
|
|||||||
bool quote_denormals = true;
|
bool quote_denormals = true;
|
||||||
bool quote_decimals = false;
|
bool quote_decimals = false;
|
||||||
bool escape_forward_slashes = true;
|
bool escape_forward_slashes = true;
|
||||||
bool named_tuples_as_objects = false;
|
bool read_named_tuples_as_objects = false;
|
||||||
|
bool write_named_tuples_as_objects = false;
|
||||||
|
bool defaults_for_missing_elements_in_named_tuple = false;
|
||||||
bool serialize_as_strings = false;
|
bool serialize_as_strings = false;
|
||||||
bool read_bools_as_numbers = true;
|
bool read_bools_as_numbers = true;
|
||||||
bool read_numbers_as_strings = true;
|
bool read_numbers_as_strings = true;
|
||||||
|
@ -28,13 +28,29 @@ DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTTableIdentifier &
|
|||||||
database = current_database;
|
database = current_database;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTIdentifier & identifier, const String & current_database)
|
||||||
|
{
|
||||||
|
alias = identifier.tryGetAlias();
|
||||||
|
|
||||||
|
if (identifier.name_parts.size() == 2)
|
||||||
|
std::tie(database, table) = std::tie(identifier.name_parts[0], identifier.name_parts[1]);
|
||||||
|
else if (identifier.name_parts.size() == 1)
|
||||||
|
table = identifier.name_parts[0];
|
||||||
|
else
|
||||||
|
throw Exception("Logical error: invalid identifier", ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
|
if (database.empty())
|
||||||
|
database = current_database;
|
||||||
|
}
|
||||||
|
|
||||||
DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTPtr & node, const String & current_database)
|
DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTPtr & node, const String & current_database)
|
||||||
{
|
{
|
||||||
const auto * identifier = node->as<ASTTableIdentifier>();
|
if (const auto * table_identifier = node->as<ASTTableIdentifier>())
|
||||||
if (!identifier)
|
*this = DatabaseAndTableWithAlias(*table_identifier, current_database);
|
||||||
throw Exception("Logical error: table identifier expected", ErrorCodes::LOGICAL_ERROR);
|
else if (const auto * identifier = node->as<ASTIdentifier>())
|
||||||
|
*this = DatabaseAndTableWithAlias(*identifier, current_database);
|
||||||
*this = DatabaseAndTableWithAlias(*identifier, current_database);
|
else
|
||||||
|
throw Exception("Logical error: identifier or table identifier expected", ErrorCodes::LOGICAL_ERROR);
|
||||||
}
|
}
|
||||||
|
|
||||||
DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTTableExpression & table_expression, const String & current_database)
|
DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTTableExpression & table_expression, const String & current_database)
|
||||||
|
@ -14,6 +14,7 @@ namespace DB
|
|||||||
{
|
{
|
||||||
|
|
||||||
class ASTSelectQuery;
|
class ASTSelectQuery;
|
||||||
|
class ASTIdentifier;
|
||||||
class ASTTableIdentifier;
|
class ASTTableIdentifier;
|
||||||
struct ASTTableExpression;
|
struct ASTTableExpression;
|
||||||
|
|
||||||
@ -28,6 +29,7 @@ struct DatabaseAndTableWithAlias
|
|||||||
|
|
||||||
DatabaseAndTableWithAlias() = default;
|
DatabaseAndTableWithAlias() = default;
|
||||||
explicit DatabaseAndTableWithAlias(const ASTPtr & identifier_node, const String & current_database = "");
|
explicit DatabaseAndTableWithAlias(const ASTPtr & identifier_node, const String & current_database = "");
|
||||||
|
explicit DatabaseAndTableWithAlias(const ASTIdentifier & identifier, const String & current_database = "");
|
||||||
explicit DatabaseAndTableWithAlias(const ASTTableIdentifier & identifier, const String & current_database = "");
|
explicit DatabaseAndTableWithAlias(const ASTTableIdentifier & identifier, const String & current_database = "");
|
||||||
explicit DatabaseAndTableWithAlias(const ASTTableExpression & table_expression, const String & current_database = "");
|
explicit DatabaseAndTableWithAlias(const ASTTableExpression & table_expression, const String & current_database = "");
|
||||||
|
|
||||||
|
@ -25,6 +25,7 @@ static const std::unordered_map<String, String> quantile_fuse_name_mapping = {
|
|||||||
{NameQuantileExactInclusive::name, NameQuantilesExactInclusive::name},
|
{NameQuantileExactInclusive::name, NameQuantilesExactInclusive::name},
|
||||||
{NameQuantileExactLow::name, NameQuantilesExactLow::name},
|
{NameQuantileExactLow::name, NameQuantilesExactLow::name},
|
||||||
{NameQuantileExactWeighted::name, NameQuantilesExactWeighted::name},
|
{NameQuantileExactWeighted::name, NameQuantilesExactWeighted::name},
|
||||||
|
{NameQuantileInterpolatedWeighted::name, NameQuantilesInterpolatedWeighted::name},
|
||||||
{NameQuantileTDigest::name, NameQuantilesTDigest::name},
|
{NameQuantileTDigest::name, NameQuantilesTDigest::name},
|
||||||
{NameQuantileTDigestWeighted::name, NameQuantilesTDigestWeighted::name},
|
{NameQuantileTDigestWeighted::name, NameQuantilesTDigestWeighted::name},
|
||||||
{NameQuantileTiming::name, NameQuantilesTiming::name},
|
{NameQuantileTiming::name, NameQuantilesTiming::name},
|
||||||
@ -61,9 +62,11 @@ void GatherFunctionQuantileData::FuseQuantileAggregatesData::addFuncNode(ASTPtr
|
|||||||
|
|
||||||
const auto & arguments = func->arguments->children;
|
const auto & arguments = func->arguments->children;
|
||||||
|
|
||||||
|
|
||||||
bool need_two_args = func->name == NameQuantileDeterministic::name || func->name == NameQuantileExactWeighted::name
|
bool need_two_args = func->name == NameQuantileDeterministic::name || func->name == NameQuantileExactWeighted::name
|
||||||
|| func->name == NameQuantileTimingWeighted::name || func->name == NameQuantileTDigestWeighted::name
|
|| func->name == NameQuantileInterpolatedWeighted::name || func->name == NameQuantileTimingWeighted::name
|
||||||
|| func->name == NameQuantileBFloat16Weighted::name;
|
|| func->name == NameQuantileTDigestWeighted::name || func->name == NameQuantileBFloat16Weighted::name;
|
||||||
|
|
||||||
if (arguments.size() != (need_two_args ? 2 : 1))
|
if (arguments.size() != (need_two_args ? 2 : 1))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
@ -487,7 +487,7 @@ BlockIO InterpreterSystemQuery::execute()
|
|||||||
dropDatabaseReplica(query);
|
dropDatabaseReplica(query);
|
||||||
break;
|
break;
|
||||||
case Type::SYNC_REPLICA:
|
case Type::SYNC_REPLICA:
|
||||||
syncReplica(query);
|
syncReplica();
|
||||||
break;
|
break;
|
||||||
case Type::SYNC_DATABASE_REPLICA:
|
case Type::SYNC_DATABASE_REPLICA:
|
||||||
syncReplicatedDatabase(query);
|
syncReplicatedDatabase(query);
|
||||||
@ -507,6 +507,9 @@ BlockIO InterpreterSystemQuery::execute()
|
|||||||
case Type::RESTORE_REPLICA:
|
case Type::RESTORE_REPLICA:
|
||||||
restoreReplica();
|
restoreReplica();
|
||||||
break;
|
break;
|
||||||
|
case Type::WAIT_LOADING_PARTS:
|
||||||
|
waitLoadingParts();
|
||||||
|
break;
|
||||||
case Type::RESTART_DISK:
|
case Type::RESTART_DISK:
|
||||||
restartDisk(query.disk);
|
restartDisk(query.disk);
|
||||||
case Type::FLUSH_LOGS:
|
case Type::FLUSH_LOGS:
|
||||||
@ -852,7 +855,7 @@ void InterpreterSystemQuery::dropDatabaseReplica(ASTSystemQuery & query)
|
|||||||
throw Exception("Invalid query", ErrorCodes::LOGICAL_ERROR);
|
throw Exception("Invalid query", ErrorCodes::LOGICAL_ERROR);
|
||||||
}
|
}
|
||||||
|
|
||||||
void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
|
void InterpreterSystemQuery::syncReplica()
|
||||||
{
|
{
|
||||||
getContext()->checkAccess(AccessType::SYSTEM_SYNC_REPLICA, table_id);
|
getContext()->checkAccess(AccessType::SYSTEM_SYNC_REPLICA, table_id);
|
||||||
StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext());
|
StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext());
|
||||||
@ -872,6 +875,23 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
|
|||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs());
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void InterpreterSystemQuery::waitLoadingParts()
|
||||||
|
{
|
||||||
|
getContext()->checkAccess(AccessType::SYSTEM_WAIT_LOADING_PARTS, table_id);
|
||||||
|
StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext());
|
||||||
|
|
||||||
|
if (auto * merge_tree = dynamic_cast<MergeTreeData *>(table.get()))
|
||||||
|
{
|
||||||
|
LOG_TRACE(log, "Waiting for loading of parts of table {}", table_id.getFullTableName());
|
||||||
|
merge_tree->waitForOutdatedPartsToBeLoaded();
|
||||||
|
LOG_TRACE(log, "Finished waiting for loading of parts of table {}", table_id.getFullTableName());
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||||
|
"Command WAIT LOADING PARTS is supported only for MergeTree table, but got: {}", table->getName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query)
|
void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query)
|
||||||
{
|
{
|
||||||
@ -1071,6 +1091,11 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
|
|||||||
required_access.emplace_back(AccessType::SYSTEM_RESTART_REPLICA);
|
required_access.emplace_back(AccessType::SYSTEM_RESTART_REPLICA);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
case Type::WAIT_LOADING_PARTS:
|
||||||
|
{
|
||||||
|
required_access.emplace_back(AccessType::SYSTEM_WAIT_LOADING_PARTS, query.getDatabase(), query.getTable());
|
||||||
|
break;
|
||||||
|
}
|
||||||
case Type::SYNC_DATABASE_REPLICA:
|
case Type::SYNC_DATABASE_REPLICA:
|
||||||
{
|
{
|
||||||
required_access.emplace_back(AccessType::SYSTEM_SYNC_DATABASE_REPLICA, query.getDatabase());
|
required_access.emplace_back(AccessType::SYSTEM_SYNC_DATABASE_REPLICA, query.getDatabase());
|
||||||
|
@ -56,7 +56,8 @@ private:
|
|||||||
|
|
||||||
void restartReplica(const StorageID & replica, ContextMutablePtr system_context);
|
void restartReplica(const StorageID & replica, ContextMutablePtr system_context);
|
||||||
void restartReplicas(ContextMutablePtr system_context);
|
void restartReplicas(ContextMutablePtr system_context);
|
||||||
void syncReplica(ASTSystemQuery & query);
|
void syncReplica();
|
||||||
|
void waitLoadingParts();
|
||||||
|
|
||||||
void syncReplicatedDatabase(ASTSystemQuery & query);
|
void syncReplicatedDatabase(ASTSystemQuery & query);
|
||||||
|
|
||||||
|
@ -49,7 +49,8 @@ ASTPtr makeSubqueryTemplate()
|
|||||||
ASTPtr makeSubqueryQualifiedAsterisk()
|
ASTPtr makeSubqueryQualifiedAsterisk()
|
||||||
{
|
{
|
||||||
auto asterisk = std::make_shared<ASTQualifiedAsterisk>();
|
auto asterisk = std::make_shared<ASTQualifiedAsterisk>();
|
||||||
asterisk->children.emplace_back(std::make_shared<ASTTableIdentifier>("--.s"));
|
asterisk->qualifier = std::make_shared<ASTIdentifier>("--.s");
|
||||||
|
asterisk->children.push_back(asterisk->qualifier);
|
||||||
return asterisk;
|
return asterisk;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -153,24 +154,34 @@ private:
|
|||||||
for (auto & table_name : data.tables_order)
|
for (auto & table_name : data.tables_order)
|
||||||
data.addTableColumns(table_name, columns);
|
data.addTableColumns(table_name, columns);
|
||||||
|
|
||||||
for (const auto & transformer : asterisk->children)
|
if (asterisk->transformers)
|
||||||
IASTColumnsTransformer::transform(transformer, columns);
|
{
|
||||||
|
for (const auto & transformer : asterisk->transformers->children)
|
||||||
|
IASTColumnsTransformer::transform(transformer, columns);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else if (const auto * qualified_asterisk = child->as<ASTQualifiedAsterisk>())
|
else if (const auto * qualified_asterisk = child->as<ASTQualifiedAsterisk>())
|
||||||
{
|
{
|
||||||
has_asterisks = true;
|
has_asterisks = true;
|
||||||
|
|
||||||
auto & identifier = child->children[0]->as<ASTTableIdentifier &>();
|
if (!qualified_asterisk->qualifier)
|
||||||
|
throw Exception("Logical error: qualified asterisk must have a qualifier", ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
|
auto & identifier = qualified_asterisk->qualifier->as<ASTIdentifier &>();
|
||||||
|
|
||||||
data.addTableColumns(identifier.name(), columns);
|
data.addTableColumns(identifier.name(), columns);
|
||||||
|
|
||||||
// QualifiedAsterisk's transformers start to appear at child 1
|
if (qualified_asterisk->transformers)
|
||||||
for (const auto * it = qualified_asterisk->children.begin() + 1; it != qualified_asterisk->children.end(); ++it)
|
|
||||||
{
|
{
|
||||||
if (it->get()->as<ASTColumnsApplyTransformer>() || it->get()->as<ASTColumnsExceptTransformer>() || it->get()->as<ASTColumnsReplaceTransformer>())
|
for (const auto & transformer : qualified_asterisk->transformers->children)
|
||||||
IASTColumnsTransformer::transform(*it, columns);
|
{
|
||||||
else
|
if (transformer->as<ASTColumnsApplyTransformer>() ||
|
||||||
throw Exception("Logical error: qualified asterisk must only have children of IASTColumnsTransformer type", ErrorCodes::LOGICAL_ERROR);
|
transformer->as<ASTColumnsExceptTransformer>() ||
|
||||||
|
transformer->as<ASTColumnsReplaceTransformer>())
|
||||||
|
IASTColumnsTransformer::transform(transformer, columns);
|
||||||
|
else
|
||||||
|
throw Exception("Logical error: qualified asterisk must only have children of IASTColumnsTransformer type", ErrorCodes::LOGICAL_ERROR);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if (const auto * columns_list_matcher = child->as<ASTColumnsListMatcher>())
|
else if (const auto * columns_list_matcher = child->as<ASTColumnsListMatcher>())
|
||||||
@ -180,8 +191,11 @@ private:
|
|||||||
for (const auto & ident : columns_list_matcher->column_list->children)
|
for (const auto & ident : columns_list_matcher->column_list->children)
|
||||||
columns.emplace_back(ident->clone());
|
columns.emplace_back(ident->clone());
|
||||||
|
|
||||||
for (const auto & transformer : columns_list_matcher->children)
|
if (columns_list_matcher->transformers)
|
||||||
IASTColumnsTransformer::transform(transformer, columns);
|
{
|
||||||
|
for (const auto & transformer : columns_list_matcher->transformers->children)
|
||||||
|
IASTColumnsTransformer::transform(transformer, columns);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else if (const auto * columns_regexp_matcher = child->as<ASTColumnsRegexpMatcher>())
|
else if (const auto * columns_regexp_matcher = child->as<ASTColumnsRegexpMatcher>())
|
||||||
{
|
{
|
||||||
@ -193,8 +207,11 @@ private:
|
|||||||
columns,
|
columns,
|
||||||
[&](const String & column_name) { return columns_regexp_matcher->isColumnMatching(column_name); });
|
[&](const String & column_name) { return columns_regexp_matcher->isColumnMatching(column_name); });
|
||||||
|
|
||||||
for (const auto & transformer : columns_regexp_matcher->children)
|
if (columns_regexp_matcher->transformers)
|
||||||
IASTColumnsTransformer::transform(transformer, columns);
|
{
|
||||||
|
for (const auto & transformer : columns_regexp_matcher->transformers->children)
|
||||||
|
IASTColumnsTransformer::transform(transformer, columns);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
data.new_select_expression_list->children.push_back(child);
|
data.new_select_expression_list->children.push_back(child);
|
||||||
@ -425,6 +442,7 @@ private:
|
|||||||
{
|
{
|
||||||
if (data.expression_list->children.empty())
|
if (data.expression_list->children.empty())
|
||||||
data.expression_list->children.emplace_back(std::make_shared<ASTAsterisk>());
|
data.expression_list->children.emplace_back(std::make_shared<ASTAsterisk>());
|
||||||
|
|
||||||
select.setExpression(ASTSelectQuery::Expression::SELECT, std::move(data.expression_list));
|
select.setExpression(ASTSelectQuery::Expression::SELECT, std::move(data.expression_list));
|
||||||
}
|
}
|
||||||
data.done = true;
|
data.done = true;
|
||||||
|
@ -154,7 +154,7 @@ private:
|
|||||||
|
|
||||||
static void visit(const ASTQualifiedAsterisk & node, const ASTPtr &, Data & data)
|
static void visit(const ASTQualifiedAsterisk & node, const ASTPtr &, Data & data)
|
||||||
{
|
{
|
||||||
auto & identifier = node.children[0]->as<ASTTableIdentifier &>();
|
auto & identifier = node.qualifier->as<ASTIdentifier &>();
|
||||||
bool rewritten = false;
|
bool rewritten = false;
|
||||||
for (const auto & table : data)
|
for (const auto & table : data)
|
||||||
{
|
{
|
||||||
|
@ -303,7 +303,6 @@ bool MergeTreeTransaction::rollback() noexcept
|
|||||||
part->version.unlockRemovalTID(tid, TransactionInfoContext{part->storage.getStorageID(), part->name});
|
part->version.unlockRemovalTID(tid, TransactionInfoContext{part->storage.getStorageID(), part->name});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
assert([&]()
|
assert([&]()
|
||||||
{
|
{
|
||||||
std::lock_guard lock{mutex};
|
std::lock_guard lock{mutex};
|
||||||
|
@ -156,21 +156,19 @@ void TranslateQualifiedNamesMatcher::visit(ASTFunction & node, const ASTPtr &, D
|
|||||||
func_arguments->children.clear();
|
func_arguments->children.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
void TranslateQualifiedNamesMatcher::visit(const ASTQualifiedAsterisk &, const ASTPtr & ast, Data & data)
|
void TranslateQualifiedNamesMatcher::visit(const ASTQualifiedAsterisk & node, const ASTPtr &, Data & data)
|
||||||
{
|
{
|
||||||
if (ast->children.empty())
|
if (!node.qualifier)
|
||||||
throw Exception("Logical error: qualified asterisk must have children", ErrorCodes::LOGICAL_ERROR);
|
throw Exception("Logical error: qualified asterisk must have a qualifier", ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
auto & ident = ast->children[0];
|
|
||||||
|
|
||||||
/// @note it could contain table alias as table name.
|
/// @note it could contain table alias as table name.
|
||||||
DatabaseAndTableWithAlias db_and_table(ident);
|
DatabaseAndTableWithAlias db_and_table(node.qualifier);
|
||||||
|
|
||||||
for (const auto & known_table : data.tables)
|
for (const auto & known_table : data.tables)
|
||||||
if (db_and_table.satisfies(known_table.table, true))
|
if (db_and_table.satisfies(known_table.table, true))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
throw Exception("Unknown qualified identifier: " + ident->getAliasOrColumnName(), ErrorCodes::UNKNOWN_IDENTIFIER);
|
throw Exception("Unknown qualified identifier: " + node.qualifier->getAliasOrColumnName(), ErrorCodes::UNKNOWN_IDENTIFIER);
|
||||||
}
|
}
|
||||||
|
|
||||||
void TranslateQualifiedNamesMatcher::visit(ASTTableJoin & join, const ASTPtr & , Data & data)
|
void TranslateQualifiedNamesMatcher::visit(ASTTableJoin & join, const ASTPtr & , Data & data)
|
||||||
@ -266,16 +264,22 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
|
|||||||
first_table = false;
|
first_table = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const auto & transformer : asterisk->children)
|
if (asterisk->transformers)
|
||||||
IASTColumnsTransformer::transform(transformer, columns);
|
{
|
||||||
|
for (const auto & transformer : asterisk->transformers->children)
|
||||||
|
IASTColumnsTransformer::transform(transformer, columns);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else if (auto * asterisk_column_list = child->as<ASTColumnsListMatcher>())
|
else if (auto * asterisk_column_list = child->as<ASTColumnsListMatcher>())
|
||||||
{
|
{
|
||||||
for (const auto & ident : asterisk_column_list->column_list->children)
|
for (const auto & ident : asterisk_column_list->column_list->children)
|
||||||
columns.emplace_back(ident->clone());
|
columns.emplace_back(ident->clone());
|
||||||
|
|
||||||
for (const auto & transformer : asterisk_column_list->children)
|
if (asterisk_column_list->transformers)
|
||||||
IASTColumnsTransformer::transform(transformer, columns);
|
{
|
||||||
|
for (const auto & transformer : asterisk_column_list->transformers->children)
|
||||||
|
IASTColumnsTransformer::transform(transformer, columns);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else if (const auto * asterisk_regexp_pattern = child->as<ASTColumnsRegexpMatcher>())
|
else if (const auto * asterisk_regexp_pattern = child->as<ASTColumnsRegexpMatcher>())
|
||||||
{
|
{
|
||||||
@ -292,12 +296,15 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
|
|||||||
first_table = false;
|
first_table = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const auto & transformer : asterisk_regexp_pattern->children)
|
if (asterisk_regexp_pattern->transformers)
|
||||||
IASTColumnsTransformer::transform(transformer, columns);
|
{
|
||||||
|
for (const auto & transformer : asterisk_regexp_pattern->transformers->children)
|
||||||
|
IASTColumnsTransformer::transform(transformer, columns);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else if (const auto * qualified_asterisk = child->as<ASTQualifiedAsterisk>())
|
else if (const auto * qualified_asterisk = child->as<ASTQualifiedAsterisk>())
|
||||||
{
|
{
|
||||||
DatabaseAndTableWithAlias ident_db_and_name(qualified_asterisk->children[0]);
|
DatabaseAndTableWithAlias ident_db_and_name(qualified_asterisk->qualifier);
|
||||||
|
|
||||||
for (const auto & table : tables_with_columns)
|
for (const auto & table : tables_with_columns)
|
||||||
{
|
{
|
||||||
@ -309,10 +316,10 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// QualifiedAsterisk's transformers start to appear at child 1
|
if (qualified_asterisk->transformers)
|
||||||
for (const auto * it = qualified_asterisk->children.begin() + 1; it != qualified_asterisk->children.end(); ++it)
|
|
||||||
{
|
{
|
||||||
IASTColumnsTransformer::transform(*it, columns);
|
for (const auto & transformer : qualified_asterisk->transformers->children)
|
||||||
|
IASTColumnsTransformer::transform(transformer, columns);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -8,21 +8,37 @@ namespace DB
|
|||||||
ASTPtr ASTAsterisk::clone() const
|
ASTPtr ASTAsterisk::clone() const
|
||||||
{
|
{
|
||||||
auto clone = std::make_shared<ASTAsterisk>(*this);
|
auto clone = std::make_shared<ASTAsterisk>(*this);
|
||||||
clone->cloneChildren();
|
|
||||||
|
if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); }
|
||||||
|
if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
|
||||||
|
|
||||||
return clone;
|
return clone;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ASTAsterisk::appendColumnName(WriteBuffer & ostr) const { ostr.write('*'); }
|
void ASTAsterisk::appendColumnName(WriteBuffer & ostr) const
|
||||||
|
{
|
||||||
|
if (expression)
|
||||||
|
{
|
||||||
|
expression->appendColumnName(ostr);
|
||||||
|
writeCString(".", ostr);
|
||||||
|
}
|
||||||
|
|
||||||
|
ostr.write('*');
|
||||||
|
}
|
||||||
|
|
||||||
void ASTAsterisk::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
void ASTAsterisk::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||||
{
|
{
|
||||||
|
if (expression)
|
||||||
|
{
|
||||||
|
expression->formatImpl(settings, state, frame);
|
||||||
|
settings.ostr << ".";
|
||||||
|
}
|
||||||
|
|
||||||
settings.ostr << "*";
|
settings.ostr << "*";
|
||||||
|
|
||||||
/// Format column transformers
|
if (transformers)
|
||||||
for (const auto & child : children)
|
|
||||||
{
|
{
|
||||||
settings.ostr << ' ';
|
transformers->formatImpl(settings, state, frame);
|
||||||
child->formatImpl(settings, state, frame);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -16,6 +16,8 @@ public:
|
|||||||
ASTPtr clone() const override;
|
ASTPtr clone() const override;
|
||||||
void appendColumnName(WriteBuffer & ostr) const override;
|
void appendColumnName(WriteBuffer & ostr) const override;
|
||||||
|
|
||||||
|
ASTPtr expression;
|
||||||
|
ASTPtr transformers;
|
||||||
protected:
|
protected:
|
||||||
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
||||||
};
|
};
|
||||||
|
@ -18,12 +18,20 @@ namespace ErrorCodes
|
|||||||
ASTPtr ASTColumnsRegexpMatcher::clone() const
|
ASTPtr ASTColumnsRegexpMatcher::clone() const
|
||||||
{
|
{
|
||||||
auto clone = std::make_shared<ASTColumnsRegexpMatcher>(*this);
|
auto clone = std::make_shared<ASTColumnsRegexpMatcher>(*this);
|
||||||
clone->cloneChildren();
|
|
||||||
|
if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); }
|
||||||
|
if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
|
||||||
|
|
||||||
return clone;
|
return clone;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ASTColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) const
|
void ASTColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) const
|
||||||
{
|
{
|
||||||
|
if (expression)
|
||||||
|
{
|
||||||
|
expression->appendColumnName(ostr);
|
||||||
|
writeCString(".", ostr);
|
||||||
|
}
|
||||||
writeCString("COLUMNS(", ostr);
|
writeCString("COLUMNS(", ostr);
|
||||||
writeQuotedString(original_pattern, ostr);
|
writeQuotedString(original_pattern, ostr);
|
||||||
writeChar(')', ostr);
|
writeChar(')', ostr);
|
||||||
@ -38,15 +46,21 @@ void ASTColumnsRegexpMatcher::updateTreeHashImpl(SipHash & hash_state) const
|
|||||||
|
|
||||||
void ASTColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
void ASTColumnsRegexpMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||||
{
|
{
|
||||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
|
settings.ostr << (settings.hilite ? hilite_keyword : "");
|
||||||
|
|
||||||
|
if (expression)
|
||||||
|
{
|
||||||
|
expression->formatImpl(settings, state, frame);
|
||||||
|
settings.ostr << ".";
|
||||||
|
}
|
||||||
|
|
||||||
|
settings.ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
|
||||||
settings.ostr << quoteString(original_pattern);
|
settings.ostr << quoteString(original_pattern);
|
||||||
settings.ostr << ")";
|
settings.ostr << ")";
|
||||||
|
|
||||||
/// Format column transformers
|
if (transformers)
|
||||||
for (const auto & child : children)
|
|
||||||
{
|
{
|
||||||
settings.ostr << ' ';
|
transformers->formatImpl(settings, state, frame);
|
||||||
child->formatImpl(settings, state, frame);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -60,6 +74,11 @@ void ASTColumnsRegexpMatcher::setPattern(String pattern)
|
|||||||
DB::ErrorCodes::CANNOT_COMPILE_REGEXP);
|
DB::ErrorCodes::CANNOT_COMPILE_REGEXP);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const String & ASTColumnsRegexpMatcher::getPattern() const
|
||||||
|
{
|
||||||
|
return original_pattern;
|
||||||
|
}
|
||||||
|
|
||||||
const std::shared_ptr<re2::RE2> & ASTColumnsRegexpMatcher::getMatcher() const
|
const std::shared_ptr<re2::RE2> & ASTColumnsRegexpMatcher::getMatcher() const
|
||||||
{
|
{
|
||||||
return column_matcher;
|
return column_matcher;
|
||||||
@ -73,19 +92,23 @@ bool ASTColumnsRegexpMatcher::isColumnMatching(const String & column_name) const
|
|||||||
ASTPtr ASTColumnsListMatcher::clone() const
|
ASTPtr ASTColumnsListMatcher::clone() const
|
||||||
{
|
{
|
||||||
auto clone = std::make_shared<ASTColumnsListMatcher>(*this);
|
auto clone = std::make_shared<ASTColumnsListMatcher>(*this);
|
||||||
clone->column_list = column_list->clone();
|
|
||||||
clone->cloneChildren();
|
|
||||||
return clone;
|
|
||||||
}
|
|
||||||
|
|
||||||
void ASTColumnsListMatcher::updateTreeHashImpl(SipHash & hash_state) const
|
if (expression) { clone->expression = expression->clone(); clone->children.push_back(clone->expression); }
|
||||||
{
|
if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
|
||||||
column_list->updateTreeHash(hash_state);
|
|
||||||
IAST::updateTreeHashImpl(hash_state);
|
clone->column_list = column_list->clone();
|
||||||
|
clone->children.push_back(clone->column_list);
|
||||||
|
|
||||||
|
return clone;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ASTColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
|
void ASTColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
|
||||||
{
|
{
|
||||||
|
if (expression)
|
||||||
|
{
|
||||||
|
expression->appendColumnName(ostr);
|
||||||
|
writeCString(".", ostr);
|
||||||
|
}
|
||||||
writeCString("COLUMNS(", ostr);
|
writeCString("COLUMNS(", ostr);
|
||||||
for (auto * it = column_list->children.begin(); it != column_list->children.end(); ++it)
|
for (auto * it = column_list->children.begin(); it != column_list->children.end(); ++it)
|
||||||
{
|
{
|
||||||
@ -99,7 +122,15 @@ void ASTColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
|
|||||||
|
|
||||||
void ASTColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
void ASTColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||||
{
|
{
|
||||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
|
settings.ostr << (settings.hilite ? hilite_keyword : "");
|
||||||
|
|
||||||
|
if (expression)
|
||||||
|
{
|
||||||
|
expression->formatImpl(settings, state, frame);
|
||||||
|
settings.ostr << ".";
|
||||||
|
}
|
||||||
|
|
||||||
|
settings.ostr << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
|
||||||
|
|
||||||
for (ASTs::const_iterator it = column_list->children.begin(); it != column_list->children.end(); ++it)
|
for (ASTs::const_iterator it = column_list->children.begin(); it != column_list->children.end(); ++it)
|
||||||
{
|
{
|
||||||
@ -111,33 +142,39 @@ void ASTColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatSt
|
|||||||
}
|
}
|
||||||
settings.ostr << ")";
|
settings.ostr << ")";
|
||||||
|
|
||||||
/// Format column transformers
|
if (transformers)
|
||||||
for (const auto & child : children)
|
|
||||||
{
|
{
|
||||||
settings.ostr << ' ';
|
transformers->formatImpl(settings, state, frame);
|
||||||
child->formatImpl(settings, state, frame);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ASTPtr ASTQualifiedColumnsRegexpMatcher::clone() const
|
ASTPtr ASTQualifiedColumnsRegexpMatcher::clone() const
|
||||||
{
|
{
|
||||||
auto clone = std::make_shared<ASTQualifiedColumnsRegexpMatcher>(*this);
|
auto clone = std::make_shared<ASTQualifiedColumnsRegexpMatcher>(*this);
|
||||||
clone->cloneChildren();
|
|
||||||
|
if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
|
||||||
|
|
||||||
|
clone->qualifier = qualifier->clone();
|
||||||
|
clone->children.push_back(clone->qualifier);
|
||||||
|
|
||||||
return clone;
|
return clone;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ASTQualifiedColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) const
|
void ASTQualifiedColumnsRegexpMatcher::appendColumnName(WriteBuffer & ostr) const
|
||||||
{
|
{
|
||||||
const auto & qualifier = children.at(0);
|
|
||||||
qualifier->appendColumnName(ostr);
|
qualifier->appendColumnName(ostr);
|
||||||
writeCString(".COLUMNS(", ostr);
|
writeCString(".COLUMNS(", ostr);
|
||||||
writeQuotedString(original_pattern, ostr);
|
writeQuotedString(original_pattern, ostr);
|
||||||
writeChar(')', ostr);
|
writeChar(')', ostr);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ASTQualifiedColumnsRegexpMatcher::setPattern(String pattern)
|
void ASTQualifiedColumnsRegexpMatcher::setPattern(String pattern, bool set_matcher)
|
||||||
{
|
{
|
||||||
original_pattern = std::move(pattern);
|
original_pattern = std::move(pattern);
|
||||||
|
|
||||||
|
if (!set_matcher)
|
||||||
|
return;
|
||||||
|
|
||||||
column_matcher = std::make_shared<RE2>(original_pattern, RE2::Quiet);
|
column_matcher = std::make_shared<RE2>(original_pattern, RE2::Quiet);
|
||||||
if (!column_matcher->ok())
|
if (!column_matcher->ok())
|
||||||
throw DB::Exception(
|
throw DB::Exception(
|
||||||
@ -166,35 +203,35 @@ void ASTQualifiedColumnsRegexpMatcher::formatImpl(const FormatSettings & setting
|
|||||||
{
|
{
|
||||||
settings.ostr << (settings.hilite ? hilite_keyword : "");
|
settings.ostr << (settings.hilite ? hilite_keyword : "");
|
||||||
|
|
||||||
const auto & qualifier = children.at(0);
|
|
||||||
qualifier->formatImpl(settings, state, frame);
|
qualifier->formatImpl(settings, state, frame);
|
||||||
|
|
||||||
settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
|
settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
|
||||||
settings.ostr << quoteString(original_pattern);
|
settings.ostr << quoteString(original_pattern);
|
||||||
settings.ostr << ")";
|
settings.ostr << ")";
|
||||||
|
|
||||||
/// Format column transformers
|
if (transformers)
|
||||||
size_t children_size = children.size();
|
|
||||||
|
|
||||||
for (size_t i = 1; i < children_size; ++i)
|
|
||||||
{
|
{
|
||||||
const auto & child = children[i];
|
transformers->formatImpl(settings, state, frame);
|
||||||
settings.ostr << ' ';
|
|
||||||
child->formatImpl(settings, state, frame);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ASTPtr ASTQualifiedColumnsListMatcher::clone() const
|
ASTPtr ASTQualifiedColumnsListMatcher::clone() const
|
||||||
{
|
{
|
||||||
auto clone = std::make_shared<ASTQualifiedColumnsListMatcher>(*this);
|
auto clone = std::make_shared<ASTQualifiedColumnsListMatcher>(*this);
|
||||||
|
|
||||||
|
if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
|
||||||
|
|
||||||
|
clone->qualifier = qualifier->clone();
|
||||||
clone->column_list = column_list->clone();
|
clone->column_list = column_list->clone();
|
||||||
clone->cloneChildren();
|
|
||||||
|
clone->children.push_back(clone->qualifier);
|
||||||
|
clone->children.push_back(clone->column_list);
|
||||||
|
|
||||||
return clone;
|
return clone;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ASTQualifiedColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
|
void ASTQualifiedColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
|
||||||
{
|
{
|
||||||
const auto & qualifier = children.at(0);
|
|
||||||
qualifier->appendColumnName(ostr);
|
qualifier->appendColumnName(ostr);
|
||||||
writeCString(".COLUMNS(", ostr);
|
writeCString(".COLUMNS(", ostr);
|
||||||
|
|
||||||
@ -208,19 +245,10 @@ void ASTQualifiedColumnsListMatcher::appendColumnName(WriteBuffer & ostr) const
|
|||||||
writeChar(')', ostr);
|
writeChar(')', ostr);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ASTQualifiedColumnsListMatcher::updateTreeHashImpl(SipHash & hash_state) const
|
|
||||||
{
|
|
||||||
column_list->updateTreeHash(hash_state);
|
|
||||||
IAST::updateTreeHashImpl(hash_state);
|
|
||||||
}
|
|
||||||
|
|
||||||
void ASTQualifiedColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
void ASTQualifiedColumnsListMatcher::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||||
{
|
{
|
||||||
settings.ostr << (settings.hilite ? hilite_keyword : "");
|
settings.ostr << (settings.hilite ? hilite_keyword : "");
|
||||||
|
|
||||||
const auto & qualifier = children.at(0);
|
|
||||||
qualifier->formatImpl(settings, state, frame);
|
qualifier->formatImpl(settings, state, frame);
|
||||||
|
|
||||||
settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
|
settings.ostr << ".COLUMNS" << (settings.hilite ? hilite_none : "") << "(";
|
||||||
|
|
||||||
for (ASTs::const_iterator it = column_list->children.begin(); it != column_list->children.end(); ++it)
|
for (ASTs::const_iterator it = column_list->children.begin(); it != column_list->children.end(); ++it)
|
||||||
@ -232,14 +260,9 @@ void ASTQualifiedColumnsListMatcher::formatImpl(const FormatSettings & settings,
|
|||||||
}
|
}
|
||||||
settings.ostr << ")";
|
settings.ostr << ")";
|
||||||
|
|
||||||
/// Format column transformers
|
if (transformers)
|
||||||
size_t children_size = children.size();
|
|
||||||
|
|
||||||
for (size_t i = 1; i < children_size; ++i)
|
|
||||||
{
|
{
|
||||||
const auto & child = children[i];
|
transformers->formatImpl(settings, state, frame);
|
||||||
settings.ostr << ' ';
|
|
||||||
child->formatImpl(settings, state, frame);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -24,10 +24,13 @@ public:
|
|||||||
|
|
||||||
void appendColumnName(WriteBuffer & ostr) const override;
|
void appendColumnName(WriteBuffer & ostr) const override;
|
||||||
void setPattern(String pattern);
|
void setPattern(String pattern);
|
||||||
|
const String & getPattern() const;
|
||||||
const std::shared_ptr<re2::RE2> & getMatcher() const;
|
const std::shared_ptr<re2::RE2> & getMatcher() const;
|
||||||
bool isColumnMatching(const String & column_name) const;
|
bool isColumnMatching(const String & column_name) const;
|
||||||
void updateTreeHashImpl(SipHash & hash_state) const override;
|
void updateTreeHashImpl(SipHash & hash_state) const override;
|
||||||
|
|
||||||
|
ASTPtr expression;
|
||||||
|
ASTPtr transformers;
|
||||||
protected:
|
protected:
|
||||||
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
||||||
|
|
||||||
@ -43,9 +46,10 @@ public:
|
|||||||
String getID(char) const override { return "ColumnsListMatcher"; }
|
String getID(char) const override { return "ColumnsListMatcher"; }
|
||||||
ASTPtr clone() const override;
|
ASTPtr clone() const override;
|
||||||
void appendColumnName(WriteBuffer & ostr) const override;
|
void appendColumnName(WriteBuffer & ostr) const override;
|
||||||
void updateTreeHashImpl(SipHash & hash_state) const override;
|
|
||||||
|
|
||||||
|
ASTPtr expression;
|
||||||
ASTPtr column_list;
|
ASTPtr column_list;
|
||||||
|
ASTPtr transformers;
|
||||||
protected:
|
protected:
|
||||||
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
||||||
};
|
};
|
||||||
@ -59,10 +63,12 @@ public:
|
|||||||
|
|
||||||
void appendColumnName(WriteBuffer & ostr) const override;
|
void appendColumnName(WriteBuffer & ostr) const override;
|
||||||
const std::shared_ptr<re2::RE2> & getMatcher() const;
|
const std::shared_ptr<re2::RE2> & getMatcher() const;
|
||||||
void setPattern(String pattern);
|
void setPattern(String pattern, bool set_matcher = true);
|
||||||
void setMatcher(std::shared_ptr<re2::RE2> matcher);
|
void setMatcher(std::shared_ptr<re2::RE2> matcher);
|
||||||
void updateTreeHashImpl(SipHash & hash_state) const override;
|
void updateTreeHashImpl(SipHash & hash_state) const override;
|
||||||
|
|
||||||
|
ASTPtr qualifier;
|
||||||
|
ASTPtr transformers;
|
||||||
protected:
|
protected:
|
||||||
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
||||||
|
|
||||||
@ -78,9 +84,10 @@ public:
|
|||||||
String getID(char) const override { return "QualifiedColumnsListMatcher"; }
|
String getID(char) const override { return "QualifiedColumnsListMatcher"; }
|
||||||
ASTPtr clone() const override;
|
ASTPtr clone() const override;
|
||||||
void appendColumnName(WriteBuffer & ostr) const override;
|
void appendColumnName(WriteBuffer & ostr) const override;
|
||||||
void updateTreeHashImpl(SipHash & hash_state) const override;
|
|
||||||
|
|
||||||
|
ASTPtr qualifier;
|
||||||
ASTPtr column_list;
|
ASTPtr column_list;
|
||||||
|
ASTPtr transformers;
|
||||||
protected:
|
protected:
|
||||||
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
||||||
};
|
};
|
||||||
|
@ -19,6 +19,15 @@ namespace ErrorCodes
|
|||||||
extern const int CANNOT_COMPILE_REGEXP;
|
extern const int CANNOT_COMPILE_REGEXP;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ASTColumnsTransformerList::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||||
|
{
|
||||||
|
for (const auto & child : children)
|
||||||
|
{
|
||||||
|
settings.ostr << ' ';
|
||||||
|
child->formatImpl(settings, state, frame);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void IASTColumnsTransformer::transform(const ASTPtr & transformer, ASTs & nodes)
|
void IASTColumnsTransformer::transform(const ASTPtr & transformer, ASTs & nodes)
|
||||||
{
|
{
|
||||||
if (const auto * apply = transformer->as<ASTColumnsApplyTransformer>())
|
if (const auto * apply = transformer->as<ASTColumnsApplyTransformer>())
|
||||||
|
@ -9,6 +9,23 @@ namespace re2
|
|||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
|
/// A list of column transformers
|
||||||
|
class ASTColumnsTransformerList : public IAST
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
String getID(char) const override { return "ColumnsTransformerList"; }
|
||||||
|
ASTPtr clone() const override
|
||||||
|
{
|
||||||
|
auto clone = std::make_shared<ASTColumnsTransformerList>(*this);
|
||||||
|
clone->cloneChildren();
|
||||||
|
return clone;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected:
|
||||||
|
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
|
||||||
|
};
|
||||||
|
|
||||||
class IASTColumnsTransformer : public IAST
|
class IASTColumnsTransformer : public IAST
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
|
@ -7,22 +7,18 @@ namespace DB
|
|||||||
|
|
||||||
void ASTQualifiedAsterisk::appendColumnName(WriteBuffer & ostr) const
|
void ASTQualifiedAsterisk::appendColumnName(WriteBuffer & ostr) const
|
||||||
{
|
{
|
||||||
const auto & qualifier = children.at(0);
|
|
||||||
qualifier->appendColumnName(ostr);
|
qualifier->appendColumnName(ostr);
|
||||||
writeCString(".*", ostr);
|
writeCString(".*", ostr);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ASTQualifiedAsterisk::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
void ASTQualifiedAsterisk::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||||
{
|
{
|
||||||
const auto & qualifier = children.at(0);
|
|
||||||
qualifier->formatImpl(settings, state, frame);
|
qualifier->formatImpl(settings, state, frame);
|
||||||
settings.ostr << ".*";
|
settings.ostr << ".*";
|
||||||
|
|
||||||
/// Format column transformers
|
if (transformers)
|
||||||
for (ASTs::const_iterator it = children.begin() + 1; it != children.end(); ++it)
|
|
||||||
{
|
{
|
||||||
settings.ostr << ' ';
|
transformers->formatImpl(settings, state, frame);
|
||||||
(*it)->formatImpl(settings, state, frame);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -17,11 +17,18 @@ public:
|
|||||||
ASTPtr clone() const override
|
ASTPtr clone() const override
|
||||||
{
|
{
|
||||||
auto clone = std::make_shared<ASTQualifiedAsterisk>(*this);
|
auto clone = std::make_shared<ASTQualifiedAsterisk>(*this);
|
||||||
clone->cloneChildren();
|
|
||||||
|
if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
|
||||||
|
|
||||||
|
clone->qualifier = qualifier->clone();
|
||||||
|
clone->children.push_back(clone->qualifier);
|
||||||
|
|
||||||
return clone;
|
return clone;
|
||||||
}
|
}
|
||||||
void appendColumnName(WriteBuffer & ostr) const override;
|
void appendColumnName(WriteBuffer & ostr) const override;
|
||||||
|
|
||||||
|
ASTPtr qualifier;
|
||||||
|
ASTPtr transformers;
|
||||||
protected:
|
protected:
|
||||||
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
|
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
|
||||||
};
|
};
|
||||||
|
@ -166,6 +166,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &,
|
|||||||
else if ( type == Type::RESTART_REPLICA
|
else if ( type == Type::RESTART_REPLICA
|
||||||
|| type == Type::RESTORE_REPLICA
|
|| type == Type::RESTORE_REPLICA
|
||||||
|| type == Type::SYNC_REPLICA
|
|| type == Type::SYNC_REPLICA
|
||||||
|
|| type == Type::WAIT_LOADING_PARTS
|
||||||
|| type == Type::FLUSH_DISTRIBUTED
|
|| type == Type::FLUSH_DISTRIBUTED
|
||||||
|| type == Type::RELOAD_DICTIONARY
|
|| type == Type::RELOAD_DICTIONARY
|
||||||
|| type == Type::RELOAD_MODEL
|
|| type == Type::RELOAD_MODEL
|
||||||
|
@ -35,6 +35,7 @@ public:
|
|||||||
RESTART_REPLICAS,
|
RESTART_REPLICAS,
|
||||||
RESTART_REPLICA,
|
RESTART_REPLICA,
|
||||||
RESTORE_REPLICA,
|
RESTORE_REPLICA,
|
||||||
|
WAIT_LOADING_PARTS,
|
||||||
DROP_REPLICA,
|
DROP_REPLICA,
|
||||||
DROP_DATABASE_REPLICA,
|
DROP_DATABASE_REPLICA,
|
||||||
SYNC_REPLICA,
|
SYNC_REPLICA,
|
||||||
|
@ -1657,13 +1657,21 @@ bool ParserAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
|||||||
{
|
{
|
||||||
++pos;
|
++pos;
|
||||||
auto asterisk = std::make_shared<ASTAsterisk>();
|
auto asterisk = std::make_shared<ASTAsterisk>();
|
||||||
|
auto transformers = std::make_shared<ASTColumnsTransformerList>();
|
||||||
ParserColumnsTransformers transformers_p(allowed_transformers);
|
ParserColumnsTransformers transformers_p(allowed_transformers);
|
||||||
ASTPtr transformer;
|
ASTPtr transformer;
|
||||||
while (transformers_p.parse(pos, transformer, expected))
|
while (transformers_p.parse(pos, transformer, expected))
|
||||||
{
|
{
|
||||||
asterisk->children.push_back(transformer);
|
transformers->children.push_back(transformer);
|
||||||
}
|
}
|
||||||
node = asterisk;
|
|
||||||
|
if (!transformers->children.empty())
|
||||||
|
{
|
||||||
|
asterisk->transformers = std::move(transformers);
|
||||||
|
asterisk->children.push_back(asterisk->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
|
node = std::move(asterisk);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
@ -1672,7 +1680,7 @@ bool ParserAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
|||||||
|
|
||||||
bool ParserQualifiedAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
bool ParserQualifiedAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||||
{
|
{
|
||||||
if (!ParserCompoundIdentifier(true, true).parse(pos, node, expected))
|
if (!ParserCompoundIdentifier(false, true).parse(pos, node, expected))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
if (pos->type != TokenType::Dot)
|
if (pos->type != TokenType::Dot)
|
||||||
@ -1684,13 +1692,23 @@ bool ParserQualifiedAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
|
|||||||
++pos;
|
++pos;
|
||||||
|
|
||||||
auto res = std::make_shared<ASTQualifiedAsterisk>();
|
auto res = std::make_shared<ASTQualifiedAsterisk>();
|
||||||
res->children.push_back(node);
|
auto transformers = std::make_shared<ASTColumnsTransformerList>();
|
||||||
ParserColumnsTransformers transformers_p;
|
ParserColumnsTransformers transformers_p;
|
||||||
ASTPtr transformer;
|
ASTPtr transformer;
|
||||||
while (transformers_p.parse(pos, transformer, expected))
|
while (transformers_p.parse(pos, transformer, expected))
|
||||||
{
|
{
|
||||||
res->children.push_back(transformer);
|
transformers->children.push_back(transformer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
res->qualifier = std::move(node);
|
||||||
|
res->children.push_back(res->qualifier);
|
||||||
|
|
||||||
|
if (!transformers->children.empty())
|
||||||
|
{
|
||||||
|
res->transformers = std::move(transformers);
|
||||||
|
res->children.push_back(res->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
node = std::move(res);
|
node = std::move(res);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@ -1714,28 +1732,44 @@ static bool parseColumnsMatcherBody(IParser::Pos & pos, ASTPtr & node, Expected
|
|||||||
return false;
|
return false;
|
||||||
++pos;
|
++pos;
|
||||||
|
|
||||||
|
auto transformers = std::make_shared<ASTColumnsTransformerList>();
|
||||||
|
ParserColumnsTransformers transformers_p(allowed_transformers);
|
||||||
|
ASTPtr transformer;
|
||||||
|
while (transformers_p.parse(pos, transformer, expected))
|
||||||
|
{
|
||||||
|
transformers->children.push_back(transformer);
|
||||||
|
}
|
||||||
|
|
||||||
ASTPtr res;
|
ASTPtr res;
|
||||||
if (column_list)
|
if (column_list)
|
||||||
{
|
{
|
||||||
auto list_matcher = std::make_shared<ASTColumnsListMatcher>();
|
auto list_matcher = std::make_shared<ASTColumnsListMatcher>();
|
||||||
list_matcher->column_list = column_list;
|
|
||||||
res = list_matcher;
|
list_matcher->column_list = std::move(column_list);
|
||||||
|
list_matcher->children.push_back(list_matcher->column_list);
|
||||||
|
|
||||||
|
if (!transformers->children.empty())
|
||||||
|
{
|
||||||
|
list_matcher->transformers = std::move(transformers);
|
||||||
|
list_matcher->children.push_back(list_matcher->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
|
node = std::move(list_matcher);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
auto regexp_matcher = std::make_shared<ASTColumnsRegexpMatcher>();
|
auto regexp_matcher = std::make_shared<ASTColumnsRegexpMatcher>();
|
||||||
regexp_matcher->setPattern(regex_node->as<ASTLiteral &>().value.get<String>());
|
regexp_matcher->setPattern(regex_node->as<ASTLiteral &>().value.get<String>());
|
||||||
res = regexp_matcher;
|
|
||||||
|
if (!transformers->children.empty())
|
||||||
|
{
|
||||||
|
regexp_matcher->transformers = std::move(transformers);
|
||||||
|
regexp_matcher->children.push_back(regexp_matcher->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
|
node = std::move(regexp_matcher);
|
||||||
}
|
}
|
||||||
|
|
||||||
ParserColumnsTransformers transformers_p(allowed_transformers);
|
|
||||||
ASTPtr transformer;
|
|
||||||
while (transformers_p.parse(pos, transformer, expected))
|
|
||||||
{
|
|
||||||
res->children.push_back(transformer);
|
|
||||||
}
|
|
||||||
|
|
||||||
node = std::move(res);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1751,29 +1785,19 @@ bool ParserColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
|
|||||||
|
|
||||||
bool ParserQualifiedColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
bool ParserQualifiedColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||||
{
|
{
|
||||||
if (!ParserCompoundIdentifier(true, true).parse(pos, node, expected))
|
if (!ParserCompoundIdentifier(false, true).parse(pos, node, expected))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
auto identifier_node = node;
|
auto identifier_node = node;
|
||||||
const auto & identifier_node_typed = identifier_node->as<ASTTableIdentifier &>();
|
auto & identifier_node_typed = identifier_node->as<ASTIdentifier &>();
|
||||||
|
auto & name_parts = identifier_node_typed.name_parts;
|
||||||
|
|
||||||
/// ParserCompoundIdentifier parse identifier.COLUMNS
|
/// ParserCompoundIdentifier parse identifier.COLUMNS
|
||||||
if (identifier_node_typed.name_parts.size() == 1 || identifier_node_typed.name_parts.back() != "COLUMNS")
|
if (name_parts.size() == 1 || name_parts.back() != "COLUMNS")
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
/// TODO: ASTTableIdentifier can contain only 2 parts
|
name_parts.pop_back();
|
||||||
|
identifier_node = std::make_shared<ASTIdentifier>(std::move(name_parts), false, std::move(node->children));
|
||||||
if (identifier_node_typed.name_parts.size() == 2)
|
|
||||||
{
|
|
||||||
auto table_name = identifier_node_typed.name_parts[0];
|
|
||||||
identifier_node = std::make_shared<ASTTableIdentifier>(table_name);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
|
||||||
"Expected identifier to contain no more than 2 parts. Actual {}",
|
|
||||||
identifier_node_typed.full_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!parseColumnsMatcherBody(pos, node, expected, allowed_transformers))
|
if (!parseColumnsMatcherBody(pos, node, expected, allowed_transformers))
|
||||||
return false;
|
return false;
|
||||||
@ -1781,28 +1805,36 @@ bool ParserQualifiedColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected
|
|||||||
if (auto * columns_list_matcher = node->as<ASTColumnsListMatcher>())
|
if (auto * columns_list_matcher = node->as<ASTColumnsListMatcher>())
|
||||||
{
|
{
|
||||||
auto result = std::make_shared<ASTQualifiedColumnsListMatcher>();
|
auto result = std::make_shared<ASTQualifiedColumnsListMatcher>();
|
||||||
|
result->qualifier = std::move(identifier_node);
|
||||||
result->column_list = std::move(columns_list_matcher->column_list);
|
result->column_list = std::move(columns_list_matcher->column_list);
|
||||||
|
|
||||||
result->children.reserve(columns_list_matcher->children.size() + 1);
|
result->children.push_back(result->qualifier);
|
||||||
result->children.push_back(std::move(identifier_node));
|
result->children.push_back(result->column_list);
|
||||||
|
|
||||||
for (auto && child : columns_list_matcher->children)
|
if (columns_list_matcher->transformers)
|
||||||
result->children.push_back(std::move(child));
|
{
|
||||||
|
result->transformers = std::move(columns_list_matcher->transformers);
|
||||||
|
result->children.push_back(result->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
node = result;
|
node = std::move(result);
|
||||||
}
|
}
|
||||||
else if (auto * column_regexp_matcher = node->as<ASTColumnsRegexpMatcher>())
|
else if (auto * column_regexp_matcher = node->as<ASTColumnsRegexpMatcher>())
|
||||||
{
|
{
|
||||||
auto result = std::make_shared<ASTQualifiedColumnsRegexpMatcher>();
|
auto result = std::make_shared<ASTQualifiedColumnsRegexpMatcher>();
|
||||||
|
result->setPattern(column_regexp_matcher->getPattern(), false);
|
||||||
result->setMatcher(column_regexp_matcher->getMatcher());
|
result->setMatcher(column_regexp_matcher->getMatcher());
|
||||||
|
|
||||||
result->children.reserve(column_regexp_matcher->children.size() + 1);
|
result->qualifier = std::move(identifier_node);
|
||||||
result->children.push_back(std::move(identifier_node));
|
result->children.push_back(result->qualifier);
|
||||||
|
|
||||||
for (auto && child : column_regexp_matcher->children)
|
if (column_regexp_matcher->transformers)
|
||||||
result->children.push_back(std::move(child));
|
{
|
||||||
|
result->transformers = std::move(column_regexp_matcher->transformers);
|
||||||
|
result->children.push_back(result->transformers);
|
||||||
|
}
|
||||||
|
|
||||||
node = result;
|
node = std::move(result);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -4,6 +4,7 @@
|
|||||||
#include <Parsers/ParserSetQuery.h>
|
#include <Parsers/ParserSetQuery.h>
|
||||||
|
|
||||||
#include <Parsers/ASTAsterisk.h>
|
#include <Parsers/ASTAsterisk.h>
|
||||||
|
#include <Parsers/ASTColumnsMatcher.h>
|
||||||
#include <Parsers/ASTExpressionList.h>
|
#include <Parsers/ASTExpressionList.h>
|
||||||
#include <Parsers/ASTFunction.h>
|
#include <Parsers/ASTFunction.h>
|
||||||
#include <Parsers/ASTFunctionWithKeyValueArguments.h>
|
#include <Parsers/ASTFunctionWithKeyValueArguments.h>
|
||||||
@ -2194,7 +2195,7 @@ struct ParserExpressionImpl
|
|||||||
using Layers = std::vector<std::unique_ptr<Layer>>;
|
using Layers = std::vector<std::unique_ptr<Layer>>;
|
||||||
|
|
||||||
Action tryParseOperand(Layers & layers, IParser::Pos & pos, Expected & expected);
|
Action tryParseOperand(Layers & layers, IParser::Pos & pos, Expected & expected);
|
||||||
static Action tryParseOperator(Layers & layers, IParser::Pos & pos, Expected & expected);
|
Action tryParseOperator(Layers & layers, IParser::Pos & pos, Expected & expected);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
@ -2523,8 +2524,6 @@ Action ParserExpressionImpl::tryParseOperand(Layers & layers, IParser::Pos & pos
|
|||||||
|
|
||||||
Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & pos, Expected & expected)
|
Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & pos, Expected & expected)
|
||||||
{
|
{
|
||||||
ASTPtr tmp;
|
|
||||||
|
|
||||||
/// ParserExpression can be called in this part of the query:
|
/// ParserExpression can be called in this part of the query:
|
||||||
/// ALTER TABLE partition_all2 CLEAR INDEX [ p ] IN PARTITION ALL
|
/// ALTER TABLE partition_all2 CLEAR INDEX [ p ] IN PARTITION ALL
|
||||||
///
|
///
|
||||||
@ -2544,17 +2543,17 @@ Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & po
|
|||||||
|
|
||||||
if (cur_op == operators_table.end())
|
if (cur_op == operators_table.end())
|
||||||
{
|
{
|
||||||
|
ASTPtr alias;
|
||||||
ParserAlias alias_parser(layers.back()->allow_alias_without_as_keyword);
|
ParserAlias alias_parser(layers.back()->allow_alias_without_as_keyword);
|
||||||
auto old_pos = pos;
|
|
||||||
if (layers.back()->allow_alias &&
|
if (layers.back()->allow_alias &&
|
||||||
!layers.back()->parsed_alias &&
|
!layers.back()->parsed_alias &&
|
||||||
alias_parser.parse(pos, tmp, expected) &&
|
alias_parser.parse(pos, alias, expected) &&
|
||||||
layers.back()->insertAlias(tmp))
|
layers.back()->insertAlias(alias))
|
||||||
{
|
{
|
||||||
layers.back()->parsed_alias = true;
|
layers.back()->parsed_alias = true;
|
||||||
return Action::OPERATOR;
|
return Action::OPERATOR;
|
||||||
}
|
}
|
||||||
pos = old_pos;
|
|
||||||
return Action::NONE;
|
return Action::NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2618,33 +2617,57 @@ Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & po
|
|||||||
layers.back()->pushOperand(function);
|
layers.back()->pushOperand(function);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Dot (TupleElement operator) can be a beginning of a .* or .COLUMNS expressions
|
||||||
|
if (op.type == OperatorType::TupleElement)
|
||||||
|
{
|
||||||
|
ASTPtr tmp;
|
||||||
|
if (asterisk_parser.parse(pos, tmp, expected) ||
|
||||||
|
columns_matcher_parser.parse(pos, tmp, expected))
|
||||||
|
{
|
||||||
|
if (auto * asterisk = tmp->as<ASTAsterisk>())
|
||||||
|
{
|
||||||
|
if (!layers.back()->popOperand(asterisk->expression))
|
||||||
|
return Action::NONE;
|
||||||
|
}
|
||||||
|
else if (auto * columns_list_matcher = tmp->as<ASTColumnsListMatcher>())
|
||||||
|
{
|
||||||
|
if (!layers.back()->popOperand(columns_list_matcher->expression))
|
||||||
|
return Action::NONE;
|
||||||
|
}
|
||||||
|
else if (auto * columns_regexp_matcher = tmp->as<ASTColumnsRegexpMatcher>())
|
||||||
|
{
|
||||||
|
if (!layers.back()->popOperand(columns_regexp_matcher->expression))
|
||||||
|
return Action::NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
layers.back()->pushOperand(std::move(tmp));
|
||||||
|
return Action::OPERATOR;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
layers.back()->pushOperator(op);
|
layers.back()->pushOperator(op);
|
||||||
|
|
||||||
if (op.type == OperatorType::ArrayElement)
|
|
||||||
layers.push_back(std::make_unique<ArrayElementLayer>());
|
|
||||||
|
|
||||||
|
|
||||||
Action next = Action::OPERAND;
|
|
||||||
|
|
||||||
/// isNull & isNotNull are postfix unary operators
|
/// isNull & isNotNull are postfix unary operators
|
||||||
if (op.type == OperatorType::IsNull)
|
if (op.type == OperatorType::IsNull)
|
||||||
next = Action::OPERATOR;
|
return Action::OPERATOR;
|
||||||
|
|
||||||
if (op.type == OperatorType::StartBetween || op.type == OperatorType::StartNotBetween)
|
|
||||||
layers.back()->between_counter++;
|
|
||||||
|
|
||||||
if (op.type == OperatorType::Cast)
|
if (op.type == OperatorType::Cast)
|
||||||
{
|
{
|
||||||
next = Action::OPERATOR;
|
|
||||||
|
|
||||||
ASTPtr type_ast;
|
ASTPtr type_ast;
|
||||||
if (!ParserDataType().parse(pos, type_ast, expected))
|
if (!ParserDataType().parse(pos, type_ast, expected))
|
||||||
return Action::NONE;
|
return Action::NONE;
|
||||||
|
|
||||||
layers.back()->pushOperand(std::make_shared<ASTLiteral>(queryToString(type_ast)));
|
layers.back()->pushOperand(std::make_shared<ASTLiteral>(queryToString(type_ast)));
|
||||||
|
return Action::OPERATOR;
|
||||||
}
|
}
|
||||||
|
|
||||||
return next;
|
if (op.type == OperatorType::ArrayElement)
|
||||||
|
layers.push_back(std::make_unique<ArrayElementLayer>());
|
||||||
|
|
||||||
|
if (op.type == OperatorType::StartBetween || op.type == OperatorType::StartNotBetween)
|
||||||
|
layers.back()->between_counter++;
|
||||||
|
|
||||||
|
return Action::OPERAND;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -253,6 +253,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &
|
|||||||
|
|
||||||
case Type::RESTART_REPLICA:
|
case Type::RESTART_REPLICA:
|
||||||
case Type::SYNC_REPLICA:
|
case Type::SYNC_REPLICA:
|
||||||
|
case Type::WAIT_LOADING_PARTS:
|
||||||
{
|
{
|
||||||
if (!parseQueryWithOnCluster(res, pos, expected))
|
if (!parseQueryWithOnCluster(res, pos, expected))
|
||||||
return false;
|
return false;
|
||||||
|
@ -60,6 +60,7 @@ namespace ErrorCodes
|
|||||||
extern const int TOO_MANY_PARTITIONS;
|
extern const int TOO_MANY_PARTITIONS;
|
||||||
extern const int DISTRIBUTED_TOO_MANY_PENDING_BYTES;
|
extern const int DISTRIBUTED_TOO_MANY_PENDING_BYTES;
|
||||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||||
|
extern const int LOGICAL_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -365,18 +366,22 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
|
|||||||
const std::string & relative_path_,
|
const std::string & relative_path_,
|
||||||
ConnectionPoolPtr pool_,
|
ConnectionPoolPtr pool_,
|
||||||
ActionBlocker & monitor_blocker_,
|
ActionBlocker & monitor_blocker_,
|
||||||
BackgroundSchedulePool & bg_pool)
|
BackgroundSchedulePool & bg_pool,
|
||||||
|
bool initialize_from_disk)
|
||||||
: storage(storage_)
|
: storage(storage_)
|
||||||
, pool(std::move(pool_))
|
, pool(std::move(pool_))
|
||||||
, disk(disk_)
|
, disk(disk_)
|
||||||
, relative_path(relative_path_)
|
, relative_path(relative_path_)
|
||||||
, path(fs::path(disk->getPath()) / relative_path / "")
|
, path(fs::path(disk->getPath()) / relative_path / "")
|
||||||
|
, broken_relative_path(fs::path(relative_path) / "broken")
|
||||||
|
, broken_path(fs::path(path) / "broken" / "")
|
||||||
, should_batch_inserts(storage.getDistributedSettingsRef().monitor_batch_inserts)
|
, should_batch_inserts(storage.getDistributedSettingsRef().monitor_batch_inserts)
|
||||||
, split_batch_on_failure(storage.getDistributedSettingsRef().monitor_split_batch_on_failure)
|
, split_batch_on_failure(storage.getDistributedSettingsRef().monitor_split_batch_on_failure)
|
||||||
, dir_fsync(storage.getDistributedSettingsRef().fsync_directories)
|
, dir_fsync(storage.getDistributedSettingsRef().fsync_directories)
|
||||||
, min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows)
|
, min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows)
|
||||||
, min_batched_block_size_bytes(storage.getContext()->getSettingsRef().min_insert_block_size_bytes)
|
, min_batched_block_size_bytes(storage.getContext()->getSettingsRef().min_insert_block_size_bytes)
|
||||||
, current_batch_file_path(path + "current_batch.txt")
|
, current_batch_file_path(path + "current_batch.txt")
|
||||||
|
, pending_files(std::numeric_limits<size_t>::max())
|
||||||
, default_sleep_time(storage.getDistributedSettingsRef().monitor_sleep_time_ms.totalMilliseconds())
|
, default_sleep_time(storage.getDistributedSettingsRef().monitor_sleep_time_ms.totalMilliseconds())
|
||||||
, sleep_time(default_sleep_time)
|
, sleep_time(default_sleep_time)
|
||||||
, max_sleep_time(storage.getDistributedSettingsRef().monitor_max_sleep_time_ms.totalMilliseconds())
|
, max_sleep_time(storage.getDistributedSettingsRef().monitor_max_sleep_time_ms.totalMilliseconds())
|
||||||
@ -385,6 +390,11 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
|
|||||||
, metric_pending_files(CurrentMetrics::DistributedFilesToInsert, 0)
|
, metric_pending_files(CurrentMetrics::DistributedFilesToInsert, 0)
|
||||||
, metric_broken_files(CurrentMetrics::BrokenDistributedFilesToInsert, 0)
|
, metric_broken_files(CurrentMetrics::BrokenDistributedFilesToInsert, 0)
|
||||||
{
|
{
|
||||||
|
fs::create_directory(broken_path);
|
||||||
|
|
||||||
|
if (initialize_from_disk)
|
||||||
|
initializeFilesFromDisk();
|
||||||
|
|
||||||
task_handle = bg_pool.createTask(getLoggerName() + "/Bg", [this]{ run(); });
|
task_handle = bg_pool.createTask(getLoggerName() + "/Bg", [this]{ run(); });
|
||||||
task_handle->activateAndSchedule();
|
task_handle->activateAndSchedule();
|
||||||
}
|
}
|
||||||
@ -392,35 +402,29 @@ StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor(
|
|||||||
|
|
||||||
StorageDistributedDirectoryMonitor::~StorageDistributedDirectoryMonitor()
|
StorageDistributedDirectoryMonitor::~StorageDistributedDirectoryMonitor()
|
||||||
{
|
{
|
||||||
if (!quit)
|
if (!pending_files.isFinished())
|
||||||
{
|
{
|
||||||
quit = true;
|
pending_files.clearAndFinish();
|
||||||
task_handle->deactivate();
|
task_handle->deactivate();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void StorageDistributedDirectoryMonitor::flushAllData()
|
void StorageDistributedDirectoryMonitor::flushAllData()
|
||||||
{
|
{
|
||||||
if (quit)
|
if (pending_files.isFinished())
|
||||||
return;
|
return;
|
||||||
|
|
||||||
std::lock_guard lock{mutex};
|
std::lock_guard lock{mutex};
|
||||||
|
if (!hasPendingFiles())
|
||||||
const auto & files = getFiles();
|
return;
|
||||||
if (!files.empty())
|
processFiles();
|
||||||
{
|
|
||||||
processFiles(files);
|
|
||||||
|
|
||||||
/// Update counters.
|
|
||||||
getFiles();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void StorageDistributedDirectoryMonitor::shutdownAndDropAllData()
|
void StorageDistributedDirectoryMonitor::shutdownAndDropAllData()
|
||||||
{
|
{
|
||||||
if (!quit)
|
if (!pending_files.isFinished())
|
||||||
{
|
{
|
||||||
quit = true;
|
pending_files.clearAndFinish();
|
||||||
task_handle->deactivate();
|
task_handle->deactivate();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -434,19 +438,21 @@ void StorageDistributedDirectoryMonitor::run()
|
|||||||
std::lock_guard lock{mutex};
|
std::lock_guard lock{mutex};
|
||||||
|
|
||||||
bool do_sleep = false;
|
bool do_sleep = false;
|
||||||
while (!quit)
|
while (!pending_files.isFinished())
|
||||||
{
|
{
|
||||||
do_sleep = true;
|
do_sleep = true;
|
||||||
|
|
||||||
const auto & files = getFiles();
|
if (!hasPendingFiles())
|
||||||
if (files.empty())
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
if (!monitor_blocker.isCancelled())
|
if (!monitor_blocker.isCancelled())
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
do_sleep = !processFiles(files);
|
processFiles();
|
||||||
|
/// No errors while processing existing files.
|
||||||
|
/// Let's see maybe there are more files to process.
|
||||||
|
do_sleep = false;
|
||||||
|
|
||||||
std::lock_guard status_lock(status_mutex);
|
std::lock_guard status_lock(status_mutex);
|
||||||
status.last_exception = std::exception_ptr{};
|
status.last_exception = std::exception_ptr{};
|
||||||
@ -470,9 +476,7 @@ void StorageDistributedDirectoryMonitor::run()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
|
||||||
LOG_DEBUG(log, "Skipping send data over distributed table.");
|
LOG_DEBUG(log, "Skipping send data over distributed table.");
|
||||||
}
|
|
||||||
|
|
||||||
const auto now = std::chrono::system_clock::now();
|
const auto now = std::chrono::system_clock::now();
|
||||||
if (now - last_decrease_time > decrease_error_count_period)
|
if (now - last_decrease_time > decrease_error_count_period)
|
||||||
@ -487,10 +491,7 @@ void StorageDistributedDirectoryMonitor::run()
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Update counters.
|
if (!pending_files.isFinished() && do_sleep)
|
||||||
getFiles();
|
|
||||||
|
|
||||||
if (!quit && do_sleep)
|
|
||||||
task_handle->scheduleAfter(sleep_time.count());
|
task_handle->scheduleAfter(sleep_time.count());
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -568,41 +569,83 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri
|
|||||||
settings.distributed_replica_error_cap);
|
settings.distributed_replica_error_cap);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool StorageDistributedDirectoryMonitor::hasPendingFiles() const
|
||||||
std::map<UInt64, std::string> StorageDistributedDirectoryMonitor::getFiles()
|
|
||||||
{
|
{
|
||||||
std::map<UInt64, std::string> files;
|
return fs::exists(current_batch_file_path) || !current_batch_file.empty() || !pending_files.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
void StorageDistributedDirectoryMonitor::initializeFilesFromDisk()
|
||||||
|
{
|
||||||
|
/// NOTE: This method does not requires to hold status_mutex, hence, no TSA
|
||||||
|
/// annotations in the header file.
|
||||||
|
|
||||||
fs::directory_iterator end;
|
fs::directory_iterator end;
|
||||||
for (fs::directory_iterator it{path}; it != end; ++it)
|
|
||||||
|
/// Initialize pending files
|
||||||
{
|
{
|
||||||
const auto & file_path_str = it->path();
|
size_t bytes_count = 0;
|
||||||
if (!it->is_directory() && startsWith(fs::path(file_path_str).extension(), ".bin"))
|
|
||||||
|
for (fs::directory_iterator it{path}; it != end; ++it)
|
||||||
{
|
{
|
||||||
files[parse<UInt64>(fs::path(file_path_str).stem())] = file_path_str;
|
const auto & file_path = it->path();
|
||||||
|
const auto & base_name = file_path.stem().string();
|
||||||
|
if (!it->is_directory() && startsWith(fs::path(file_path).extension(), ".bin") && parse<UInt64>(base_name))
|
||||||
|
{
|
||||||
|
const std::string & file_path_str = file_path.string();
|
||||||
|
if (!pending_files.push(file_path_str))
|
||||||
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot add pending file");
|
||||||
|
bytes_count += fs::file_size(file_path);
|
||||||
|
}
|
||||||
|
else if (base_name != "tmp" && base_name != "broken")
|
||||||
|
{
|
||||||
|
/// It is OK to log current_batch.txt here too (useful for debugging).
|
||||||
|
LOG_WARNING(log, "Unexpected file {} in {}", file_path.string(), path);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG_TRACE(log, "Files set to {}", pending_files.size());
|
||||||
|
LOG_TRACE(log, "Bytes set to {}", bytes_count);
|
||||||
|
|
||||||
|
metric_pending_files.changeTo(pending_files.size());
|
||||||
|
status.files_count = pending_files.size();
|
||||||
|
status.bytes_count = bytes_count;
|
||||||
}
|
}
|
||||||
|
|
||||||
return files;
|
/// Initialize broken files
|
||||||
|
{
|
||||||
|
size_t broken_bytes_count = 0;
|
||||||
|
size_t broken_files = 0;
|
||||||
|
|
||||||
|
for (fs::directory_iterator it{broken_path}; it != end; ++it)
|
||||||
|
{
|
||||||
|
const auto & file_path = it->path();
|
||||||
|
if (!it->is_directory() && startsWith(fs::path(file_path).extension(), ".bin") && parse<UInt64>(file_path.stem()))
|
||||||
|
broken_bytes_count += fs::file_size(file_path);
|
||||||
|
else
|
||||||
|
LOG_WARNING(log, "Unexpected file {} in {}", file_path.string(), broken_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
LOG_TRACE(log, "Broken files set to {}", broken_files);
|
||||||
|
LOG_TRACE(log, "Broken bytes set to {}", broken_bytes_count);
|
||||||
|
|
||||||
|
metric_broken_files.changeTo(broken_files);
|
||||||
|
status.broken_files_count = broken_files;
|
||||||
|
status.broken_bytes_count = broken_bytes_count;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
bool StorageDistributedDirectoryMonitor::processFiles(const std::map<UInt64, std::string> & files)
|
void StorageDistributedDirectoryMonitor::processFiles()
|
||||||
{
|
{
|
||||||
if (should_batch_inserts)
|
if (should_batch_inserts)
|
||||||
{
|
processFilesWithBatching();
|
||||||
processFilesWithBatching(files);
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
for (const auto & file : files)
|
/// Process unprocessed file.
|
||||||
{
|
if (!current_batch_file.empty())
|
||||||
if (quit)
|
processFile(current_batch_file);
|
||||||
return true;
|
|
||||||
|
|
||||||
processFile(file.second);
|
while (pending_files.tryPop(current_batch_file))
|
||||||
}
|
processFile(current_batch_file);
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path)
|
void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path)
|
||||||
@ -649,7 +692,11 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa
|
|||||||
thread_trace_context->root_span.addAttribute(std::current_exception());
|
thread_trace_context->root_span.addAttribute(std::current_exception());
|
||||||
|
|
||||||
e.addMessage(fmt::format("While sending {}", file_path));
|
e.addMessage(fmt::format("While sending {}", file_path));
|
||||||
maybeMarkAsBroken(file_path, e);
|
if (isFileBrokenErrorCode(e.code(), e.isRemoteException()))
|
||||||
|
{
|
||||||
|
markAsBroken(file_path);
|
||||||
|
current_batch_file.clear();
|
||||||
|
}
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
catch (...)
|
catch (...)
|
||||||
@ -662,6 +709,7 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa
|
|||||||
|
|
||||||
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
|
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
|
||||||
markAsSend(file_path);
|
markAsSend(file_path);
|
||||||
|
current_batch_file.clear();
|
||||||
LOG_TRACE(log, "Finished processing `{}` (took {} ms)", file_path, watch.elapsedMilliseconds());
|
LOG_TRACE(log, "Finished processing `{}` (took {} ms)", file_path, watch.elapsedMilliseconds());
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -701,23 +749,19 @@ struct StorageDistributedDirectoryMonitor::BatchHeader
|
|||||||
|
|
||||||
struct StorageDistributedDirectoryMonitor::Batch
|
struct StorageDistributedDirectoryMonitor::Batch
|
||||||
{
|
{
|
||||||
std::vector<UInt64> file_indices;
|
|
||||||
size_t total_rows = 0;
|
size_t total_rows = 0;
|
||||||
size_t total_bytes = 0;
|
size_t total_bytes = 0;
|
||||||
bool recovered = false;
|
bool recovered = false;
|
||||||
|
|
||||||
StorageDistributedDirectoryMonitor & parent;
|
StorageDistributedDirectoryMonitor & parent;
|
||||||
const std::map<UInt64, String> & file_index_to_path;
|
std::vector<std::string> files;
|
||||||
|
|
||||||
bool split_batch_on_failure = true;
|
bool split_batch_on_failure = true;
|
||||||
bool fsync = false;
|
bool fsync = false;
|
||||||
bool dir_fsync = false;
|
bool dir_fsync = false;
|
||||||
|
|
||||||
Batch(
|
explicit Batch(StorageDistributedDirectoryMonitor & parent_)
|
||||||
StorageDistributedDirectoryMonitor & parent_,
|
|
||||||
const std::map<UInt64, String> & file_index_to_path_)
|
|
||||||
: parent(parent_)
|
: parent(parent_)
|
||||||
, file_index_to_path(file_index_to_path_)
|
|
||||||
, split_batch_on_failure(parent.split_batch_on_failure)
|
, split_batch_on_failure(parent.split_batch_on_failure)
|
||||||
, fsync(parent.storage.getDistributedSettingsRef().fsync_after_insert)
|
, fsync(parent.storage.getDistributedSettingsRef().fsync_after_insert)
|
||||||
, dir_fsync(parent.dir_fsync)
|
, dir_fsync(parent.dir_fsync)
|
||||||
@ -732,7 +776,7 @@ struct StorageDistributedDirectoryMonitor::Batch
|
|||||||
|
|
||||||
void send()
|
void send()
|
||||||
{
|
{
|
||||||
if (file_indices.empty())
|
if (files.empty())
|
||||||
return;
|
return;
|
||||||
|
|
||||||
CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend};
|
CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend};
|
||||||
@ -775,7 +819,7 @@ struct StorageDistributedDirectoryMonitor::Batch
|
|||||||
}
|
}
|
||||||
catch (const Exception & e)
|
catch (const Exception & e)
|
||||||
{
|
{
|
||||||
if (split_batch_on_failure && file_indices.size() > 1 && isSplittableErrorCode(e.code(), e.isRemoteException()))
|
if (split_batch_on_failure && files.size() > 1 && isSplittableErrorCode(e.code(), e.isRemoteException()))
|
||||||
{
|
{
|
||||||
tryLogCurrentException(parent.log, "Trying to split batch due to");
|
tryLogCurrentException(parent.log, "Trying to split batch due to");
|
||||||
sendSeparateFiles();
|
sendSeparateFiles();
|
||||||
@ -795,44 +839,28 @@ struct StorageDistributedDirectoryMonitor::Batch
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
std::vector<std::string> files;
|
e.addMessage(fmt::format("While sending a batch of {} files, files: {}", files.size(), fmt::join(files, "\n")));
|
||||||
for (const auto && file_info : file_index_to_path | boost::adaptors::indexed())
|
|
||||||
{
|
|
||||||
if (file_info.index() > 8)
|
|
||||||
{
|
|
||||||
files.push_back("...");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
files.push_back(file_info.value().second);
|
|
||||||
}
|
|
||||||
e.addMessage(fmt::format("While sending batch, nums: {}, files: {}", file_index_to_path.size(), fmt::join(files, "\n")));
|
|
||||||
|
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!batch_broken)
|
if (!batch_broken)
|
||||||
{
|
{
|
||||||
LOG_TRACE(parent.log, "Sent a batch of {} files (took {} ms).", file_indices.size(), watch.elapsedMilliseconds());
|
LOG_TRACE(parent.log, "Sent a batch of {} files (took {} ms).", files.size(), watch.elapsedMilliseconds());
|
||||||
|
|
||||||
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, parent.disk, parent.relative_path);
|
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, parent.disk, parent.relative_path);
|
||||||
for (UInt64 file_index : file_indices)
|
for (const auto & file : files)
|
||||||
parent.markAsSend(file_index_to_path.at(file_index));
|
parent.markAsSend(file);
|
||||||
}
|
}
|
||||||
else if (!batch_marked_as_broken)
|
else if (!batch_marked_as_broken)
|
||||||
{
|
{
|
||||||
LOG_ERROR(parent.log, "Marking a batch of {} files as broken.", file_indices.size());
|
LOG_ERROR(parent.log, "Marking a batch of {} files as broken, files: {}", files.size(), fmt::join(files, "\n"));
|
||||||
|
|
||||||
for (UInt64 file_idx : file_indices)
|
for (const auto & file : files)
|
||||||
{
|
parent.markAsBroken(file);
|
||||||
auto file_path = file_index_to_path.find(file_idx);
|
|
||||||
if (file_path != file_index_to_path.end())
|
|
||||||
parent.markAsBroken(file_path->second);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file_indices.clear();
|
files.clear();
|
||||||
total_rows = 0;
|
total_rows = 0;
|
||||||
total_bytes = 0;
|
total_bytes = 0;
|
||||||
recovered = false;
|
recovered = false;
|
||||||
@ -842,8 +870,11 @@ struct StorageDistributedDirectoryMonitor::Batch
|
|||||||
|
|
||||||
void writeText(WriteBuffer & out)
|
void writeText(WriteBuffer & out)
|
||||||
{
|
{
|
||||||
for (UInt64 file_idx : file_indices)
|
for (const auto & file : files)
|
||||||
out << file_idx << '\n';
|
{
|
||||||
|
UInt64 file_index = parse<UInt64>(fs::path(file).stem());
|
||||||
|
out << file_index << '\n';
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void readText(ReadBuffer & in)
|
void readText(ReadBuffer & in)
|
||||||
@ -852,8 +883,9 @@ struct StorageDistributedDirectoryMonitor::Batch
|
|||||||
{
|
{
|
||||||
UInt64 idx;
|
UInt64 idx;
|
||||||
in >> idx >> "\n";
|
in >> idx >> "\n";
|
||||||
file_indices.push_back(idx);
|
files.push_back(fmt::format("{}/{}.bin", parent.path, idx));
|
||||||
}
|
}
|
||||||
|
|
||||||
recovered = true;
|
recovered = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -865,14 +897,9 @@ private:
|
|||||||
|
|
||||||
IConnectionPool::Entry connection;
|
IConnectionPool::Entry connection;
|
||||||
|
|
||||||
for (UInt64 file_idx : file_indices)
|
for (const auto & file : files)
|
||||||
{
|
{
|
||||||
auto file_path = file_index_to_path.find(file_idx);
|
ReadBufferFromFile in(file);
|
||||||
if (file_path == file_index_to_path.end())
|
|
||||||
throw Exception(ErrorCodes::DISTRIBUTED_BROKEN_BATCH_INFO,
|
|
||||||
"Failed to send batch: file with index {} is absent", file_idx);
|
|
||||||
|
|
||||||
ReadBufferFromFile in(file_path->second);
|
|
||||||
const auto & distributed_header = readDistributedHeader(in, parent.log);
|
const auto & distributed_header = readDistributedHeader(in, parent.log);
|
||||||
|
|
||||||
OpenTelemetry::TracingContextHolder thread_trace_context(__PRETTY_FUNCTION__,
|
OpenTelemetry::TracingContextHolder thread_trace_context(__PRETTY_FUNCTION__,
|
||||||
@ -886,7 +913,7 @@ private:
|
|||||||
compression_expected = connection->getCompression() == Protocol::Compression::Enable;
|
compression_expected = connection->getCompression() == Protocol::Compression::Enable;
|
||||||
|
|
||||||
LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).",
|
LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).",
|
||||||
file_indices.size(),
|
files.size(),
|
||||||
connection->getDescription(),
|
connection->getDescription(),
|
||||||
formatReadableQuantity(total_rows),
|
formatReadableQuantity(total_rows),
|
||||||
formatReadableSizeWithBinarySuffix(total_bytes));
|
formatReadableSizeWithBinarySuffix(total_bytes));
|
||||||
@ -907,19 +934,11 @@ private:
|
|||||||
{
|
{
|
||||||
size_t broken_files = 0;
|
size_t broken_files = 0;
|
||||||
|
|
||||||
for (UInt64 file_idx : file_indices)
|
for (const auto & file : files)
|
||||||
{
|
{
|
||||||
auto file_path = file_index_to_path.find(file_idx);
|
|
||||||
if (file_path == file_index_to_path.end())
|
|
||||||
{
|
|
||||||
LOG_ERROR(parent.log, "Failed to send one file from batch: file with index {} is absent", file_idx);
|
|
||||||
++broken_files;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
ReadBufferFromFile in(file_path->second);
|
ReadBufferFromFile in(file);
|
||||||
const auto & distributed_header = readDistributedHeader(in, parent.log);
|
const auto & distributed_header = readDistributedHeader(in, parent.log);
|
||||||
|
|
||||||
// this function is called in a separated thread, so we set up the trace context from the file
|
// this function is called in a separated thread, so we set up the trace context from the file
|
||||||
@ -941,9 +960,11 @@ private:
|
|||||||
}
|
}
|
||||||
catch (Exception & e)
|
catch (Exception & e)
|
||||||
{
|
{
|
||||||
e.addMessage(fmt::format("While sending {}", file_path->second));
|
if (isFileBrokenErrorCode(e.code(), e.isRemoteException()))
|
||||||
parent.maybeMarkAsBroken(file_path->second, e);
|
{
|
||||||
++broken_files;
|
parent.markAsBroken(file);
|
||||||
|
++broken_files;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1023,13 +1044,18 @@ std::shared_ptr<ISource> StorageDistributedDirectoryMonitor::createSourceFromFil
|
|||||||
return std::make_shared<DirectoryMonitorSource>(file_name);
|
return std::make_shared<DirectoryMonitorSource>(file_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool StorageDistributedDirectoryMonitor::addAndSchedule(size_t file_size, size_t ms)
|
bool StorageDistributedDirectoryMonitor::addAndSchedule(const std::string & file_path, size_t file_size, size_t ms)
|
||||||
{
|
{
|
||||||
if (quit)
|
/// NOTE: It is better not to throw in this case, since the file is already
|
||||||
|
/// on disk (see DistributedSink), and it will be processed next time.
|
||||||
|
if (pending_files.isFinished())
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
|
if (!pending_files.push(file_path))
|
||||||
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot add pending file");
|
||||||
|
|
||||||
{
|
{
|
||||||
std::lock_guard status_lock(status_mutex);
|
std::lock_guard lock(status_mutex);
|
||||||
metric_pending_files.add();
|
metric_pending_files.add();
|
||||||
status.bytes_count += file_size;
|
status.bytes_count += file_size;
|
||||||
++status.files_count;
|
++status.files_count;
|
||||||
@ -1045,33 +1071,25 @@ StorageDistributedDirectoryMonitor::Status StorageDistributedDirectoryMonitor::g
|
|||||||
return current_status;
|
return current_status;
|
||||||
}
|
}
|
||||||
|
|
||||||
void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map<UInt64, std::string> & files)
|
void StorageDistributedDirectoryMonitor::processFilesWithBatching()
|
||||||
{
|
{
|
||||||
std::unordered_set<UInt64> file_indices_to_skip;
|
/// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch.
|
||||||
|
|
||||||
if (fs::exists(current_batch_file_path))
|
if (fs::exists(current_batch_file_path))
|
||||||
{
|
{
|
||||||
/// Possibly, we failed to send a batch on the previous iteration. Try to send exactly the same batch.
|
Batch batch(*this);
|
||||||
Batch batch(*this, files);
|
|
||||||
ReadBufferFromFile in{current_batch_file_path};
|
ReadBufferFromFile in{current_batch_file_path};
|
||||||
batch.readText(in);
|
batch.readText(in);
|
||||||
file_indices_to_skip.insert(batch.file_indices.begin(), batch.file_indices.end());
|
|
||||||
batch.send();
|
batch.send();
|
||||||
|
|
||||||
|
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
|
||||||
|
fs::remove(current_batch_file_path);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unordered_map<BatchHeader, Batch, BatchHeader::Hash> header_to_batch;
|
std::unordered_map<BatchHeader, Batch, BatchHeader::Hash> header_to_batch;
|
||||||
|
|
||||||
for (const auto & file : files)
|
std::string file_path;
|
||||||
|
while (pending_files.tryPop(file_path))
|
||||||
{
|
{
|
||||||
if (quit)
|
|
||||||
return;
|
|
||||||
|
|
||||||
UInt64 file_idx = file.first;
|
|
||||||
const String & file_path = file.second;
|
|
||||||
|
|
||||||
if (file_indices_to_skip.contains(file_idx))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
size_t total_rows = 0;
|
size_t total_rows = 0;
|
||||||
size_t total_bytes = 0;
|
size_t total_bytes = 0;
|
||||||
Block header;
|
Block header;
|
||||||
@ -1110,8 +1128,9 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
|
|||||||
}
|
}
|
||||||
catch (const Exception & e)
|
catch (const Exception & e)
|
||||||
{
|
{
|
||||||
if (maybeMarkAsBroken(file_path, e))
|
if (isFileBrokenErrorCode(e.code(), e.isRemoteException()))
|
||||||
{
|
{
|
||||||
|
markAsBroken(file_path);
|
||||||
tryLogCurrentException(log, "File is marked broken due to");
|
tryLogCurrentException(log, "File is marked broken due to");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@ -1125,9 +1144,9 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
|
|||||||
std::move(distributed_header.client_info),
|
std::move(distributed_header.client_info),
|
||||||
std::move(header)
|
std::move(header)
|
||||||
);
|
);
|
||||||
Batch & batch = header_to_batch.try_emplace(batch_header, *this, files).first->second;
|
Batch & batch = header_to_batch.try_emplace(batch_header, *this).first->second;
|
||||||
|
|
||||||
batch.file_indices.push_back(file_idx);
|
batch.files.push_back(file_path);
|
||||||
batch.total_rows += total_rows;
|
batch.total_rows += total_rows;
|
||||||
batch.total_bytes += total_bytes;
|
batch.total_bytes += total_bytes;
|
||||||
|
|
||||||
@ -1155,16 +1174,10 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map
|
|||||||
|
|
||||||
void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_path)
|
void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_path)
|
||||||
{
|
{
|
||||||
const auto last_path_separator_pos = file_path.rfind('/');
|
const String & broken_file_path = fs::path(broken_path) / fs::path(file_path).filename();
|
||||||
const auto & base_path = file_path.substr(0, last_path_separator_pos + 1);
|
|
||||||
const auto & file_name = file_path.substr(last_path_separator_pos + 1);
|
|
||||||
const String & broken_path = fs::path(base_path) / "broken/";
|
|
||||||
const String & broken_file_path = fs::path(broken_path) / file_name;
|
|
||||||
|
|
||||||
fs::create_directory(broken_path);
|
|
||||||
|
|
||||||
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
|
auto dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, relative_path);
|
||||||
auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, fs::path(relative_path) / "broken/");
|
auto broken_dir_sync_guard = getDirectorySyncGuard(dir_fsync, disk, broken_relative_path);
|
||||||
|
|
||||||
{
|
{
|
||||||
std::lock_guard status_lock(status_mutex);
|
std::lock_guard status_lock(status_mutex);
|
||||||
@ -1198,21 +1211,9 @@ void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_pat
|
|||||||
fs::remove(file_path);
|
fs::remove(file_path);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e)
|
|
||||||
{
|
|
||||||
/// Mark file as broken if necessary.
|
|
||||||
if (isFileBrokenErrorCode(e.code(), e.isRemoteException()))
|
|
||||||
{
|
|
||||||
markAsBroken(file_path);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string StorageDistributedDirectoryMonitor::getLoggerName() const
|
std::string StorageDistributedDirectoryMonitor::getLoggerName() const
|
||||||
{
|
{
|
||||||
return storage.getStorageID().getFullTableName() + ".DirectoryMonitor";
|
return storage.getStorageID().getFullTableName() + ".DirectoryMonitor." + disk->getName();
|
||||||
}
|
}
|
||||||
|
|
||||||
void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_relative_path)
|
void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_relative_path)
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <Core/BackgroundSchedulePool.h>
|
#include <Core/BackgroundSchedulePool.h>
|
||||||
|
#include <Common/ConcurrentBoundedQueue.h>
|
||||||
#include <Client/ConnectionPool.h>
|
#include <Client/ConnectionPool.h>
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
@ -38,7 +39,8 @@ public:
|
|||||||
const std::string & relative_path_,
|
const std::string & relative_path_,
|
||||||
ConnectionPoolPtr pool_,
|
ConnectionPoolPtr pool_,
|
||||||
ActionBlocker & monitor_blocker_,
|
ActionBlocker & monitor_blocker_,
|
||||||
BackgroundSchedulePool & bg_pool);
|
BackgroundSchedulePool & bg_pool,
|
||||||
|
bool initialize_from_disk);
|
||||||
|
|
||||||
~StorageDistributedDirectoryMonitor();
|
~StorageDistributedDirectoryMonitor();
|
||||||
|
|
||||||
@ -53,7 +55,7 @@ public:
|
|||||||
static std::shared_ptr<ISource> createSourceFromFile(const String & file_name);
|
static std::shared_ptr<ISource> createSourceFromFile(const String & file_name);
|
||||||
|
|
||||||
/// For scheduling via DistributedSink.
|
/// For scheduling via DistributedSink.
|
||||||
bool addAndSchedule(size_t file_size, size_t ms);
|
bool addAndSchedule(const std::string & file_path, size_t file_size, size_t ms);
|
||||||
|
|
||||||
struct InternalStatus
|
struct InternalStatus
|
||||||
{
|
{
|
||||||
@ -78,14 +80,15 @@ public:
|
|||||||
private:
|
private:
|
||||||
void run();
|
void run();
|
||||||
|
|
||||||
std::map<UInt64, std::string> getFiles();
|
bool hasPendingFiles() const;
|
||||||
bool processFiles(const std::map<UInt64, std::string> & files);
|
|
||||||
|
void initializeFilesFromDisk();
|
||||||
|
void processFiles();
|
||||||
void processFile(const std::string & file_path);
|
void processFile(const std::string & file_path);
|
||||||
void processFilesWithBatching(const std::map<UInt64, std::string> & files);
|
void processFilesWithBatching();
|
||||||
|
|
||||||
void markAsBroken(const std::string & file_path);
|
void markAsBroken(const std::string & file_path);
|
||||||
void markAsSend(const std::string & file_path);
|
void markAsSend(const std::string & file_path);
|
||||||
bool maybeMarkAsBroken(const std::string & file_path, const Exception & e);
|
|
||||||
|
|
||||||
std::string getLoggerName() const;
|
std::string getLoggerName() const;
|
||||||
|
|
||||||
@ -95,25 +98,33 @@ private:
|
|||||||
DiskPtr disk;
|
DiskPtr disk;
|
||||||
std::string relative_path;
|
std::string relative_path;
|
||||||
std::string path;
|
std::string path;
|
||||||
|
std::string broken_relative_path;
|
||||||
|
std::string broken_path;
|
||||||
|
|
||||||
const bool should_batch_inserts = false;
|
const bool should_batch_inserts = false;
|
||||||
const bool split_batch_on_failure = true;
|
const bool split_batch_on_failure = true;
|
||||||
const bool dir_fsync = false;
|
const bool dir_fsync = false;
|
||||||
const size_t min_batched_block_size_rows = 0;
|
const size_t min_batched_block_size_rows = 0;
|
||||||
const size_t min_batched_block_size_bytes = 0;
|
const size_t min_batched_block_size_bytes = 0;
|
||||||
String current_batch_file_path;
|
|
||||||
|
/// This is pending data (due to some error) for should_batch_inserts==true
|
||||||
|
std::string current_batch_file_path;
|
||||||
|
/// This is pending data (due to some error) for should_batch_inserts==false
|
||||||
|
std::string current_batch_file;
|
||||||
|
|
||||||
struct BatchHeader;
|
struct BatchHeader;
|
||||||
struct Batch;
|
struct Batch;
|
||||||
|
|
||||||
std::mutex status_mutex;
|
std::mutex status_mutex;
|
||||||
|
|
||||||
InternalStatus status;
|
InternalStatus status;
|
||||||
|
|
||||||
|
ConcurrentBoundedQueue<std::string> pending_files;
|
||||||
|
|
||||||
const std::chrono::milliseconds default_sleep_time;
|
const std::chrono::milliseconds default_sleep_time;
|
||||||
std::chrono::milliseconds sleep_time;
|
std::chrono::milliseconds sleep_time;
|
||||||
const std::chrono::milliseconds max_sleep_time;
|
const std::chrono::milliseconds max_sleep_time;
|
||||||
std::chrono::time_point<std::chrono::system_clock> last_decrease_time {std::chrono::system_clock::now()};
|
std::chrono::time_point<std::chrono::system_clock> last_decrease_time {std::chrono::system_clock::now()};
|
||||||
std::atomic<bool> quit {false};
|
|
||||||
std::mutex mutex;
|
std::mutex mutex;
|
||||||
Poco::Logger * log;
|
Poco::Logger * log;
|
||||||
ActionBlocker & monitor_blocker;
|
ActionBlocker & monitor_blocker;
|
||||||
|
@ -724,6 +724,9 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
|
|||||||
return guard;
|
return guard;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
std::vector<std::string> bin_files;
|
||||||
|
bin_files.reserve(dir_names.size());
|
||||||
|
|
||||||
auto it = dir_names.begin();
|
auto it = dir_names.begin();
|
||||||
/// on first iteration write block to a temporary directory for subsequent
|
/// on first iteration write block to a temporary directory for subsequent
|
||||||
/// hardlinking to ensure the inode is not freed until we're done
|
/// hardlinking to ensure the inode is not freed until we're done
|
||||||
@ -802,8 +805,8 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Create hardlink here to reuse increment number
|
// Create hardlink here to reuse increment number
|
||||||
const std::string block_file_path(fs::path(path) / file_name);
|
bin_files.push_back(fs::path(path) / file_name);
|
||||||
createHardLink(first_file_tmp_path, block_file_path);
|
createHardLink(first_file_tmp_path, bin_files.back());
|
||||||
auto dir_sync_guard = make_directory_sync_guard(*it);
|
auto dir_sync_guard = make_directory_sync_guard(*it);
|
||||||
}
|
}
|
||||||
++it;
|
++it;
|
||||||
@ -814,8 +817,8 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
|
|||||||
const std::string path(fs::path(disk_path) / (data_path + *it));
|
const std::string path(fs::path(disk_path) / (data_path + *it));
|
||||||
fs::create_directory(path);
|
fs::create_directory(path);
|
||||||
|
|
||||||
const std::string block_file_path(fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin"));
|
bin_files.push_back(fs::path(path) / (toString(storage.file_names_increment.get()) + ".bin"));
|
||||||
createHardLink(first_file_tmp_path, block_file_path);
|
createHardLink(first_file_tmp_path, bin_files.back());
|
||||||
auto dir_sync_guard = make_directory_sync_guard(*it);
|
auto dir_sync_guard = make_directory_sync_guard(*it);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -826,10 +829,13 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
|
|||||||
|
|
||||||
/// Notify
|
/// Notify
|
||||||
auto sleep_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms;
|
auto sleep_ms = context->getSettingsRef().distributed_directory_monitor_sleep_time_ms;
|
||||||
for (const auto & dir_name : dir_names)
|
for (size_t i = 0; i < dir_names.size(); ++i)
|
||||||
{
|
{
|
||||||
|
const auto & dir_name = dir_names[i];
|
||||||
|
const auto & bin_file = bin_files[i];
|
||||||
|
|
||||||
auto & directory_monitor = storage.requireDirectoryMonitor(disk, dir_name, /* startup= */ false);
|
auto & directory_monitor = storage.requireDirectoryMonitor(disk, dir_name, /* startup= */ false);
|
||||||
directory_monitor.addAndSchedule(file_size, sleep_ms.totalMilliseconds());
|
directory_monitor.addAndSchedule(bin_file, file_size, sleep_ms.totalMilliseconds());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
File diff suppressed because it is too large
Load Diff
@ -1052,6 +1052,8 @@ public:
|
|||||||
/// Returns an object that protects temporary directory from cleanup
|
/// Returns an object that protects temporary directory from cleanup
|
||||||
scope_guard getTemporaryPartDirectoryHolder(const String & part_dir_name) const;
|
scope_guard getTemporaryPartDirectoryHolder(const String & part_dir_name) const;
|
||||||
|
|
||||||
|
void waitForOutdatedPartsToBeLoaded() const;
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
friend class IMergeTreeDataPart;
|
friend class IMergeTreeDataPart;
|
||||||
friend class MergeTreeDataMergerMutator;
|
friend class MergeTreeDataMergerMutator;
|
||||||
@ -1068,7 +1070,6 @@ protected:
|
|||||||
/// under lockForShare if rename is possible.
|
/// under lockForShare if rename is possible.
|
||||||
String relative_data_path;
|
String relative_data_path;
|
||||||
|
|
||||||
|
|
||||||
/// Current column sizes in compressed and uncompressed form.
|
/// Current column sizes in compressed and uncompressed form.
|
||||||
ColumnSizeByName column_sizes;
|
ColumnSizeByName column_sizes;
|
||||||
|
|
||||||
@ -1330,6 +1331,88 @@ protected:
|
|||||||
void resetObjectColumnsFromActiveParts(const DataPartsLock & lock);
|
void resetObjectColumnsFromActiveParts(const DataPartsLock & lock);
|
||||||
void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock);
|
void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock);
|
||||||
|
|
||||||
|
/** A structure that explicitly represents a "merge tree" of parts
|
||||||
|
* which is implicitly presented by min-max block numbers and levels of parts.
|
||||||
|
* The children of node are parts which are covered by parent part.
|
||||||
|
* This tree provides the order of loading of parts.
|
||||||
|
*
|
||||||
|
* We start to traverse tree from the top level and load parts
|
||||||
|
* corresposponded to nodes. If part is loaded successfully then
|
||||||
|
* we stop traversal at this node. Otherwise part is broken and we
|
||||||
|
* traverse its children and try to load covered parts which will
|
||||||
|
* replace broken covering part. Unloaded nodes represent outdated parts
|
||||||
|
* nd they are pushed to background task and loaded asynchronoulsy.
|
||||||
|
*/
|
||||||
|
class PartLoadingTree
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
struct Node
|
||||||
|
{
|
||||||
|
Node(const MergeTreePartInfo & info_, const String & name_, const DiskPtr & disk_)
|
||||||
|
: info(info_), name(name_), disk(disk_)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
const MergeTreePartInfo info;
|
||||||
|
const String name;
|
||||||
|
const DiskPtr disk;
|
||||||
|
|
||||||
|
bool is_loaded = false;
|
||||||
|
std::map<MergeTreePartInfo, std::shared_ptr<Node>> children;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct PartLoadingInfo
|
||||||
|
{
|
||||||
|
PartLoadingInfo(const MergeTreePartInfo & info_, const String & name_, const DiskPtr & disk_)
|
||||||
|
: info(info_), name(name_), disk(disk_)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Store name explicitly because it cannot be easily
|
||||||
|
/// retrieved from info in tables with old syntax.
|
||||||
|
MergeTreePartInfo info;
|
||||||
|
String name;
|
||||||
|
DiskPtr disk;
|
||||||
|
};
|
||||||
|
|
||||||
|
using NodePtr = std::shared_ptr<Node>;
|
||||||
|
using PartLoadingInfos = std::vector<PartLoadingInfo>;
|
||||||
|
|
||||||
|
/// Builds a tree from the list of part infos.
|
||||||
|
static PartLoadingTree build(PartLoadingInfos nodes);
|
||||||
|
|
||||||
|
/// Traverses a tree and call @func on each node.
|
||||||
|
/// If recursive is false traverses only the top level.
|
||||||
|
template <typename Func>
|
||||||
|
void traverse(bool recursive, Func && func);
|
||||||
|
|
||||||
|
private:
|
||||||
|
/// NOTE: Parts should be added in descending order of their levels
|
||||||
|
/// because rearranging tree to the new root is not supported.
|
||||||
|
void add(const MergeTreePartInfo & info, const String & name, const DiskPtr & disk);
|
||||||
|
std::unordered_map<String, NodePtr> root_by_partition;
|
||||||
|
};
|
||||||
|
|
||||||
|
using PartLoadingTreeNodes = std::vector<PartLoadingTree::NodePtr>;
|
||||||
|
|
||||||
|
struct LoadPartResult
|
||||||
|
{
|
||||||
|
bool is_broken = false;
|
||||||
|
std::optional<size_t> size_of_part;
|
||||||
|
MutableDataPartPtr part;
|
||||||
|
};
|
||||||
|
|
||||||
|
mutable std::mutex outdated_data_parts_mutex;
|
||||||
|
mutable std::condition_variable outdated_data_parts_cv;
|
||||||
|
|
||||||
|
BackgroundSchedulePool::TaskHolder outdated_data_parts_loading_task;
|
||||||
|
PartLoadingTreeNodes outdated_unloaded_data_parts TSA_GUARDED_BY(outdated_data_parts_mutex);
|
||||||
|
bool outdated_data_parts_loading_canceled TSA_GUARDED_BY(outdated_data_parts_mutex) = false;
|
||||||
|
|
||||||
|
void loadOutdatedDataParts(bool is_async);
|
||||||
|
void startOutdatedDataPartsLoadingTask();
|
||||||
|
void stopOutdatedDataPartsLoadingTask();
|
||||||
|
|
||||||
static void incrementInsertedPartsProfileEvent(MergeTreeDataPartType type);
|
static void incrementInsertedPartsProfileEvent(MergeTreeDataPartType type);
|
||||||
static void incrementMergedPartsProfileEvent(MergeTreeDataPartType type);
|
static void incrementMergedPartsProfileEvent(MergeTreeDataPartType type);
|
||||||
|
|
||||||
@ -1408,18 +1491,20 @@ private:
|
|||||||
/// Returns default settings for storage with possible changes from global config.
|
/// Returns default settings for storage with possible changes from global config.
|
||||||
virtual std::unique_ptr<MergeTreeSettings> getDefaultSettings() const = 0;
|
virtual std::unique_ptr<MergeTreeSettings> getDefaultSettings() const = 0;
|
||||||
|
|
||||||
void loadDataPartsFromDisk(
|
LoadPartResult loadDataPart(
|
||||||
MutableDataPartsVector & broken_parts_to_detach,
|
const MergeTreePartInfo & part_info,
|
||||||
MutableDataPartsVector & duplicate_parts_to_remove,
|
const String & part_name,
|
||||||
|
const DiskPtr & part_disk_ptr,
|
||||||
|
MergeTreeDataPartState to_state,
|
||||||
|
std::mutex & part_loading_mutex);
|
||||||
|
|
||||||
|
std::vector<LoadPartResult> loadDataPartsFromDisk(
|
||||||
ThreadPool & pool,
|
ThreadPool & pool,
|
||||||
size_t num_parts,
|
size_t num_parts,
|
||||||
std::queue<std::vector<std::pair<String, DiskPtr>>> & parts_queue,
|
std::queue<PartLoadingTreeNodes> & parts_queue,
|
||||||
bool skip_sanity_checks,
|
|
||||||
const MergeTreeSettingsPtr & settings);
|
const MergeTreeSettingsPtr & settings);
|
||||||
|
|
||||||
void loadDataPartsFromWAL(
|
void loadDataPartsFromWAL(MutableDataPartsVector & parts_from_wal);
|
||||||
MutableDataPartsVector & duplicate_parts_to_remove,
|
|
||||||
MutableDataPartsVector & parts_from_wal);
|
|
||||||
|
|
||||||
/// Create zero-copy exclusive lock for part and disk. Useful for coordination of
|
/// Create zero-copy exclusive lock for part and disk. Useful for coordination of
|
||||||
/// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree.
|
/// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree.
|
||||||
@ -1430,7 +1515,7 @@ private:
|
|||||||
/// Otherwise, in non-parallel case will break and return.
|
/// Otherwise, in non-parallel case will break and return.
|
||||||
void clearPartsFromFilesystemImpl(const DataPartsVector & parts, NameSet * part_names_succeed);
|
void clearPartsFromFilesystemImpl(const DataPartsVector & parts, NameSet * part_names_succeed);
|
||||||
|
|
||||||
static MutableDataPartPtr preparePartForRemoval(const DataPartPtr & part);
|
static MutableDataPartPtr asMutableDeletingPart(const DataPartPtr & part);
|
||||||
|
|
||||||
mutable TemporaryParts temporary_parts;
|
mutable TemporaryParts temporary_parts;
|
||||||
};
|
};
|
||||||
|
@ -193,7 +193,8 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(
|
|||||||
if (!metadata_snapshot->hasPartitionKey()) /// Table is not partitioned.
|
if (!metadata_snapshot->hasPartitionKey()) /// Table is not partitioned.
|
||||||
{
|
{
|
||||||
result.emplace_back(Block(block), Row{});
|
result.emplace_back(Block(block), Row{});
|
||||||
result[0].offsets = chunk_offsets;
|
if (chunk_offsets != nullptr)
|
||||||
|
result[0].offsets = std::move(chunk_offsets->offsets);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -230,7 +231,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(
|
|||||||
/// do not interfere with possible calculated primary key columns of the same name.
|
/// do not interfere with possible calculated primary key columns of the same name.
|
||||||
result.emplace_back(Block(block), get_partition(0));
|
result.emplace_back(Block(block), get_partition(0));
|
||||||
if (!chunk_offsets_with_partition.empty())
|
if (!chunk_offsets_with_partition.empty())
|
||||||
result[0].offsets = chunk_offsets_with_partition[0];
|
result[0].offsets = std::move(chunk_offsets_with_partition[0]->offsets);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -245,7 +246,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (size_t i = 0; i < chunk_offsets_with_partition.size(); ++i)
|
for (size_t i = 0; i < chunk_offsets_with_partition.size(); ++i)
|
||||||
result[i].offsets = chunk_offsets_with_partition[i];
|
result[i].offsets = std::move(chunk_offsets_with_partition[i]->offsets);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
@ -22,15 +22,15 @@ struct BlockWithPartition
|
|||||||
{
|
{
|
||||||
Block block;
|
Block block;
|
||||||
Row partition;
|
Row partition;
|
||||||
ChunkOffsetsPtr offsets;
|
std::vector<size_t> offsets;
|
||||||
|
|
||||||
BlockWithPartition(Block && block_, Row && partition_)
|
BlockWithPartition(Block && block_, Row && partition_)
|
||||||
: block(block_), partition(std::move(partition_))
|
: block(block_), partition(std::move(partition_))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
BlockWithPartition(Block && block_, Row && partition_, ChunkOffsetsPtr chunk_offsets_)
|
BlockWithPartition(Block && block_, Row && partition_, std::vector<size_t> && offsets_)
|
||||||
: block(block_), partition(std::move(partition_)), offsets(chunk_offsets_)
|
: block(block_), partition(std::move(partition_)), offsets(std::move(offsets_))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -147,7 +147,6 @@ bool ReplicatedMergeTreeRestartingThread::runImpl()
|
|||||||
storage.part_check_thread.start();
|
storage.part_check_thread.start();
|
||||||
|
|
||||||
LOG_DEBUG(log, "Table started successfully");
|
LOG_DEBUG(log, "Table started successfully");
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,15 +41,17 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
|
|||||||
{
|
{
|
||||||
struct Partition
|
struct Partition
|
||||||
{
|
{
|
||||||
|
Poco::Logger * log;
|
||||||
MergeTreeDataWriter::TemporaryPart temp_part;
|
MergeTreeDataWriter::TemporaryPart temp_part;
|
||||||
UInt64 elapsed_ns;
|
UInt64 elapsed_ns;
|
||||||
BlockIDsType block_id;
|
BlockIDsType block_id;
|
||||||
BlockWithPartition block_with_partition;
|
BlockWithPartition block_with_partition;
|
||||||
std::unordered_map<String, size_t> block_id_to_offset_idx;
|
std::unordered_map<String, std::vector<size_t>> block_id_to_offset_idx;
|
||||||
|
|
||||||
Partition() = default;
|
Partition() = default;
|
||||||
Partition(MergeTreeDataWriter::TemporaryPart && temp_part_, UInt64 elapsed_ns_, BlockIDsType && block_id_, BlockWithPartition && block_)
|
Partition(Poco::Logger * log_, MergeTreeDataWriter::TemporaryPart && temp_part_, UInt64 elapsed_ns_, BlockIDsType && block_id_, BlockWithPartition && block_)
|
||||||
: temp_part(std::move(temp_part_)),
|
: log(log_),
|
||||||
|
temp_part(std::move(temp_part_)),
|
||||||
elapsed_ns(elapsed_ns_),
|
elapsed_ns(elapsed_ns_),
|
||||||
block_id(std::move(block_id_)),
|
block_id(std::move(block_id_)),
|
||||||
block_with_partition(std::move(block_))
|
block_with_partition(std::move(block_))
|
||||||
@ -64,11 +66,105 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
|
|||||||
block_id_to_offset_idx.clear();
|
block_id_to_offset_idx.clear();
|
||||||
for (size_t i = 0; i < block_id.size(); ++i)
|
for (size_t i = 0; i < block_id.size(); ++i)
|
||||||
{
|
{
|
||||||
block_id_to_offset_idx[block_id[i]] = i;
|
block_id_to_offset_idx[block_id[i]].push_back(i);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// this function check if the block contains duplicate inserts.
|
||||||
|
/// if so, we keep only one insert for every duplicate ones.
|
||||||
|
bool filterSelfDuplicate()
|
||||||
|
{
|
||||||
|
if constexpr (async_insert)
|
||||||
|
{
|
||||||
|
std::vector<String> dup_block_ids;
|
||||||
|
for (const auto & [hash_id, offset_indexes] : block_id_to_offset_idx)
|
||||||
|
{
|
||||||
|
/// It means more than one inserts have the same hash id, in this case, we should keep only one of them.
|
||||||
|
if (offset_indexes.size() > 1)
|
||||||
|
dup_block_ids.push_back(hash_id);
|
||||||
|
}
|
||||||
|
if (dup_block_ids.empty())
|
||||||
|
return false;
|
||||||
|
|
||||||
|
filterBlockDuplicate(dup_block_ids, true);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// remove the conflict parts of block for rewriting again.
|
||||||
|
void filterBlockDuplicate(const std::vector<String> & block_paths, bool self_dedup)
|
||||||
|
{
|
||||||
|
if constexpr (async_insert)
|
||||||
|
{
|
||||||
|
std::vector<size_t> offset_idx;
|
||||||
|
for (const auto & raw_path : block_paths)
|
||||||
|
{
|
||||||
|
std::filesystem::path p(raw_path);
|
||||||
|
String conflict_block_id = p.filename();
|
||||||
|
auto it = block_id_to_offset_idx.find(conflict_block_id);
|
||||||
|
if (it == block_id_to_offset_idx.end())
|
||||||
|
throw Exception("Unknown conflict path " + conflict_block_id, ErrorCodes::LOGICAL_ERROR);
|
||||||
|
/// if this filter is for self_dedup, that means the block paths is selected by `filterSelfDuplicate`, which is a self purge.
|
||||||
|
/// in this case, we don't know if zk has this insert, then we should keep one insert, to avoid missing this insert.
|
||||||
|
offset_idx.insert(std::end(offset_idx), std::begin(it->second) + self_dedup, std::end(it->second));
|
||||||
|
}
|
||||||
|
std::sort(offset_idx.begin(), offset_idx.end());
|
||||||
|
|
||||||
|
auto & offsets = block_with_partition.offsets;
|
||||||
|
size_t idx = 0, remove_count = 0;
|
||||||
|
auto it = offset_idx.begin();
|
||||||
|
std::vector<size_t> new_offsets;
|
||||||
|
std::vector<String> new_block_ids;
|
||||||
|
|
||||||
|
/// construct filter
|
||||||
|
size_t rows = block_with_partition.block.rows();
|
||||||
|
auto filter_col = ColumnUInt8::create(rows, 1u);
|
||||||
|
ColumnUInt8::Container & vec = filter_col->getData();
|
||||||
|
UInt8 * pos = vec.data();
|
||||||
|
for (auto & offset : offsets)
|
||||||
|
{
|
||||||
|
if (it != offset_idx.end() && *it == idx)
|
||||||
|
{
|
||||||
|
size_t start_pos = idx > 0 ? offsets[idx - 1] : 0;
|
||||||
|
size_t end_pos = offset;
|
||||||
|
remove_count += end_pos - start_pos;
|
||||||
|
while (start_pos < end_pos)
|
||||||
|
{
|
||||||
|
*(pos + start_pos) = 0;
|
||||||
|
start_pos++;
|
||||||
|
}
|
||||||
|
it++;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
new_offsets.push_back(offset - remove_count);
|
||||||
|
new_block_ids.push_back(block_id[idx]);
|
||||||
|
}
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
|
||||||
|
LOG_TRACE(log, "New block IDs: {}, new offsets: {}, size: {}", toString(new_block_ids), toString(new_offsets), new_offsets.size());
|
||||||
|
|
||||||
|
block_with_partition.offsets = std::move(new_offsets);
|
||||||
|
block_id = std::move(new_block_ids);
|
||||||
|
auto cols = block_with_partition.block.getColumns();
|
||||||
|
for (auto & col : cols)
|
||||||
|
{
|
||||||
|
col = col->filter(vec, rows - remove_count);
|
||||||
|
}
|
||||||
|
block_with_partition.block.setColumns(cols);
|
||||||
|
|
||||||
|
LOG_TRACE(log, "New block rows {}", block_with_partition.block.rows());
|
||||||
|
|
||||||
|
initBlockIDMap();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "sync insert should not call rewriteBlock");
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
DelayedChunk() = default;
|
DelayedChunk() = default;
|
||||||
@ -79,6 +175,30 @@ struct ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk
|
|||||||
std::vector<Partition> partitions;
|
std::vector<Partition> partitions;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
std::vector<Int64> testSelfDeduplicate(std::vector<Int64> data, std::vector<size_t> offsets, std::vector<String> hashes)
|
||||||
|
{
|
||||||
|
MutableColumnPtr column = DataTypeInt64().createColumn();
|
||||||
|
for (auto datum : data)
|
||||||
|
{
|
||||||
|
column->insert(datum);
|
||||||
|
}
|
||||||
|
Block block({ColumnWithTypeAndName(std::move(column), DataTypePtr(new DataTypeInt64()), "a")});
|
||||||
|
|
||||||
|
BlockWithPartition block1(std::move(block), Row(), std::move(offsets));
|
||||||
|
ReplicatedMergeTreeSinkImpl<true>::DelayedChunk::Partition part(
|
||||||
|
&Poco::Logger::get("testSelfDeduplicate"), MergeTreeDataWriter::TemporaryPart(), 0, std::move(hashes), std::move(block1));
|
||||||
|
|
||||||
|
part.filterSelfDuplicate();
|
||||||
|
|
||||||
|
ColumnPtr col = part.block_with_partition.block.getColumns()[0];
|
||||||
|
std::vector<Int64> result;
|
||||||
|
for (size_t i = 0; i < col->size(); i++)
|
||||||
|
{
|
||||||
|
result.push_back(col->getInt(i));
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
namespace
|
namespace
|
||||||
{
|
{
|
||||||
/// Convert block id vector to string. Output at most 50 ids.
|
/// Convert block id vector to string. Output at most 50 ids.
|
||||||
@ -90,76 +210,12 @@ namespace
|
|||||||
return fmt::format("({})", fmt::join(vec.begin(), vec.begin() + size, ","));
|
return fmt::format("({})", fmt::join(vec.begin(), vec.begin() + size, ","));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// remove the conflict parts of block for rewriting again.
|
|
||||||
void rewriteBlock(Poco::Logger * log, typename ReplicatedMergeTreeSinkImpl<true>::DelayedChunk::Partition & partition, const std::vector<String> & block_paths)
|
|
||||||
{
|
|
||||||
std::vector<size_t> offset_idx;
|
|
||||||
for (const auto & raw_path : block_paths)
|
|
||||||
{
|
|
||||||
std::filesystem::path p(raw_path);
|
|
||||||
String conflict_block_id = p.filename();
|
|
||||||
auto it = partition.block_id_to_offset_idx.find(conflict_block_id);
|
|
||||||
if (it == partition.block_id_to_offset_idx.end())
|
|
||||||
throw Exception("Unknown conflict path " + conflict_block_id, ErrorCodes::LOGICAL_ERROR);
|
|
||||||
offset_idx.push_back(it->second);
|
|
||||||
}
|
|
||||||
std::sort(offset_idx.begin(), offset_idx.end());
|
|
||||||
|
|
||||||
auto & offsets = partition.block_with_partition.offsets->offsets;
|
|
||||||
size_t idx = 0, remove_count = 0;
|
|
||||||
auto it = offset_idx.begin();
|
|
||||||
std::vector<size_t> new_offsets;
|
|
||||||
std::vector<String> new_block_ids;
|
|
||||||
|
|
||||||
/// construct filter
|
|
||||||
size_t rows = partition.block_with_partition.block.rows();
|
|
||||||
auto filter_col = ColumnUInt8::create(rows, 1u);
|
|
||||||
ColumnUInt8::Container & vec = filter_col->getData();
|
|
||||||
UInt8 * pos = vec.data();
|
|
||||||
for (auto & offset : offsets)
|
|
||||||
{
|
|
||||||
if (it != offset_idx.end() && *it == idx)
|
|
||||||
{
|
|
||||||
size_t start_pos = idx > 0 ? offsets[idx - 1] : 0;
|
|
||||||
size_t end_pos = offset;
|
|
||||||
remove_count += end_pos - start_pos;
|
|
||||||
while (start_pos < end_pos)
|
|
||||||
{
|
|
||||||
*(pos + start_pos) = 0;
|
|
||||||
start_pos ++;
|
|
||||||
}
|
|
||||||
it++;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
new_offsets.push_back(offset - remove_count);
|
|
||||||
new_block_ids.push_back(partition.block_id[idx]);
|
|
||||||
}
|
|
||||||
idx++;
|
|
||||||
}
|
|
||||||
|
|
||||||
LOG_TRACE(log, "New block IDs: {}, new offsets: {}, size: {}", toString(new_block_ids), toString(new_offsets), new_offsets.size());
|
|
||||||
|
|
||||||
offsets = std::move(new_offsets);
|
|
||||||
partition.block_id = std::move(new_block_ids);
|
|
||||||
auto cols = partition.block_with_partition.block.getColumns();
|
|
||||||
for (auto & col : cols)
|
|
||||||
{
|
|
||||||
col = col->filter(vec, rows - remove_count);
|
|
||||||
}
|
|
||||||
partition.block_with_partition.block.setColumns(cols);
|
|
||||||
|
|
||||||
LOG_TRACE(log, "New block rows {}", partition.block_with_partition.block.rows());
|
|
||||||
|
|
||||||
partition.initBlockIDMap();
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<String> getHashesForBlocks(BlockWithPartition & block, String partition_id)
|
std::vector<String> getHashesForBlocks(BlockWithPartition & block, String partition_id)
|
||||||
{
|
{
|
||||||
size_t start = 0;
|
size_t start = 0;
|
||||||
auto cols = block.block.getColumns();
|
auto cols = block.block.getColumns();
|
||||||
std::vector<String> block_id_vec;
|
std::vector<String> block_id_vec;
|
||||||
for (auto offset : block.offsets->offsets)
|
for (auto offset : block.offsets)
|
||||||
{
|
{
|
||||||
SipHash hash;
|
SipHash hash;
|
||||||
for (size_t i = start; i < offset; ++i)
|
for (size_t i = start; i < offset; ++i)
|
||||||
@ -369,7 +425,7 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
|
|||||||
{
|
{
|
||||||
/// TODO consider insert_deduplication_token
|
/// TODO consider insert_deduplication_token
|
||||||
block_id = getHashesForBlocks(current_block, temp_part.part->info.partition_id);
|
block_id = getHashesForBlocks(current_block, temp_part.part->info.partition_id);
|
||||||
LOG_TRACE(log, "async insert part, part id {}, block id {}, offsets {}, size {}", temp_part.part->info.partition_id, toString(block_id), toString(current_block.offsets->offsets), current_block.offsets->offsets.size());
|
LOG_TRACE(log, "async insert part, part id {}, block id {}, offsets {}, size {}", temp_part.part->info.partition_id, toString(block_id), toString(current_block.offsets), current_block.offsets.size());
|
||||||
}
|
}
|
||||||
else if (deduplicate)
|
else if (deduplicate)
|
||||||
{
|
{
|
||||||
@ -416,6 +472,7 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
|
|||||||
}
|
}
|
||||||
|
|
||||||
partitions.emplace_back(DelayedPartition(
|
partitions.emplace_back(DelayedPartition(
|
||||||
|
log,
|
||||||
std::move(temp_part),
|
std::move(temp_part),
|
||||||
elapsed_ns,
|
elapsed_ns,
|
||||||
std::move(block_id),
|
std::move(block_id),
|
||||||
@ -479,6 +536,14 @@ void ReplicatedMergeTreeSinkImpl<true>::finishDelayedChunk(const ZooKeeperWithFa
|
|||||||
for (auto & partition: delayed_chunk->partitions)
|
for (auto & partition: delayed_chunk->partitions)
|
||||||
{
|
{
|
||||||
int retry_times = 0;
|
int retry_times = 0;
|
||||||
|
/// users may have lots of same inserts. It will be helpful to deduplicate in advance.
|
||||||
|
if (partition.filterSelfDuplicate())
|
||||||
|
{
|
||||||
|
LOG_TRACE(log, "found duplicated inserts in the block");
|
||||||
|
partition.block_with_partition.partition = std::move(partition.temp_part.part->partition.value);
|
||||||
|
partition.temp_part = storage.writer.writeTempPart(partition.block_with_partition, metadata_snapshot, context);
|
||||||
|
}
|
||||||
|
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
partition.temp_part.finalize();
|
partition.temp_part.finalize();
|
||||||
@ -488,7 +553,7 @@ void ReplicatedMergeTreeSinkImpl<true>::finishDelayedChunk(const ZooKeeperWithFa
|
|||||||
++retry_times;
|
++retry_times;
|
||||||
LOG_DEBUG(log, "Found duplicate block IDs: {}, retry times {}", toString(conflict_block_ids), retry_times);
|
LOG_DEBUG(log, "Found duplicate block IDs: {}, retry times {}", toString(conflict_block_ids), retry_times);
|
||||||
/// partition clean conflict
|
/// partition clean conflict
|
||||||
rewriteBlock(log, partition, conflict_block_ids);
|
partition.filterBlockDuplicate(conflict_block_ids, false);
|
||||||
if (partition.block_id.empty())
|
if (partition.block_id.empty())
|
||||||
break;
|
break;
|
||||||
partition.block_with_partition.partition = std::move(partition.temp_part.part->partition.value);
|
partition.block_with_partition.partition = std::move(partition.temp_part.part->partition.value);
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
|
#include "Storages/MergeTree/ReplicatedMergeTreeSink.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
@ -42,4 +43,21 @@ TEST(AsyncInsertsTest, testScatterOffsetsBySelector)
|
|||||||
test_impl({3,6,10}, {1,1,1,2,2,2,0,0,0,0}, 3, {{4},{3},{3}});
|
test_impl({3,6,10}, {1,1,1,2,2,2,0,0,0,0}, 3, {{4},{3},{3}});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::vector<Int64> testSelfDeduplicate(std::vector<Int64> data, std::vector<size_t> offsets, std::vector<String> hashes);
|
||||||
|
|
||||||
|
TEST(AsyncInsertsTest, testSelfDeduplicate)
|
||||||
|
{
|
||||||
|
auto test_impl = [](std::vector<Int64> data, std::vector<size_t> offsets, std::vector<String> hashes, std::vector<Int64> answer)
|
||||||
|
{
|
||||||
|
auto result = testSelfDeduplicate(data, offsets, hashes);
|
||||||
|
ASSERT_EQ(answer.size(), result.size());
|
||||||
|
for (size_t i = 0; i < result.size(); i++)
|
||||||
|
ASSERT_EQ(answer[i], result[i]);
|
||||||
|
};
|
||||||
|
test_impl({1,2,3,1,2,3,4,5,6,1,2,3},{3,6,9,12},{"a","a","b","a"},{1,2,3,4,5,6});
|
||||||
|
test_impl({1,2,3,1,2,3,1,2,3,1,2,3},{2,3,5,6,8,9,11,12},{"a","b","a","b","a","b","a","b"},{1,2,3});
|
||||||
|
test_impl({1,2,3,1,2,4,1,2,5,1,2},{2,3,5,6,8,9,11},{"a","b","a","c","a","d","a"},{1,2,3,4,5});
|
||||||
|
test_impl({1,2,1,2,1,2,1,2,1,2},{2,4,6,8,10},{"a","a","a","a","a"},{1,2});
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -1205,12 +1205,15 @@ void StorageDistributed::createDirectoryMonitors(const DiskPtr & disk)
|
|||||||
const auto & dir_path = it->path();
|
const auto & dir_path = it->path();
|
||||||
if (std::filesystem::is_directory(dir_path))
|
if (std::filesystem::is_directory(dir_path))
|
||||||
{
|
{
|
||||||
|
/// Created by DistributedSink
|
||||||
const auto & tmp_path = dir_path / "tmp";
|
const auto & tmp_path = dir_path / "tmp";
|
||||||
|
|
||||||
/// "tmp" created by DistributedSink
|
|
||||||
if (std::filesystem::is_directory(tmp_path) && std::filesystem::is_empty(tmp_path))
|
if (std::filesystem::is_directory(tmp_path) && std::filesystem::is_empty(tmp_path))
|
||||||
std::filesystem::remove(tmp_path);
|
std::filesystem::remove(tmp_path);
|
||||||
|
|
||||||
|
const auto & broken_path = dir_path / "broken";
|
||||||
|
if (std::filesystem::is_directory(broken_path) && std::filesystem::is_empty(broken_path))
|
||||||
|
std::filesystem::remove(broken_path);
|
||||||
|
|
||||||
if (std::filesystem::is_empty(dir_path))
|
if (std::filesystem::is_empty(dir_path))
|
||||||
{
|
{
|
||||||
LOG_DEBUG(log, "Removing {} (used for async INSERT into Distributed)", dir_path.string());
|
LOG_DEBUG(log, "Removing {} (used for async INSERT into Distributed)", dir_path.string());
|
||||||
@ -1239,7 +1242,8 @@ StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor(
|
|||||||
*this, disk, relative_data_path + name,
|
*this, disk, relative_data_path + name,
|
||||||
data.connection_pool,
|
data.connection_pool,
|
||||||
monitors_blocker,
|
monitors_blocker,
|
||||||
getContext()->getDistributedSchedulePool());
|
getContext()->getDistributedSchedulePool(),
|
||||||
|
/* initialize_from_disk= */ startup);
|
||||||
return data;
|
return data;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -112,7 +112,6 @@ StorageMergeTree::StorageMergeTree(
|
|||||||
increment.set(getMaxBlockNumber());
|
increment.set(getMaxBlockNumber());
|
||||||
|
|
||||||
loadMutations();
|
loadMutations();
|
||||||
|
|
||||||
loadDeduplicationLog();
|
loadDeduplicationLog();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -138,6 +137,7 @@ void StorageMergeTree::startup()
|
|||||||
{
|
{
|
||||||
background_operations_assignee.start();
|
background_operations_assignee.start();
|
||||||
startBackgroundMovesIfNeeded();
|
startBackgroundMovesIfNeeded();
|
||||||
|
startOutdatedDataPartsLoadingTask();
|
||||||
}
|
}
|
||||||
catch (...)
|
catch (...)
|
||||||
{
|
{
|
||||||
@ -171,6 +171,8 @@ void StorageMergeTree::shutdown()
|
|||||||
if (shutdown_called.exchange(true))
|
if (shutdown_called.exchange(true))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
stopOutdatedDataPartsLoadingTask();
|
||||||
|
|
||||||
/// Unlock all waiting mutations
|
/// Unlock all waiting mutations
|
||||||
{
|
{
|
||||||
std::lock_guard lock(mutation_wait_mutex);
|
std::lock_guard lock(mutation_wait_mutex);
|
||||||
@ -1189,6 +1191,7 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign
|
|||||||
scheduled = true;
|
scheduled = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
return scheduled;
|
return scheduled;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1509,6 +1512,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont
|
|||||||
/// Asks to complete merges and does not allow them to start.
|
/// Asks to complete merges and does not allow them to start.
|
||||||
/// This protects against "revival" of data for a removed partition after completion of merge.
|
/// This protects against "revival" of data for a removed partition after completion of merge.
|
||||||
auto merge_blocker = stopMergesAndWait();
|
auto merge_blocker = stopMergesAndWait();
|
||||||
|
waitForOutdatedPartsToBeLoaded();
|
||||||
|
|
||||||
Stopwatch watch;
|
Stopwatch watch;
|
||||||
|
|
||||||
|
@ -907,6 +907,16 @@ void StorageReplicatedMergeTree::drop()
|
|||||||
dropReplica(zookeeper, zookeeper_path, replica_name, log, getSettings());
|
dropReplica(zookeeper, zookeeper_path, replica_name, log, getSettings());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Wait for loading of all outdated parts because
|
||||||
|
/// in case of zero copy recursive removal of directory
|
||||||
|
/// is not supported and table cannot be dropped.
|
||||||
|
if (canUseZeroCopyReplication())
|
||||||
|
{
|
||||||
|
/// Load remaining parts synchronously because task
|
||||||
|
/// for loading is already cancelled in shutdown().
|
||||||
|
loadOutdatedDataParts(/*is_async=*/ false);
|
||||||
|
}
|
||||||
|
|
||||||
dropAllData();
|
dropAllData();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1866,6 +1876,11 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::executeFetchShared(
|
|||||||
void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry)
|
void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry)
|
||||||
{
|
{
|
||||||
LOG_TRACE(log, "Executing DROP_RANGE {}", entry.new_part_name);
|
LOG_TRACE(log, "Executing DROP_RANGE {}", entry.new_part_name);
|
||||||
|
|
||||||
|
/// Wait for loading of outdated parts because DROP_RANGE
|
||||||
|
/// command must be applied to all parts on disk.
|
||||||
|
waitForOutdatedPartsToBeLoaded();
|
||||||
|
|
||||||
auto drop_range_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version);
|
auto drop_range_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version);
|
||||||
getContext()->getMergeList().cancelInPartition(getStorageID(), drop_range_info.partition_id, drop_range_info.max_block);
|
getContext()->getMergeList().cancelInPartition(getStorageID(), drop_range_info.partition_id, drop_range_info.max_block);
|
||||||
queue.removePartProducingOpsInRange(getZooKeeper(), drop_range_info, entry, /* fetch_entry_znode= */ {});
|
queue.removePartProducingOpsInRange(getZooKeeper(), drop_range_info, entry, /* fetch_entry_znode= */ {});
|
||||||
@ -1930,6 +1945,11 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry)
|
|||||||
LOG_DEBUG(log, "Executing log entry {} to replace parts range {} with {} parts from {}.{}",
|
LOG_DEBUG(log, "Executing log entry {} to replace parts range {} with {} parts from {}.{}",
|
||||||
entry.znode_name, entry_replace.drop_range_part_name, entry_replace.new_part_names.size(),
|
entry.znode_name, entry_replace.drop_range_part_name, entry_replace.new_part_names.size(),
|
||||||
entry_replace.from_database, entry_replace.from_table);
|
entry_replace.from_database, entry_replace.from_table);
|
||||||
|
|
||||||
|
/// Wait for loading of outdated parts because REPLACE_RANGE
|
||||||
|
/// command must be applied to all parts on disk.
|
||||||
|
waitForOutdatedPartsToBeLoaded();
|
||||||
|
|
||||||
auto metadata_snapshot = getInMemoryMetadataPtr();
|
auto metadata_snapshot = getInMemoryMetadataPtr();
|
||||||
auto storage_settings_ptr = getSettings();
|
auto storage_settings_ptr = getSettings();
|
||||||
|
|
||||||
@ -4286,6 +4306,7 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart(
|
|||||||
|
|
||||||
void StorageReplicatedMergeTree::startup()
|
void StorageReplicatedMergeTree::startup()
|
||||||
{
|
{
|
||||||
|
startOutdatedDataPartsLoadingTask();
|
||||||
if (attach_thread)
|
if (attach_thread)
|
||||||
{
|
{
|
||||||
attach_thread->start();
|
attach_thread->start();
|
||||||
@ -4402,6 +4423,7 @@ void StorageReplicatedMergeTree::shutdown()
|
|||||||
return;
|
return;
|
||||||
|
|
||||||
session_expired_callback_handler.reset();
|
session_expired_callback_handler.reset();
|
||||||
|
stopOutdatedDataPartsLoadingTask();
|
||||||
|
|
||||||
/// Cancel fetches, merges and mutations to force the queue_task to finish ASAP.
|
/// Cancel fetches, merges and mutations to force the queue_task to finish ASAP.
|
||||||
fetcher.blocker.cancelForever();
|
fetcher.blocker.cancelForever();
|
||||||
@ -5295,7 +5317,6 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper()
|
|||||||
if (!is_readonly)
|
if (!is_readonly)
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Replica must be readonly");
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Replica must be readonly");
|
||||||
|
|
||||||
|
|
||||||
if (getZooKeeper()->exists(replica_path))
|
if (getZooKeeper()->exists(replica_path))
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||||
"Replica path is present at {} - nothing to restore. "
|
"Replica path is present at {} - nothing to restore. "
|
||||||
@ -5312,6 +5333,7 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper()
|
|||||||
|
|
||||||
auto metadata_snapshot = getInMemoryMetadataPtr();
|
auto metadata_snapshot = getInMemoryMetadataPtr();
|
||||||
|
|
||||||
|
waitForOutdatedPartsToBeLoaded();
|
||||||
const DataPartsVector all_parts = getAllDataPartsVector();
|
const DataPartsVector all_parts = getAllDataPartsVector();
|
||||||
Strings active_parts_names;
|
Strings active_parts_names;
|
||||||
|
|
||||||
@ -5426,6 +5448,7 @@ void StorageReplicatedMergeTree::truncate(
|
|||||||
if (!is_leader)
|
if (!is_leader)
|
||||||
throw Exception("TRUNCATE cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER);
|
throw Exception("TRUNCATE cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER);
|
||||||
|
|
||||||
|
waitForOutdatedPartsToBeLoaded();
|
||||||
zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly();
|
zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly();
|
||||||
dropAllPartitionsImpl(zookeeper, /* detach */ false, query_context);
|
dropAllPartitionsImpl(zookeeper, /* detach */ false, query_context);
|
||||||
}
|
}
|
||||||
|
@ -38,13 +38,18 @@ class Reviews:
|
|||||||
self.approved_at = datetime.fromtimestamp(0)
|
self.approved_at = datetime.fromtimestamp(0)
|
||||||
for r in self.reviews:
|
for r in self.reviews:
|
||||||
user = r.user
|
user = r.user
|
||||||
if self._review_per_user.get(user):
|
if r.state not in self.STATES:
|
||||||
if r.state in self.STATES:
|
|
||||||
self._review_per_user[user] = r
|
|
||||||
if r.state == "APPROVED":
|
|
||||||
self.approved_at = max(r.submitted_at, self.approved_at)
|
|
||||||
continue
|
continue
|
||||||
self._review_per_user[user] = r
|
|
||||||
|
if r.state == "APPROVED":
|
||||||
|
self.approved_at = max(r.submitted_at, self.approved_at)
|
||||||
|
|
||||||
|
if not self._review_per_user.get(user):
|
||||||
|
self._review_per_user[user] = r
|
||||||
|
continue
|
||||||
|
|
||||||
|
if r.submitted_at < self._review_per_user[user].submitted_at:
|
||||||
|
self._review_per_user[user] = r
|
||||||
|
|
||||||
def is_approved(self, team: List[NamedUser]) -> bool:
|
def is_approved(self, team: List[NamedUser]) -> bool:
|
||||||
"""Checks if the PR is approved, and no changes made after the last approval"""
|
"""Checks if the PR is approved, and no changes made after the last approval"""
|
||||||
|
@ -181,7 +181,7 @@ def test_grant_all_on_table():
|
|||||||
== "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, "
|
== "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, "
|
||||||
"DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, "
|
"DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, "
|
||||||
"SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, "
|
"SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, "
|
||||||
"SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
|
"SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
|
||||||
)
|
)
|
||||||
instance.query("REVOKE ALL ON test.table FROM B", user="A")
|
instance.query("REVOKE ALL ON test.table FROM B", user="A")
|
||||||
assert instance.query("SHOW GRANTS FOR B") == ""
|
assert instance.query("SHOW GRANTS FOR B") == ""
|
||||||
|
@ -0,0 +1,9 @@
|
|||||||
|
<clickhouse>
|
||||||
|
<background_processing_pool_thread_sleep_seconds>1</background_processing_pool_thread_sleep_seconds>
|
||||||
|
<background_processing_pool_thread_sleep_seconds_random_part>0</background_processing_pool_thread_sleep_seconds_random_part>
|
||||||
|
<background_processing_pool_thread_sleep_seconds_if_nothing_to_do>0.0</background_processing_pool_thread_sleep_seconds_if_nothing_to_do>
|
||||||
|
<background_processing_pool_task_sleep_seconds_when_no_work_min>0</background_processing_pool_task_sleep_seconds_when_no_work_min>
|
||||||
|
<background_processing_pool_task_sleep_seconds_when_no_work_max>1</background_processing_pool_task_sleep_seconds_when_no_work_max>
|
||||||
|
<background_processing_pool_task_sleep_seconds_when_no_work_multiplier>1</background_processing_pool_task_sleep_seconds_when_no_work_multiplier>
|
||||||
|
<background_processing_pool_task_sleep_seconds_when_no_work_random_part>0</background_processing_pool_task_sleep_seconds_when_no_work_random_part>
|
||||||
|
</clickhouse>
|
196
tests/integration/test_merge_tree_load_parts/test.py
Normal file
196
tests/integration/test_merge_tree_load_parts/test.py
Normal file
@ -0,0 +1,196 @@
|
|||||||
|
import pytest
|
||||||
|
import helpers.client
|
||||||
|
import helpers.cluster
|
||||||
|
import time
|
||||||
|
from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk
|
||||||
|
|
||||||
|
|
||||||
|
cluster = helpers.cluster.ClickHouseCluster(__file__)
|
||||||
|
node1 = cluster.add_instance(
|
||||||
|
"node1",
|
||||||
|
main_configs=["configs/fast_background_pool.xml"],
|
||||||
|
with_zookeeper=True,
|
||||||
|
stay_alive=True,
|
||||||
|
)
|
||||||
|
node2 = cluster.add_instance(
|
||||||
|
"node2",
|
||||||
|
main_configs=["configs/fast_background_pool.xml"],
|
||||||
|
with_zookeeper=True,
|
||||||
|
stay_alive=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def started_cluster():
|
||||||
|
try:
|
||||||
|
cluster.start()
|
||||||
|
yield cluster
|
||||||
|
|
||||||
|
finally:
|
||||||
|
cluster.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_tree_load_parts(started_cluster):
|
||||||
|
node1.query(
|
||||||
|
"""
|
||||||
|
CREATE TABLE mt_load_parts (pk UInt32, id UInt32, s String)
|
||||||
|
ENGINE = MergeTree ORDER BY id PARTITION BY pk"""
|
||||||
|
)
|
||||||
|
|
||||||
|
node1.query("SYSTEM STOP MERGES mt_load_parts")
|
||||||
|
|
||||||
|
for i in range(20):
|
||||||
|
node1.query(
|
||||||
|
f"INSERT INTO mt_load_parts VALUES (44, {i}, randomPrintableASCII(10))"
|
||||||
|
)
|
||||||
|
|
||||||
|
node1.restart_clickhouse(kill=True)
|
||||||
|
for i in range(1, 21):
|
||||||
|
assert node1.contains_in_log(f"Loading Active part 44_{i}_{i}_0")
|
||||||
|
|
||||||
|
node1.query("OPTIMIZE TABLE mt_load_parts FINAL")
|
||||||
|
node1.restart_clickhouse(kill=True)
|
||||||
|
|
||||||
|
node1.query("SYSTEM WAIT LOADING PARTS mt_load_parts")
|
||||||
|
|
||||||
|
assert node1.contains_in_log("Loading Active part 44_1_20")
|
||||||
|
for i in range(1, 21):
|
||||||
|
assert not node1.contains_in_log(f"Loading Active part 44_{i}_{i}_0")
|
||||||
|
assert node1.contains_in_log(f"Loading Outdated part 44_{i}_{i}_0")
|
||||||
|
|
||||||
|
assert node1.query("SELECT count() FROM mt_load_parts") == "20\n"
|
||||||
|
|
||||||
|
assert (
|
||||||
|
node1.query(
|
||||||
|
"SELECT count() FROM system.parts WHERE table = 'mt_load_parts' AND active"
|
||||||
|
)
|
||||||
|
== "1\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
node1.query("ALTER TABLE mt_load_parts MODIFY SETTING old_parts_lifetime = 1")
|
||||||
|
node1.query("DETACH TABLE mt_load_parts")
|
||||||
|
node1.query("ATTACH TABLE mt_load_parts")
|
||||||
|
|
||||||
|
node1.query("SYSTEM WAIT LOADING PARTS mt_load_parts")
|
||||||
|
|
||||||
|
table_path = node1.query(
|
||||||
|
"SELECT data_paths[1] FROM system.tables WHERE table = 'mt_load_parts'"
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
part_dirs = node1.exec_in_container(["bash", "-c", f"ls {table_path}"], user="root")
|
||||||
|
|
||||||
|
part_dirs = list(
|
||||||
|
set(part_dirs.strip().split("\n")) - {"detached", "format_version.txt"}
|
||||||
|
)
|
||||||
|
|
||||||
|
MAX_RETRY = 10
|
||||||
|
part_dirs_ok = False
|
||||||
|
for _ in range(MAX_RETRY):
|
||||||
|
part_dirs = node1.exec_in_container(
|
||||||
|
["bash", "-c", f"ls {table_path}"], user="root"
|
||||||
|
)
|
||||||
|
part_dirs = list(
|
||||||
|
set(part_dirs.strip().split("\n")) - {"detached", "format_version.txt"}
|
||||||
|
)
|
||||||
|
part_dirs_ok = len(part_dirs) == 1 and part_dirs[0].startswith("44_1_20")
|
||||||
|
if part_dirs_ok:
|
||||||
|
break
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
assert part_dirs_ok
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_tree_load_parts_corrupted(started_cluster):
|
||||||
|
for i, node in enumerate([node1, node2]):
|
||||||
|
node.query(
|
||||||
|
f"""
|
||||||
|
CREATE TABLE mt_load_parts_2 (pk UInt32, id UInt32, s String)
|
||||||
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/mt_load_parts_2', '{i}') ORDER BY id PARTITION BY pk"""
|
||||||
|
)
|
||||||
|
|
||||||
|
"""min-max blocks in created parts: 1_1_0, 2_2_0, 1_2_1, 3_3_0, 1_3_2"""
|
||||||
|
for partition in [111, 222, 333]:
|
||||||
|
node1.query(
|
||||||
|
f"INSERT INTO mt_load_parts_2 VALUES ({partition}, 0, randomPrintableASCII(10))"
|
||||||
|
)
|
||||||
|
|
||||||
|
node1.query(
|
||||||
|
f"INSERT INTO mt_load_parts_2 VALUES ({partition}, 1, randomPrintableASCII(10))"
|
||||||
|
)
|
||||||
|
|
||||||
|
node1.query(f"OPTIMIZE TABLE mt_load_parts_2 PARTITION {partition} FINAL")
|
||||||
|
|
||||||
|
node1.query(
|
||||||
|
f"INSERT INTO mt_load_parts_2 VALUES ({partition}, 2, randomPrintableASCII(10))"
|
||||||
|
)
|
||||||
|
|
||||||
|
node1.query(f"OPTIMIZE TABLE mt_load_parts_2 PARTITION {partition} FINAL")
|
||||||
|
|
||||||
|
node2.query("SYSTEM SYNC REPLICA mt_load_parts_2", timeout=30)
|
||||||
|
|
||||||
|
def get_part_name(node, partition, min_block, max_block):
|
||||||
|
return node.query(
|
||||||
|
f"""
|
||||||
|
SELECT name FROM system.parts
|
||||||
|
WHERE table = 'mt_load_parts_2'
|
||||||
|
AND partition = '{partition}'
|
||||||
|
AND min_block_number = {min_block}
|
||||||
|
AND max_block_number = {max_block}"""
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 111, 0, 2))
|
||||||
|
corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 222, 0, 2))
|
||||||
|
corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 222, 0, 1))
|
||||||
|
corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 333, 0, 1))
|
||||||
|
corrupt_part_data_on_disk(node1, "mt_load_parts_2", get_part_name(node1, 333, 2, 2))
|
||||||
|
|
||||||
|
node1.restart_clickhouse(kill=True)
|
||||||
|
node1.query("SYSTEM WAIT LOADING PARTS mt_load_parts_2")
|
||||||
|
|
||||||
|
def check_parts_loading(node, partition, loaded, failed, skipped):
|
||||||
|
for (min_block, max_block) in loaded:
|
||||||
|
part_name = f"{partition}_{min_block}_{max_block}"
|
||||||
|
assert node.contains_in_log(f"Loading Active part {part_name}")
|
||||||
|
assert node.contains_in_log(f"Finished loading Active part {part_name}")
|
||||||
|
|
||||||
|
for (min_block, max_block) in failed:
|
||||||
|
part_name = f"{partition}_{min_block}_{max_block}"
|
||||||
|
assert node.contains_in_log(f"Loading Active part {part_name}")
|
||||||
|
assert not node.contains_in_log(f"Finished loading Active part {part_name}")
|
||||||
|
|
||||||
|
for (min_block, max_block) in skipped:
|
||||||
|
part_name = f"{partition}_{min_block}_{max_block}"
|
||||||
|
assert not node.contains_in_log(f"Loading Active part {part_name}")
|
||||||
|
assert not node.contains_in_log(f"Finished loading Active part {part_name}")
|
||||||
|
|
||||||
|
check_parts_loading(
|
||||||
|
node1, 111, loaded=[(0, 1), (2, 2)], failed=[(0, 2)], skipped=[(0, 0), (1, 1)]
|
||||||
|
)
|
||||||
|
check_parts_loading(
|
||||||
|
node1, 222, loaded=[(0, 0), (1, 1), (2, 2)], failed=[(0, 2), (0, 1)], skipped=[]
|
||||||
|
)
|
||||||
|
check_parts_loading(
|
||||||
|
node1, 333, loaded=[(0, 2)], failed=[], skipped=[(0, 0), (1, 1), (2, 2), (0, 1)]
|
||||||
|
)
|
||||||
|
|
||||||
|
node1.query("SYSTEM SYNC REPLICA mt_load_parts_2", timeout=30)
|
||||||
|
node1.query("OPTIMIZE TABLE mt_load_parts_2 FINAL")
|
||||||
|
node1.query("SYSTEM SYNC REPLICA mt_load_parts_2", timeout=30)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
node1.query(
|
||||||
|
"""
|
||||||
|
SELECT pk, count() FROM mt_load_parts_2
|
||||||
|
GROUP BY pk ORDER BY pk"""
|
||||||
|
)
|
||||||
|
== "111\t3\n222\t3\n333\t3\n"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
node1.query(
|
||||||
|
"""
|
||||||
|
SELECT partition, count()
|
||||||
|
FROM system.parts WHERE table = 'mt_load_parts_2' AND active
|
||||||
|
GROUP BY partition ORDER BY partition"""
|
||||||
|
)
|
||||||
|
== "111\t1\n222\t1\n333\t1\n"
|
||||||
|
)
|
@ -26,7 +26,7 @@ def tx(session, query):
|
|||||||
|
|
||||||
def test_rollback_unfinished_on_restart1(start_cluster):
|
def test_rollback_unfinished_on_restart1(start_cluster):
|
||||||
node.query(
|
node.query(
|
||||||
"create table mt (n int, m int) engine=MergeTree order by n partition by n % 2"
|
"create table mt (n int, m int) engine=MergeTree order by n partition by n % 2 settings remove_empty_parts = 0"
|
||||||
)
|
)
|
||||||
node.query("insert into mt values (1, 10), (2, 20)")
|
node.query("insert into mt values (1, 10), (2, 20)")
|
||||||
tid0 = "(1,1,'00000000-0000-0000-0000-000000000000')"
|
tid0 = "(1,1,'00000000-0000-0000-0000-000000000000')"
|
||||||
@ -82,6 +82,7 @@ def test_rollback_unfinished_on_restart1(start_cluster):
|
|||||||
).strip()
|
).strip()
|
||||||
|
|
||||||
node.restart_clickhouse(kill=True)
|
node.restart_clickhouse(kill=True)
|
||||||
|
node.query("SYSTEM WAIT LOADING PARTS mt")
|
||||||
|
|
||||||
assert (
|
assert (
|
||||||
node.query("select *, _part from mt order by n")
|
node.query("select *, _part from mt order by n")
|
||||||
@ -116,7 +117,7 @@ def test_rollback_unfinished_on_restart1(start_cluster):
|
|||||||
|
|
||||||
def test_rollback_unfinished_on_restart2(start_cluster):
|
def test_rollback_unfinished_on_restart2(start_cluster):
|
||||||
node.query(
|
node.query(
|
||||||
"create table mt2 (n int, m int) engine=MergeTree order by n partition by n % 2"
|
"create table mt2 (n int, m int) engine=MergeTree order by n partition by n % 2 settings remove_empty_parts = 0"
|
||||||
)
|
)
|
||||||
node.query("insert into mt2 values (1, 10), (2, 20)")
|
node.query("insert into mt2 values (1, 10), (2, 20)")
|
||||||
tid0 = "(1,1,'00000000-0000-0000-0000-000000000000')"
|
tid0 = "(1,1,'00000000-0000-0000-0000-000000000000')"
|
||||||
@ -171,6 +172,7 @@ def test_rollback_unfinished_on_restart2(start_cluster):
|
|||||||
).strip()
|
).strip()
|
||||||
|
|
||||||
node.restart_clickhouse(kill=True)
|
node.restart_clickhouse(kill=True)
|
||||||
|
node.query("SYSTEM WAIT LOADING PARTS mt2")
|
||||||
|
|
||||||
assert (
|
assert (
|
||||||
node.query("select *, _part from mt2 order by n")
|
node.query("select *, _part from mt2 order by n")
|
||||||
|
@ -13,6 +13,7 @@
|
|||||||
<value>Delta</value>
|
<value>Delta</value>
|
||||||
<value>T64</value>
|
<value>T64</value>
|
||||||
<value>DoubleDelta</value>
|
<value>DoubleDelta</value>
|
||||||
|
<value>Gorilla</value>
|
||||||
</values>
|
</values>
|
||||||
</substitution>
|
</substitution>
|
||||||
<substitution>
|
<substitution>
|
||||||
|
@ -13,6 +13,7 @@
|
|||||||
<value>Delta</value>
|
<value>Delta</value>
|
||||||
<value>T64</value>
|
<value>T64</value>
|
||||||
<value>DoubleDelta</value>
|
<value>DoubleDelta</value>
|
||||||
|
<value>Gorilla</value>
|
||||||
</values>
|
</values>
|
||||||
</substitution>
|
</substitution>
|
||||||
<substitution>
|
<substitution>
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
10 [1,1,1,1,10,10,10,10,100,100,100]
|
10 [1,1,1,1,10,10,10,10,100,100,100]
|
||||||
|
10 [1,1,2,4,7,10,35,61,87,100,100]
|
||||||
100 100
|
100 100
|
||||||
61 61
|
61 61
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
SELECT quantileExactWeighted(0.5)(x, 1) AS q5, quantilesExactWeighted(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1)(x, 1) AS qs FROM (SELECT arrayJoin([1, 1, 1, 10, 10, 10, 10, 100, 100, 100]) AS x);
|
SELECT quantileExactWeighted(0.5)(x, 1) AS q5, quantilesExactWeighted(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1)(x, 1) AS qs FROM (SELECT arrayJoin([1, 1, 1, 10, 10, 10, 10, 100, 100, 100]) AS x);
|
||||||
|
SELECT quantileInterpolatedWeighted(0.5)(x, 1) AS q5, quantilesInterpolatedWeighted(0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1)(x, 1) AS qs FROM (SELECT arrayJoin([1, 1, 1, 10, 10, 10, 10, 100, 100, 100]) AS x);
|
||||||
SELECT quantileExact(0)(x), quantileTiming(0)(x) FROM (SELECT number + 100 AS x FROM system.numbers LIMIT 10000);
|
SELECT quantileExact(0)(x), quantileTiming(0)(x) FROM (SELECT number + 100 AS x FROM system.numbers LIMIT 10000);
|
||||||
SELECT quantileExact(x), quantileTiming(x) FROM (SELECT number % 123 AS x FROM system.numbers LIMIT 10000);
|
SELECT quantileExact(x), quantileTiming(x) FROM (SELECT number % 123 AS x FROM system.numbers LIMIT 10000);
|
||||||
|
@ -0,0 +1 @@
|
|||||||
|
[(-1,-1,2)]
|
6
tests/queries/0_stateless/00647_histogram_negative.sql
Normal file
6
tests/queries/0_stateless/00647_histogram_negative.sql
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
drop table if exists histogram;
|
||||||
|
create table histogram(num Int64) engine=TinyLog;
|
||||||
|
insert into histogram values(-1);
|
||||||
|
insert into histogram values(-1);
|
||||||
|
select histogram(2)(num) from histogram;
|
||||||
|
drop table if exists histogram;
|
@ -6,6 +6,8 @@
|
|||||||
['2016-06-15 23:00:00']
|
['2016-06-15 23:00:00']
|
||||||
2016-06-15 23:00:00
|
2016-06-15 23:00:00
|
||||||
['2016-06-15 23:00:00']
|
['2016-06-15 23:00:00']
|
||||||
|
2016-06-15 23:00:00
|
||||||
|
['2016-06-15 23:00:00']
|
||||||
30000
|
30000
|
||||||
[30000]
|
[30000]
|
||||||
30000
|
30000
|
||||||
|
@ -15,6 +15,9 @@ SELECT quantilesExact(0.2)(d) FROM datetime;
|
|||||||
SELECT quantileExactWeighted(0.2)(d, 1) FROM datetime;
|
SELECT quantileExactWeighted(0.2)(d, 1) FROM datetime;
|
||||||
SELECT quantilesExactWeighted(0.2)(d, 1) FROM datetime;
|
SELECT quantilesExactWeighted(0.2)(d, 1) FROM datetime;
|
||||||
|
|
||||||
|
SELECT quantileInterpolatedWeighted(0.2)(d, 1) FROM datetime;
|
||||||
|
SELECT quantilesInterpolatedWeighted(0.2)(d, 1) FROM datetime;
|
||||||
|
|
||||||
SELECT quantileTiming(0.2)(d) FROM datetime;
|
SELECT quantileTiming(0.2)(d) FROM datetime;
|
||||||
SELECT quantilesTiming(0.2)(d) FROM datetime;
|
SELECT quantilesTiming(0.2)(d) FROM datetime;
|
||||||
|
|
||||||
|
@ -121,6 +121,7 @@ SYSTEM DROP REPLICA ['DROP REPLICA'] TABLE SYSTEM
|
|||||||
SYSTEM SYNC REPLICA ['SYNC REPLICA'] TABLE SYSTEM
|
SYSTEM SYNC REPLICA ['SYNC REPLICA'] TABLE SYSTEM
|
||||||
SYSTEM RESTART REPLICA ['RESTART REPLICA'] TABLE SYSTEM
|
SYSTEM RESTART REPLICA ['RESTART REPLICA'] TABLE SYSTEM
|
||||||
SYSTEM RESTORE REPLICA ['RESTORE REPLICA'] TABLE SYSTEM
|
SYSTEM RESTORE REPLICA ['RESTORE REPLICA'] TABLE SYSTEM
|
||||||
|
SYSTEM WAIT LOADING PARTS ['WAIT LOADING PARTS'] TABLE SYSTEM
|
||||||
SYSTEM SYNC DATABASE REPLICA ['SYNC DATABASE REPLICA'] DATABASE SYSTEM
|
SYSTEM SYNC DATABASE REPLICA ['SYNC DATABASE REPLICA'] DATABASE SYSTEM
|
||||||
SYSTEM SYNC TRANSACTION LOG ['SYNC TRANSACTION LOG'] GLOBAL SYSTEM
|
SYSTEM SYNC TRANSACTION LOG ['SYNC TRANSACTION LOG'] GLOBAL SYSTEM
|
||||||
SYSTEM FLUSH DISTRIBUTED ['FLUSH DISTRIBUTED'] TABLE SYSTEM FLUSH
|
SYSTEM FLUSH DISTRIBUTED ['FLUSH DISTRIBUTED'] TABLE SYSTEM FLUSH
|
||||||
|
@ -6,8 +6,6 @@ CREATE TABLE default.codecs5\n(\n `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE =
|
|||||||
CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
CREATE TABLE default.codecs9\n(\n `a` Float64 CODEC(Delta(8), FPC(12))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
|
||||||
CREATE TABLE default.codecs10\n(\n `a` Float64 CODEC(DoubleDelta, Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
|
||||||
CREATE TABLE default.codecs1\n(\n `a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
CREATE TABLE default.codecs1\n(\n `a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
CREATE TABLE default.codecs2\n(\n `a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
CREATE TABLE default.codecs2\n(\n `a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
CREATE TABLE default.codecs3\n(\n `a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
CREATE TABLE default.codecs3\n(\n `a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
@ -16,5 +14,3 @@ CREATE TABLE default.codecs5\n(\n `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE =
|
|||||||
CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
CREATE TABLE default.codecs9\n(\n `a` Float64 CODEC(Delta(8), FPC(12))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
|
||||||
CREATE TABLE default.codecs10\n(\n `a` Float64 CODEC(DoubleDelta, Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
|
||||||
|
@ -11,28 +11,24 @@ CREATE TABLE codecs
|
|||||||
c Float32 CODEC(Gorilla),
|
c Float32 CODEC(Gorilla),
|
||||||
d UInt8 CODEC(Delta, LZ4),
|
d UInt8 CODEC(Delta, LZ4),
|
||||||
e Float64 CODEC(Gorilla, ZSTD),
|
e Float64 CODEC(Gorilla, ZSTD),
|
||||||
f UInt64 CODEC(Delta, Delta, T64),
|
f UInt32 CODEC(Delta, Delta, Gorilla),
|
||||||
g DateTime CODEC(DoubleDelta),
|
g DateTime CODEC(DoubleDelta),
|
||||||
h DateTime64 CODEC(DoubleDelta, LZ4),
|
h DateTime64 CODEC(DoubleDelta, LZ4),
|
||||||
i String CODEC(NONE),
|
i String CODEC(NONE)
|
||||||
j Float64 (Gorilla, Delta),
|
|
||||||
k Float32 (FPC, DoubleDelta)
|
|
||||||
) ENGINE = MergeTree ORDER BY tuple();
|
) ENGINE = MergeTree ORDER BY tuple();
|
||||||
|
|
||||||
DROP TABLE codecs;
|
DROP TABLE codecs;
|
||||||
|
|
||||||
-- test what should not work
|
-- test what should not work
|
||||||
|
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(NONE, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
CREATE TABLE codecs (a UInt8 CODEC(NONE, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(NONE, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
CREATE TABLE codecs (a UInt8 CODEC(NONE, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(LZ4, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
CREATE TABLE codecs (a UInt8 CODEC(LZ4, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(LZ4, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
CREATE TABLE codecs (a UInt8 CODEC(LZ4, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 }
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(Delta, FPC)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
|
||||||
CREATE TABLE codecs (a UInt8 CODEC(DoubleDelta, Gorilla)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS }
|
|
||||||
|
|
||||||
-- test that sanity check is not performed in ATTACH query
|
-- test that sanity check is not performed in ATTACH query
|
||||||
|
|
||||||
@ -44,8 +40,6 @@ DROP TABLE IF EXISTS codecs5;
|
|||||||
DROP TABLE IF EXISTS codecs6;
|
DROP TABLE IF EXISTS codecs6;
|
||||||
DROP TABLE IF EXISTS codecs7;
|
DROP TABLE IF EXISTS codecs7;
|
||||||
DROP TABLE IF EXISTS codecs8;
|
DROP TABLE IF EXISTS codecs8;
|
||||||
DROP TABLE IF EXISTS codecs9;
|
|
||||||
DROP TABLE IF EXISTS codecs10;
|
|
||||||
|
|
||||||
SET allow_suspicious_codecs = 1;
|
SET allow_suspicious_codecs = 1;
|
||||||
|
|
||||||
@ -57,8 +51,6 @@ CREATE TABLE codecs5 (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tupl
|
|||||||
CREATE TABLE codecs6 (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple();
|
CREATE TABLE codecs6 (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple();
|
||||||
CREATE TABLE codecs7 (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple();
|
CREATE TABLE codecs7 (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple();
|
||||||
CREATE TABLE codecs8 (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple();
|
CREATE TABLE codecs8 (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple();
|
||||||
CREATE TABLE codecs9 (a Float64 CODEC(Delta, FPC)) ENGINE = MergeTree ORDER BY tuple();
|
|
||||||
CREATE TABLE codecs10 (a Float64 CODEC(DoubleDelta, Gorilla)) ENGINE = MergeTree ORDER BY tuple();
|
|
||||||
|
|
||||||
SET allow_suspicious_codecs = 0;
|
SET allow_suspicious_codecs = 0;
|
||||||
|
|
||||||
@ -70,8 +62,6 @@ SHOW CREATE TABLE codecs5;
|
|||||||
SHOW CREATE TABLE codecs6;
|
SHOW CREATE TABLE codecs6;
|
||||||
SHOW CREATE TABLE codecs7;
|
SHOW CREATE TABLE codecs7;
|
||||||
SHOW CREATE TABLE codecs8;
|
SHOW CREATE TABLE codecs8;
|
||||||
SHOW CREATE TABLE codecs9;
|
|
||||||
SHOW CREATE TABLE codecs10;
|
|
||||||
|
|
||||||
DETACH TABLE codecs1;
|
DETACH TABLE codecs1;
|
||||||
DETACH TABLE codecs2;
|
DETACH TABLE codecs2;
|
||||||
@ -81,8 +71,6 @@ DETACH TABLE codecs5;
|
|||||||
DETACH TABLE codecs6;
|
DETACH TABLE codecs6;
|
||||||
DETACH TABLE codecs7;
|
DETACH TABLE codecs7;
|
||||||
DETACH TABLE codecs8;
|
DETACH TABLE codecs8;
|
||||||
DETACH TABLE codecs9;
|
|
||||||
DETACH TABLE codecs10;
|
|
||||||
|
|
||||||
ATTACH TABLE codecs1;
|
ATTACH TABLE codecs1;
|
||||||
ATTACH TABLE codecs2;
|
ATTACH TABLE codecs2;
|
||||||
@ -92,8 +80,6 @@ ATTACH TABLE codecs5;
|
|||||||
ATTACH TABLE codecs6;
|
ATTACH TABLE codecs6;
|
||||||
ATTACH TABLE codecs7;
|
ATTACH TABLE codecs7;
|
||||||
ATTACH TABLE codecs8;
|
ATTACH TABLE codecs8;
|
||||||
ATTACH TABLE codecs9;
|
|
||||||
ATTACH TABLE codecs10;
|
|
||||||
|
|
||||||
SHOW CREATE TABLE codecs1;
|
SHOW CREATE TABLE codecs1;
|
||||||
SHOW CREATE TABLE codecs2;
|
SHOW CREATE TABLE codecs2;
|
||||||
@ -103,8 +89,6 @@ SHOW CREATE TABLE codecs5;
|
|||||||
SHOW CREATE TABLE codecs6;
|
SHOW CREATE TABLE codecs6;
|
||||||
SHOW CREATE TABLE codecs7;
|
SHOW CREATE TABLE codecs7;
|
||||||
SHOW CREATE TABLE codecs8;
|
SHOW CREATE TABLE codecs8;
|
||||||
SHOW CREATE TABLE codecs9;
|
|
||||||
SHOW CREATE TABLE codecs10;
|
|
||||||
|
|
||||||
SELECT * FROM codecs1;
|
SELECT * FROM codecs1;
|
||||||
SELECT * FROM codecs2;
|
SELECT * FROM codecs2;
|
||||||
@ -114,8 +98,6 @@ SELECT * FROM codecs5;
|
|||||||
SELECT * FROM codecs6;
|
SELECT * FROM codecs6;
|
||||||
SELECT * FROM codecs7;
|
SELECT * FROM codecs7;
|
||||||
SELECT * FROM codecs8;
|
SELECT * FROM codecs8;
|
||||||
SELECT * FROM codecs9;
|
|
||||||
SELECT * FROM codecs10;
|
|
||||||
|
|
||||||
DROP TABLE codecs1;
|
DROP TABLE codecs1;
|
||||||
DROP TABLE codecs2;
|
DROP TABLE codecs2;
|
||||||
@ -125,5 +107,3 @@ DROP TABLE codecs5;
|
|||||||
DROP TABLE codecs6;
|
DROP TABLE codecs6;
|
||||||
DROP TABLE codecs7;
|
DROP TABLE codecs7;
|
||||||
DROP TABLE codecs8;
|
DROP TABLE codecs8;
|
||||||
DROP TABLE codecs9;
|
|
||||||
DROP TABLE codecs10;
|
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
<Warning> DistributedSink: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done.
|
<Warning> DistributedSink: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done.
|
||||||
<Warning> DistributedSink: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done.
|
<Warning> DistributedSink: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done.
|
||||||
<Warning> default.dist_01683.DirectoryMonitor: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done
|
<Warning> default.dist_01683.DirectoryMonitor.default: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done
|
||||||
<Warning> default.dist_01683.DirectoryMonitor: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done
|
<Warning> default.dist_01683.DirectoryMonitor.default: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done
|
||||||
1
|
1
|
||||||
1
|
1
|
||||||
2
|
2
|
||||||
|
@ -1 +1,3 @@
|
|||||||
|
-- Tags: no-ordinary-database
|
||||||
|
|
||||||
create table test_table engine MergeTree order by a as select a_table.a, b_table.b_arr from (select arrayJoin(range(10000)) as a) a_table cross join (select range(10000) as b_arr) b_table settings max_memory_usage = 1; -- { serverError MEMORY_LIMIT_EXCEEDED }
|
create table test_table engine MergeTree order by a as select a_table.a, b_table.b_arr from (select arrayJoin(range(10000)) as a) a_table cross join (select range(10000) as b_arr) b_table settings max_memory_usage = 1; -- { serverError MEMORY_LIMIT_EXCEEDED }
|
||||||
|
@ -286,7 +286,7 @@ CREATE TABLE system.grants
|
|||||||
(
|
(
|
||||||
`user_name` Nullable(String),
|
`user_name` Nullable(String),
|
||||||
`role_name` Nullable(String),
|
`role_name` Nullable(String),
|
||||||
`access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM SYNC DATABASE REPLICA' = 123, 'SYSTEM SYNC TRANSACTION LOG' = 124, 'SYSTEM FLUSH DISTRIBUTED' = 125, 'SYSTEM FLUSH LOGS' = 126, 'SYSTEM FLUSH' = 127, 'SYSTEM THREAD FUZZER' = 128, 'SYSTEM UNFREEZE' = 129, 'SYSTEM' = 130, 'dictGet' = 131, 'addressToLine' = 132, 'addressToLineWithInlines' = 133, 'addressToSymbol' = 134, 'demangle' = 135, 'INTROSPECTION' = 136, 'FILE' = 137, 'URL' = 138, 'REMOTE' = 139, 'MONGO' = 140, 'MEILISEARCH' = 141, 'MYSQL' = 142, 'POSTGRES' = 143, 'SQLITE' = 144, 'ODBC' = 145, 'JDBC' = 146, 'HDFS' = 147, 'S3' = 148, 'HIVE' = 149, 'SOURCES' = 150, 'CLUSTER' = 151, 'ALL' = 152, 'NONE' = 153),
|
`access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM WAIT LOADING PARTS' = 123, 'SYSTEM SYNC DATABASE REPLICA' = 124, 'SYSTEM SYNC TRANSACTION LOG' = 125, 'SYSTEM FLUSH DISTRIBUTED' = 126, 'SYSTEM FLUSH LOGS' = 127, 'SYSTEM FLUSH' = 128, 'SYSTEM THREAD FUZZER' = 129, 'SYSTEM UNFREEZE' = 130, 'SYSTEM' = 131, 'dictGet' = 132, 'addressToLine' = 133, 'addressToLineWithInlines' = 134, 'addressToSymbol' = 135, 'demangle' = 136, 'INTROSPECTION' = 137, 'FILE' = 138, 'URL' = 139, 'REMOTE' = 140, 'MONGO' = 141, 'MEILISEARCH' = 142, 'MYSQL' = 143, 'POSTGRES' = 144, 'SQLITE' = 145, 'ODBC' = 146, 'JDBC' = 147, 'HDFS' = 148, 'S3' = 149, 'HIVE' = 150, 'SOURCES' = 151, 'CLUSTER' = 152, 'ALL' = 153, 'NONE' = 154),
|
||||||
`database` Nullable(String),
|
`database` Nullable(String),
|
||||||
`table` Nullable(String),
|
`table` Nullable(String),
|
||||||
`column` Nullable(String),
|
`column` Nullable(String),
|
||||||
@ -567,10 +567,10 @@ ENGINE = SystemPartsColumns
|
|||||||
COMMENT 'SYSTEM TABLE is built on the fly.'
|
COMMENT 'SYSTEM TABLE is built on the fly.'
|
||||||
CREATE TABLE system.privileges
|
CREATE TABLE system.privileges
|
||||||
(
|
(
|
||||||
`privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM SYNC DATABASE REPLICA' = 123, 'SYSTEM SYNC TRANSACTION LOG' = 124, 'SYSTEM FLUSH DISTRIBUTED' = 125, 'SYSTEM FLUSH LOGS' = 126, 'SYSTEM FLUSH' = 127, 'SYSTEM THREAD FUZZER' = 128, 'SYSTEM UNFREEZE' = 129, 'SYSTEM' = 130, 'dictGet' = 131, 'addressToLine' = 132, 'addressToLineWithInlines' = 133, 'addressToSymbol' = 134, 'demangle' = 135, 'INTROSPECTION' = 136, 'FILE' = 137, 'URL' = 138, 'REMOTE' = 139, 'MONGO' = 140, 'MEILISEARCH' = 141, 'MYSQL' = 142, 'POSTGRES' = 143, 'SQLITE' = 144, 'ODBC' = 145, 'JDBC' = 146, 'HDFS' = 147, 'S3' = 148, 'HIVE' = 149, 'SOURCES' = 150, 'CLUSTER' = 151, 'ALL' = 152, 'NONE' = 153),
|
`privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM WAIT LOADING PARTS' = 123, 'SYSTEM SYNC DATABASE REPLICA' = 124, 'SYSTEM SYNC TRANSACTION LOG' = 125, 'SYSTEM FLUSH DISTRIBUTED' = 126, 'SYSTEM FLUSH LOGS' = 127, 'SYSTEM FLUSH' = 128, 'SYSTEM THREAD FUZZER' = 129, 'SYSTEM UNFREEZE' = 130, 'SYSTEM' = 131, 'dictGet' = 132, 'addressToLine' = 133, 'addressToLineWithInlines' = 134, 'addressToSymbol' = 135, 'demangle' = 136, 'INTROSPECTION' = 137, 'FILE' = 138, 'URL' = 139, 'REMOTE' = 140, 'MONGO' = 141, 'MEILISEARCH' = 142, 'MYSQL' = 143, 'POSTGRES' = 144, 'SQLITE' = 145, 'ODBC' = 146, 'JDBC' = 147, 'HDFS' = 148, 'S3' = 149, 'HIVE' = 150, 'SOURCES' = 151, 'CLUSTER' = 152, 'ALL' = 153, 'NONE' = 154),
|
||||||
`aliases` Array(String),
|
`aliases` Array(String),
|
||||||
`level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)),
|
`level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)),
|
||||||
`parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM SYNC DATABASE REPLICA' = 123, 'SYSTEM SYNC TRANSACTION LOG' = 124, 'SYSTEM FLUSH DISTRIBUTED' = 125, 'SYSTEM FLUSH LOGS' = 126, 'SYSTEM FLUSH' = 127, 'SYSTEM THREAD FUZZER' = 128, 'SYSTEM UNFREEZE' = 129, 'SYSTEM' = 130, 'dictGet' = 131, 'addressToLine' = 132, 'addressToLineWithInlines' = 133, 'addressToSymbol' = 134, 'demangle' = 135, 'INTROSPECTION' = 136, 'FILE' = 137, 'URL' = 138, 'REMOTE' = 139, 'MONGO' = 140, 'MEILISEARCH' = 141, 'MYSQL' = 142, 'POSTGRES' = 143, 'SQLITE' = 144, 'ODBC' = 145, 'JDBC' = 146, 'HDFS' = 147, 'S3' = 148, 'HIVE' = 149, 'SOURCES' = 150, 'CLUSTER' = 151, 'ALL' = 152, 'NONE' = 153))
|
`parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'ACCESS MANAGEMENT' = 92, 'SYSTEM SHUTDOWN' = 93, 'SYSTEM DROP DNS CACHE' = 94, 'SYSTEM DROP MARK CACHE' = 95, 'SYSTEM DROP UNCOMPRESSED CACHE' = 96, 'SYSTEM DROP MMAP CACHE' = 97, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 98, 'SYSTEM DROP FILESYSTEM CACHE' = 99, 'SYSTEM DROP SCHEMA CACHE' = 100, 'SYSTEM DROP CACHE' = 101, 'SYSTEM RELOAD CONFIG' = 102, 'SYSTEM RELOAD USERS' = 103, 'SYSTEM RELOAD SYMBOLS' = 104, 'SYSTEM RELOAD DICTIONARY' = 105, 'SYSTEM RELOAD MODEL' = 106, 'SYSTEM RELOAD FUNCTION' = 107, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 108, 'SYSTEM RELOAD' = 109, 'SYSTEM RESTART DISK' = 110, 'SYSTEM MERGES' = 111, 'SYSTEM TTL MERGES' = 112, 'SYSTEM FETCHES' = 113, 'SYSTEM MOVES' = 114, 'SYSTEM DISTRIBUTED SENDS' = 115, 'SYSTEM REPLICATED SENDS' = 116, 'SYSTEM SENDS' = 117, 'SYSTEM REPLICATION QUEUES' = 118, 'SYSTEM DROP REPLICA' = 119, 'SYSTEM SYNC REPLICA' = 120, 'SYSTEM RESTART REPLICA' = 121, 'SYSTEM RESTORE REPLICA' = 122, 'SYSTEM WAIT LOADING PARTS' = 123, 'SYSTEM SYNC DATABASE REPLICA' = 124, 'SYSTEM SYNC TRANSACTION LOG' = 125, 'SYSTEM FLUSH DISTRIBUTED' = 126, 'SYSTEM FLUSH LOGS' = 127, 'SYSTEM FLUSH' = 128, 'SYSTEM THREAD FUZZER' = 129, 'SYSTEM UNFREEZE' = 130, 'SYSTEM' = 131, 'dictGet' = 132, 'addressToLine' = 133, 'addressToLineWithInlines' = 134, 'addressToSymbol' = 135, 'demangle' = 136, 'INTROSPECTION' = 137, 'FILE' = 138, 'URL' = 139, 'REMOTE' = 140, 'MONGO' = 141, 'MEILISEARCH' = 142, 'MYSQL' = 143, 'POSTGRES' = 144, 'SQLITE' = 145, 'ODBC' = 146, 'JDBC' = 147, 'HDFS' = 148, 'S3' = 149, 'HIVE' = 150, 'SOURCES' = 151, 'CLUSTER' = 152, 'ALL' = 153, 'NONE' = 154))
|
||||||
)
|
)
|
||||||
ENGINE = SystemPrivileges
|
ENGINE = SystemPrivileges
|
||||||
COMMENT 'SYSTEM TABLE is built on the fly.'
|
COMMENT 'SYSTEM TABLE is built on the fly.'
|
||||||
|
@ -0,0 +1,12 @@
|
|||||||
|
quantileInterpolatedWeighted
|
||||||
|
0 0 0 Decimal(38, 8)
|
||||||
|
-25.5 -8.49999999 -5.1 Decimal(38, 8)
|
||||||
|
0 0 0
|
||||||
|
9.7 3.23333333 1.94
|
||||||
|
19.9 6.63333332 3.98
|
||||||
|
30.1 10.03333333 6.02
|
||||||
|
40.3 13.43333332 8.06
|
||||||
|
50 16.66666666 10
|
||||||
|
[-50,-40.4,-30.3,-20.2,-10.1,0,10.1,20.2,30.3,40.4,50]
|
||||||
|
[-16.66666666,-13.46666666,-10.09999999,-6.73333332,-3.36666666,0,3.36666666,6.73333332,10.09999999,13.46666666,16.66666666]
|
||||||
|
[-10,-8.08,-6.06,-4.04,-2.02,0,2.02,4.04,6.06,8.08,10]
|
@ -0,0 +1,27 @@
|
|||||||
|
DROP TABLE IF EXISTS decimal;
|
||||||
|
|
||||||
|
CREATE TABLE decimal
|
||||||
|
(
|
||||||
|
a Decimal32(4),
|
||||||
|
b Decimal64(8),
|
||||||
|
c Decimal128(8)
|
||||||
|
) ENGINE = Memory;
|
||||||
|
|
||||||
|
INSERT INTO decimal (a, b, c)
|
||||||
|
SELECT toDecimal32(number - 50, 4), toDecimal64(number - 50, 8) / 3, toDecimal128(number - 50, 8) / 5
|
||||||
|
FROM system.numbers LIMIT 101;
|
||||||
|
|
||||||
|
SELECT 'quantileInterpolatedWeighted';
|
||||||
|
SELECT medianInterpolatedWeighted(a, 1), medianInterpolatedWeighted(b, 2), medianInterpolatedWeighted(c, 3) as x, toTypeName(x) FROM decimal;
|
||||||
|
SELECT quantileInterpolatedWeighted(a, 1), quantileInterpolatedWeighted(b, 2), quantileInterpolatedWeighted(c, 3) as x, toTypeName(x) FROM decimal WHERE a < 0;
|
||||||
|
SELECT quantileInterpolatedWeighted(0.0)(a, 1), quantileInterpolatedWeighted(0.0)(b, 2), quantileInterpolatedWeighted(0.0)(c, 3) FROM decimal WHERE a >= 0;
|
||||||
|
SELECT quantileInterpolatedWeighted(0.2)(a, 1), quantileInterpolatedWeighted(0.2)(b, 2), quantileInterpolatedWeighted(0.2)(c, 3) FROM decimal WHERE a >= 0;
|
||||||
|
SELECT quantileInterpolatedWeighted(0.4)(a, 1), quantileInterpolatedWeighted(0.4)(b, 2), quantileInterpolatedWeighted(0.4)(c, 3) FROM decimal WHERE a >= 0;
|
||||||
|
SELECT quantileInterpolatedWeighted(0.6)(a, 1), quantileInterpolatedWeighted(0.6)(b, 2), quantileInterpolatedWeighted(0.6)(c, 3) FROM decimal WHERE a >= 0;
|
||||||
|
SELECT quantileInterpolatedWeighted(0.8)(a, 1), quantileInterpolatedWeighted(0.8)(b, 2), quantileInterpolatedWeighted(0.8)(c, 3) FROM decimal WHERE a >= 0;
|
||||||
|
SELECT quantileInterpolatedWeighted(1.0)(a, 1), quantileInterpolatedWeighted(1.0)(b, 2), quantileInterpolatedWeighted(1.0)(c, 3) FROM decimal WHERE a >= 0;
|
||||||
|
SELECT quantilesInterpolatedWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(a, 1) FROM decimal;
|
||||||
|
SELECT quantilesInterpolatedWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(b, 2) FROM decimal;
|
||||||
|
SELECT quantilesInterpolatedWeighted(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(c, 3) FROM decimal;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS decimal;
|
@ -0,0 +1,2 @@
|
|||||||
|
CREATE TABLE default.dep\n(\n `id` Int32,\n `country` LowCardinality(String),\n `purchase_location` UInt16 MATERIALIZED if(id IN joinGet(\'default.id_join\', \'location\', \'CLICK\'), 123, 456)\n)\nENGINE = ReplicatedMergeTree(\'/test/02433/default/dep\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
||||||
|
CREATE TABLE default.dep2\n(\n `id` Int32,\n `country` LowCardinality(String),\n `purchase_location` UInt16 MATERIALIZED if(id IN joinGet(\'default.id_join\', \'location\', \'CLICK\'), 123, 456)\n)\nENGINE = ReplicatedMergeTree(\'/test/02433/default/dep\', \'2\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192
|
@ -0,0 +1,37 @@
|
|||||||
|
DROP TABLE IF EXISTS dep;
|
||||||
|
DROP TABLE IF EXISTS dep2;
|
||||||
|
DROP TABLE IF EXISTS id_join;
|
||||||
|
|
||||||
|
CREATE TABLE id_join (`country` String, `location` Array(Int32)) ENGINE = Join(ANY, LEFT, country);
|
||||||
|
|
||||||
|
INSERT INTO id_join values ('CLICK', [1234]);
|
||||||
|
|
||||||
|
CREATE TABLE dep
|
||||||
|
(
|
||||||
|
`id` Int32,
|
||||||
|
`country` LowCardinality(String),
|
||||||
|
`purchase_location` UInt16 MATERIALIZED if(id IN joinGet(concat(currentDatabase(), '.id_join'), 'location', 'CLICK'), 123, 456)
|
||||||
|
)
|
||||||
|
ENGINE = ReplicatedMergeTree('/test/02433/{database}/dep', '1') ORDER BY tuple();
|
||||||
|
|
||||||
|
SHOW CREATE TABLE dep;
|
||||||
|
|
||||||
|
TRUNCATE TABLE id_join;
|
||||||
|
|
||||||
|
CREATE TABLE dep2
|
||||||
|
(
|
||||||
|
`id` Int32,
|
||||||
|
`country` LowCardinality(String),
|
||||||
|
`purchase_location` UInt16 MATERIALIZED if(id IN joinGet(concat(currentDatabase(), '.id_join'), 'location', 'CLICK'), 123, 456)
|
||||||
|
)
|
||||||
|
ENGINE = ReplicatedMergeTree('/test/02433/{database}/dep', '2') ORDER BY tuple();
|
||||||
|
|
||||||
|
SHOW CREATE TABLE dep2;
|
||||||
|
|
||||||
|
-- Ensure that a table name cannot be passed to IN as string literal
|
||||||
|
create table test (n int, m default n in 'default.table_name') engine=Memory; -- { serverError TYPE_MISMATCH }
|
||||||
|
create table test (n int, m default in(n, 'default.table_name')) engine=Memory; -- { serverError TYPE_MISMATCH }
|
||||||
|
|
||||||
|
DROP TABLE dep;
|
||||||
|
DROP TABLE dep2;
|
||||||
|
DROP TABLE id_join;
|
@ -0,0 +1 @@
|
|||||||
|
1
|
54
tests/queries/0_stateless/02482_load_parts_refcounts.sh
Executable file
54
tests/queries/0_stateless/02482_load_parts_refcounts.sh
Executable file
@ -0,0 +1,54 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Tags: zookeeper
|
||||||
|
|
||||||
|
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||||
|
# shellcheck source=../shell_config.sh
|
||||||
|
. "$CURDIR"/../shell_config.sh
|
||||||
|
|
||||||
|
function query_with_retry
|
||||||
|
{
|
||||||
|
retry=0
|
||||||
|
until [ $retry -ge 5 ]
|
||||||
|
do
|
||||||
|
result=$($CLICKHOUSE_CLIENT $2 --query="$1" 2>&1)
|
||||||
|
if [ "$?" == 0 ]; then
|
||||||
|
echo -n "$result"
|
||||||
|
return
|
||||||
|
else
|
||||||
|
retry=$(($retry + 1))
|
||||||
|
sleep 3
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "Query '$1' failed with '$result'"
|
||||||
|
}
|
||||||
|
|
||||||
|
$CLICKHOUSE_CLIENT -n --query "
|
||||||
|
DROP TABLE IF EXISTS load_parts_refcounts SYNC;
|
||||||
|
|
||||||
|
CREATE TABLE load_parts_refcounts (id UInt32)
|
||||||
|
ENGINE = ReplicatedMergeTree('/test/02482_load_parts_refcounts/{database}/{table}', '1')
|
||||||
|
ORDER BY id;
|
||||||
|
|
||||||
|
SYSTEM STOP MERGES load_parts_refcounts;
|
||||||
|
|
||||||
|
INSERT INTO load_parts_refcounts VALUES (1);
|
||||||
|
INSERT INTO load_parts_refcounts VALUES (2);
|
||||||
|
INSERT INTO load_parts_refcounts VALUES (3);
|
||||||
|
|
||||||
|
SYSTEM START MERGES load_parts_refcounts;
|
||||||
|
"
|
||||||
|
|
||||||
|
query_with_retry "OPTIMIZE TABLE load_parts_refcounts FINAL SETTINGS optimize_throw_if_noop = 1"
|
||||||
|
|
||||||
|
$CLICKHOUSE_CLIENT --query "DETACH TABLE load_parts_refcounts"
|
||||||
|
$CLICKHOUSE_CLIENT --query "ATTACH TABLE load_parts_refcounts"
|
||||||
|
|
||||||
|
query_with_retry "
|
||||||
|
SELECT throwIf(count() == 0) FROM system.parts
|
||||||
|
WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_refcounts' AND NOT active FORMAT Null"
|
||||||
|
|
||||||
|
$CLICKHOUSE_CLIENT --query "
|
||||||
|
SELECT DISTINCT refcount FROM system.parts
|
||||||
|
WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'load_parts_refcounts' AND NOT active"
|
||||||
|
|
||||||
|
$CLICKHOUSE_CLIENT --query "DROP TABLE load_parts_refcounts SYNC"
|
@ -0,0 +1,6 @@
|
|||||||
|
(1,2,NULL)
|
||||||
|
(1,2,NULL)
|
||||||
|
(NULL,NULL,NULL)
|
||||||
|
1
|
||||||
|
1
|
||||||
|
1
|
19
tests/queries/0_stateless/02532_json_missing_named_tuple_elements.sh
Executable file
19
tests/queries/0_stateless/02532_json_missing_named_tuple_elements.sh
Executable file
@ -0,0 +1,19 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Tags: long
|
||||||
|
|
||||||
|
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||||
|
# shellcheck source=../shell_config.sh
|
||||||
|
. "$CUR_DIR"/../shell_config.sh
|
||||||
|
|
||||||
|
echo '{"t" : {"a" : 1, "b" : 2}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table"
|
||||||
|
|
||||||
|
echo '{"t" : { "a" : 1 , "b" : 2 } }' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table"
|
||||||
|
|
||||||
|
echo '{"t" : {}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table"
|
||||||
|
|
||||||
|
echo '{"t" : {"a" : 1, "b" : 2}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table" --input_format_json_defaults_for_missing_elements_in_named_tuple=0 2>&1 | grep -F "INCORRECT_DATA" -c
|
||||||
|
|
||||||
|
echo '{"t" : {"a" : 1, "d" : 2}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table" 2>&1 | grep -F "NOT_FOUND_COLUMN_IN_BLOCK" -c
|
||||||
|
|
||||||
|
echo '{"t" : {"a" : 1, "b" : 2, "c" : 3, "d" : 4}}' | $CLICKHOUSE_LOCAL --input-format=NDJSON --structure='t Tuple(a Nullable(UInt32), b Nullable(UInt32), c Nullable(UInt32))' -q "select * from table" 2>&1 | grep -F "INCORRECT_DATA" -c
|
||||||
|
|
@ -0,0 +1 @@
|
|||||||
|
ServerStartupMilliseconds
|
@ -0,0 +1 @@
|
|||||||
|
SELECT event FROM system.events WHERE event = 'ServerStartupMilliseconds'
|
@ -1,4 +0,0 @@
|
|||||||
DROP TABLE IF EXISTS test;
|
|
||||||
CREATE TABLE test (id UInt64, val Decimal(15,5) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS }
|
|
||||||
CREATE TABLE test (id UInt64, val FixedString(2) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS }
|
|
||||||
CREATE TABLE test (id UInt64, val UInt64 CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS }
|
|
Loading…
Reference in New Issue
Block a user