From 534e199c43651507216f912f86dbc59510edcc6e Mon Sep 17 00:00:00 2001 From: Daria Mozhaeva Date: Wed, 30 Sep 2020 11:32:57 +0400 Subject: [PATCH 01/30] Edit and translate to Russian. --- .../settings.md | 8 +- docs/en/operations/settings/settings.md | 98 +++---- docs/en/sql-reference/statements/system.md | 6 +- .../settings.md | 6 +- docs/ru/operations/settings/settings.md | 242 +++++++++++++----- 5 files changed, 235 insertions(+), 125 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index c1ac1d0d92d..d89f74f6bdc 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -351,15 +351,15 @@ Keys for syslog: ## send\_crash\_reports {#server_configuration_parameters-logger} Settings for opt-in sending crash reports to the ClickHouse core developers team via [Sentry](https://sentry.io). -Enabling it, especially in pre-production environments, is greatly appreciated. +Enabling it, especially in pre-production environments, is highly appreciated. -The server will need an access to public Internet via IPv4 (at the time of writing IPv6 is not supported by Sentry) for this feature to be functioning properly. +The server will need access to the public Internet via IPv4 (at the time of writing IPv6 is not supported by Sentry) for this feature to be functioning properly. Keys: - `enabled` – Boolean flag to enable the feature, `false` by default. Set to `true` to allow sending crash reports. -- `endpoint` – You can override the Sentry endpoint URL for sending crash reports. It can be either separate Sentry account or your self-hosted Sentry instance. Use the [Sentry DSN](https://docs.sentry.io/error-reporting/quickstart/?platform=native#configure-the-sdk) syntax. -- `anonymize` - Avoid attaching the server hostname to crash report. +- `endpoint` – You can override the Sentry endpoint URL for sending crash reports. It can be either a separate Sentry account or your self-hosted Sentry instance. Use the [Sentry DSN](https://docs.sentry.io/error-reporting/quickstart/?platform=native#configure-the-sdk) syntax. +- `anonymize` - Avoid attaching the server hostname to the crash report. - `http_proxy` - Configure HTTP proxy for sending crash reports. - `debug` - Sets the Sentry client into debug mode. - `tmp_path` - Filesystem path for temporary crash report state. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 4995c04f712..ee7eb1fd6be 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2,7 +2,7 @@ ## distributed\_product\_mode {#distributed-product-mode} -Changes the behavior of [distributed subqueries](../../sql-reference/operators/in.md). +Changes the behaviour of [distributed subqueries](../../sql-reference/operators/in.md). ClickHouse applies this setting when the query contains the product of distributed tables, i.e. when the query for a distributed table contains a non-GLOBAL subquery for the distributed table. @@ -42,7 +42,7 @@ Consider the following queries: If `enable_optimize_predicate_expression = 1`, then the execution time of these queries is equal because ClickHouse applies `WHERE` to the subquery when processing it. -If `enable_optimize_predicate_expression = 0`, then the execution time of the second query is much longer, because the `WHERE` clause applies to all the data after the subquery finishes. +If `enable_optimize_predicate_expression = 0`, then the execution time of the second query is much longer because the `WHERE` clause applies to all the data after the subquery finishes. ## fallback\_to\_stale\_replicas\_for\_distributed\_queries {#settings-fallback_to_stale_replicas_for_distributed_queries} @@ -215,7 +215,7 @@ Ok. ## input\_format\_values\_deduce\_templates\_of\_expressions {#settings-input_format_values_deduce_templates_of_expressions} -Enables or disables template deduction for SQL expressions in [Values](../../interfaces/formats.md#data-format-values) format. It allows parsing and interpreting expressions in `Values` much faster if expressions in consecutive rows have the same structure. ClickHouse tries to deduce template of an expression, parse the following rows using this template and evaluate the expression on a batch of successfully parsed rows. +Enables or disables template deduction for SQL expressions in [Values](../../interfaces/formats.md#data-format-values) format. It allows parsing and interpreting expressions in `Values` much faster if expressions in consecutive rows have the same structure. ClickHouse tries to deduce the template of an expression, parse the following rows using this template and evaluate the expression on a batch of successfully parsed rows. Possible values: @@ -236,7 +236,7 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( ## input\_format\_values\_accurate\_types\_of\_literals {#settings-input-format-values-accurate-types-of-literals} -This setting is used only when `input_format_values_deduce_templates_of_expressions = 1`. It can happen, that expressions for some column have the same structure, but contain numeric literals of different types, e.g. +This setting is used only when `input_format_values_deduce_templates_of_expressions = 1`. Expressions for some column may have the same structure, but contain numeric literals of different types, e.g. ``` sql (..., abs(0), ...), -- UInt64 literal @@ -278,7 +278,7 @@ Disabled by default. ## input\_format\_null\_as\_default {#settings-input-format-null-as-default} -Enables or disables using default values if input data contain `NULL`, but data type of the corresponding column in not `Nullable(T)` (for text input formats). +Enables or disables using default values if input data contain `NULL`, but the data type of the corresponding column in not `Nullable(T)` (for text input formats). ## input\_format\_skip\_unknown\_fields {#settings-input-format-skip-unknown-fields} @@ -395,7 +395,7 @@ See also: ## join\_use\_nulls {#join_use_nulls} -Sets the type of [JOIN](../../sql-reference/statements/select/join.md) behavior. When merging tables, empty cells may appear. ClickHouse fills them differently based on this setting. +Sets the type of [JOIN](../../sql-reference/statements/select/join.md) behaviour. When merging tables, empty cells may appear. ClickHouse fills them differently based on this setting. Possible values: @@ -424,8 +424,8 @@ Limits sizes of right-hand join data blocks in partial merge join algorithm for ClickHouse server: 1. Splits right-hand join data into blocks with up to the specified number of rows. -2. Indexes each block with their minimum and maximum values -3. Unloads prepared blocks to disk if possible. +2. Indexes each block with its minimum and maximum values. +3. Unloads prepared blocks to disk if it is possible. Possible values: @@ -447,25 +447,25 @@ Default value: 64. ## any\_join\_distinct\_right\_table\_keys {#any_join_distinct_right_table_keys} -Enables legacy ClickHouse server behavior in `ANY INNER|LEFT JOIN` operations. +Enables legacy ClickHouse server behaviour in `ANY INNER|LEFT JOIN` operations. !!! note "Warning" - Use this setting only for the purpose of backward compatibility if your use cases depend on legacy `JOIN` behavior. + Use this setting only for backward compatibility if your use cases depend on legacy `JOIN` behaviour. -When the legacy behavior enabled: +When the legacy behaviour enabled: - Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are not equal because ClickHouse uses the logic with many-to-one left-to-right table keys mapping. - Results of `ANY INNER JOIN` operations contain all rows from the left table like the `SEMI LEFT JOIN` operations do. -When the legacy behavior disabled: +When the legacy behaviour disabled: - Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are equal because ClickHouse uses the logic which provides one-to-many keys mapping in `ANY RIGHT JOIN` operations. -- Results of `ANY INNER JOIN` operations contain one row per key from both left and right tables. +- Results of `ANY INNER JOIN` operations contain one row per key from both the left and right tables. Possible values: -- 0 — Legacy behavior is disabled. -- 1 — Legacy behavior is enabled. +- 0 — Legacy behaviour is disabled. +- 1 — Legacy behaviour is enabled. Default value: 0. @@ -634,7 +634,7 @@ Possible values: Default value: `QUERY_START`. -Can be used to limit which entiries will goes to `query_log`, say you are interesting only in errors, then you can use `EXCEPTION_WHILE_PROCESSING`: +Can be used to limit which entities will go to `query_log`, say you are interested only in errors, then you can use `EXCEPTION_WHILE_PROCESSING`: ``` text log_queries_min_type='EXCEPTION_WHILE_PROCESSING' @@ -662,11 +662,11 @@ The setting also doesn’t have a purpose when using INSERT SELECT, since data i Default value: 1,048,576. -The default is slightly more than `max_block_size`. The reason for this is because certain table engines (`*MergeTree`) form a data part on the disk for each inserted block, which is a fairly large entity. Similarly, `*MergeTree` tables sort data during insertion and a large enough block size allow sorting more data in RAM. +The default is slightly more than `max_block_size`. The reason for this is because certain table engines (`*MergeTree`) form a data part on the disk for each inserted block, which is a fairly large entity. Similarly, `*MergeTree` tables sort data during insertion, and a large enough block size allow sorting more data in RAM. ## min\_insert\_block\_size\_rows {#min-insert-block-size-rows} -Sets minimum number of rows in block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. +Sets the minimum number of rows in the block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. Possible values: @@ -677,7 +677,7 @@ Default value: 1048576. ## min\_insert\_block\_size\_bytes {#min-insert-block-size-bytes} -Sets minimum number of bytes in block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. +Sets the minimum number of bytes in the block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. Possible values: @@ -754,7 +754,7 @@ Default value: 256 KiB. ## max\_parser\_depth {#max_parser_depth} -Limits maximum recursion depth in the recursive descent parser. Allows to control stack size. +Limits maximum recursion depth in the recursive descent parser. Allows controlling the stack size. Possible values: @@ -865,12 +865,12 @@ Yandex.Metrica uses this parameter set to 1 for implementing suggestions for seg ## replace\_running\_query\_max\_wait\_ms {#replace-running-query-max-wait-ms} -The wait time for running query with the same `query_id` to finish, when the [replace_running_query](#replace-running-query) setting is active. +The wait time for running the query with the same `query_id` to finish, when the [replace_running_query](#replace-running-query) setting is active. Possible values: - Positive integer. -- 0 — Throwing an exception that does not allow to run a new query if the server already executes a query with the same `query_id`. +- 0 — Throwing an exception that does not allow to run a new query if the server already executes a query with the same `query_id`. Default value: 5000. @@ -946,7 +946,7 @@ The `first_or_random` algorithm solves the problem of the `in_order` algorithm. load_balancing = round_robin ``` -This algorithm uses round robin policy across replicas with the same number of errors (only the queries with `round_robin` policy is accounted). +This algorithm uses a round-robin policy across replicas with the same number of errors (only the queries with `round_robin` policy is accounted). ## prefer\_localhost\_replica {#settings-prefer-localhost-replica} @@ -983,7 +983,7 @@ Replica lag is not controlled. Enable compilation of queries. By default, 0 (disabled). The compilation is only used for part of the query-processing pipeline: for the first stage of aggregation (GROUP BY). -If this portion of the pipeline was compiled, the query may run faster due to deployment of short cycles and inlining aggregate function calls. The maximum performance improvement (up to four times faster in rare cases) is seen for queries with multiple simple aggregate functions. Typically, the performance gain is insignificant. In very rare cases, it may slow down query execution. +If this portion of the pipeline was compiled, the query may run faster due to the deployment of short cycles and inlining aggregate function calls. The maximum performance improvement (up to four times faster in rare cases) is seen for queries with multiple simple aggregate functions. Typically, the performance gain is insignificant. In very rare cases, it may slow down query execution. ## min\_count\_to\_compile {#min-count-to-compile} @@ -1099,7 +1099,7 @@ When `output_format_json_quote_denormals = 1`, the query returns: ## format\_csv\_delimiter {#settings-format_csv_delimiter} -The character interpreted as a delimiter in the CSV data. By default, the delimiter is `,`. +The character is interpreted as a delimiter in the CSV data. By default, the delimiter is `,`. ## input\_format\_csv\_unquoted\_null\_literal\_as\_null {#settings-input_format_csv_unquoted_null_literal_as_null} @@ -1142,7 +1142,7 @@ See also: ## insert\_quorum\_timeout {#settings-insert_quorum_timeout} -Write to quorum timeout in seconds. If the timeout has passed and no write has taken place yet, ClickHouse will generate an exception and the client must repeat the query to write the same block to the same or any other replica. +Write to a quorum timeout in seconds. If the timeout has passed and no write has taken place yet, ClickHouse will generate an exception and the client must repeat the query to write the same block to the same or any other replica. Default value: 60 seconds. @@ -1198,8 +1198,8 @@ Default value: 0. Usage By default, deduplication is not performed for materialized views but is done upstream, in the source table. -If an INSERTed block is skipped due to deduplication in the source table, there will be no insertion into attached materialized views. This behaviour exists to enable insertion of highly aggregated data into materialized views, for cases where inserted blocks are the same after materialized view aggregation but derived from different INSERTs into the source table. -At the same time, this behaviour “breaks” `INSERT` idempotency. If an `INSERT` into the main table was successful and `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won’t receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` allows for changing this behaviour. On retry, a materialized view will receive the repeat insert and will perform deduplication check by itself, +If an INSERTed block is skipped due to deduplication in the source table, there will be no insertion into attached materialized views. This behaviour exists to enable the insertion of highly aggregated data into materialized views, for cases where inserted blocks are the same after materialized view aggregation but derived from different INSERTs into the source table. +At the same time, this behaviour “breaks” `INSERT` idempotency. If an `INSERT` into the main table was successful and `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won’t receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` allows for changing this behaviour. On retry, a materialized view will receive the repeat insert and will perform a deduplication check by itself, ignoring check result for the source table, and will insert rows lost because of the first failure. ## max\_network\_bytes {#settings-max-network-bytes} @@ -1355,7 +1355,7 @@ Default value: 0. - Type: seconds - Default value: 60 seconds -Controls how fast errors in distributed tables are zeroed. If a replica is unavailable for some time, accumulates 5 errors, and distributed\_replica\_error\_half\_life is set to 1 second, then the replica is considered normal 3 seconds after last error. +Controls how fast errors in distributed tables are zeroed. If a replica is unavailable for some time, accumulates 5 errors, and distributed\_replica\_error\_half\_life is set to 1 second, then the replica is considered normal 3 seconds after the last error. See also: @@ -1369,7 +1369,7 @@ See also: - Type: unsigned int - Default value: 1000 -Error count of each replica is capped at this value, preventing a single replica from accumulating too many errors. +The error count of each replica is capped at this value, preventing a single replica from accumulating too many errors. See also: @@ -1383,7 +1383,7 @@ See also: - Type: unsigned int - Default value: 0 -Number of errors that will be ignored while choosing replicas (according to `load_balancing` algorithm). +The number of errors that will be ignored while choosing replicas (according to `load_balancing` algorithm). See also: @@ -1414,7 +1414,7 @@ Default value: 30000 milliseconds (30 seconds). ## distributed\_directory\_monitor\_batch\_inserts {#distributed_directory_monitor_batch_inserts} -Enables/disables sending of inserted data in batches. +Enables/disables inserted data sending in batches. When batch sending is enabled, the [Distributed](../../engines/table-engines/special/distributed.md) table engine tries to send multiple files of inserted data in one operation instead of sending them separately. Batch sending improves cluster performance by better-utilizing server and network resources. @@ -1507,7 +1507,7 @@ Default value: 0. - Type: bool - Default value: True -Enable order-preserving parallel parsing of data formats. Supported only for TSV, TKSV, CSV and JSONEachRow formats. +Enable order-preserving parallel parsing of data formats. Supported only for TSV, TKSV, CSV, and JSONEachRow formats. ## min\_chunk\_bytes\_for\_parallel\_parsing {#min-chunk-bytes-for-parallel-parsing} @@ -1559,7 +1559,7 @@ Default value: 0. ## background\_pool\_size {#background_pool_size} -Sets the number of threads performing background operations in table engines (for example, merges in [MergeTree engine](../../engines/table-engines/mergetree-family/index.md) tables). This setting is applied from `default` profile at ClickHouse server start and can’t be changed in a user session. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. +Sets the number of threads performing background operations in table engines (for example, merges in [MergeTree engine](../../engines/table-engines/mergetree-family/index.md) tables). This setting is applied from the `default` profile at the ClickHouse server start and can’t be changed in a user session. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. Before changing it, please also take a look at related [MergeTree settings](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-merge_tree), such as `number_of_free_entries_in_pool_to_lower_max_size_of_merge` and `number_of_free_entries_in_pool_to_execute_mutation`. @@ -1578,8 +1578,8 @@ If we execute `INSERT INTO distributed_table_a SELECT ... FROM distributed_table Possible values: - 0 — Disabled. -- 1 — `SELECT` will be executed on each shard from underlying table of the distributed engine. -- 2 — `SELECT` and `INSERT` will be executed on each shard from/to underlying table of the distributed engine. +- 1 — `SELECT` will be executed on each shard from the underlying table of the distributed engine. +- 2 — `SELECT` and `INSERT` will be executed on each shard from/to the underlying table of the distributed engine. Default value: 0. @@ -1602,7 +1602,7 @@ Default value: `0`. - [Managing Distributed Tables](../../sql-reference/statements/system.md#query-language-system-distributed) ## background\_buffer\_flush\_schedule\_pool\_size {#background_buffer_flush_schedule_pool_size} -Sets the number of threads performing background flush in [Buffer](../../engines/table-engines/special/buffer.md)-engine tables. This setting is applied at ClickHouse server start and can’t be changed in a user session. +Sets the number of threads performing background flush in [Buffer](../../engines/table-engines/special/buffer.md)-engine tables. This setting is applied at the ClickHouse server start and can’t be changed in a user session. Possible values: @@ -1612,7 +1612,7 @@ Default value: 16. ## background\_move\_pool\_size {#background_move_pool_size} -Sets the number of threads performing background moves of data parts for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes)-engine tables. This setting is applied at ClickHouse server start and can’t be changed in a user session. +Sets the number of threads performing background moves of data parts for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes)-engine tables. This setting is applied at the ClickHouse server start and can’t be changed in a user session. Possible values: @@ -1634,7 +1634,7 @@ Default value: 16. Prohibits data parts merging in [Replicated\*MergeTree](../../engines/table-engines/mergetree-family/replication.md)-engine tables. -When merging is prohibited, the replica never merges parts and always downloads merged parts from other replicas. If there is no required data yet, the replica waits for it. CPU and disk load on the replica server decreases, but the network load on cluster increases. This setting can be useful on servers with relatively weak CPUs or slow disks, such as servers for backups storage. +When merging is prohibited, the replica never merges parts and always downloads merged parts from other replicas. If there is no required data yet, the replica waits for it. CPU and disk load on the replica server decreases, but the network load on the cluster increases. This setting can be useful on servers with relatively weak CPUs or slow disks, such as servers for backups storage. Possible values: @@ -1649,7 +1649,7 @@ Default value: 0. ## background\_distributed\_schedule\_pool\_size {#background_distributed_schedule_pool_size} -Sets the number of threads performing background tasks for [distributed](../../engines/table-engines/special/distributed.md) sends. This setting is applied at ClickHouse server start and can’t be changed in a user session. +Sets the number of threads performing background tasks for [distributed](../../engines/table-engines/special/distributed.md) sends. This setting is applied at the ClickHouse server start and can’t be changed in a user session. Possible values: @@ -1740,7 +1740,7 @@ Default value: 8192. Turns on or turns off using of single dictionary for the data part. -By default, ClickHouse server monitors the size of dictionaries and if a dictionary overflows then the server starts to write the next one. To prohibit creating several dictionaries set `low_cardinality_use_single_dictionary_for_part = 1`. +By default, the ClickHouse server monitors the size of dictionaries and if a dictionary overflows then the server starts to write the next one. To prohibit creating several dictionaries set `low_cardinality_use_single_dictionary_for_part = 1`. Possible values: @@ -1785,7 +1785,7 @@ Default value: 0. ## min\_insert\_block\_size\_rows\_for\_materialized\_views {#min-insert-block-size-rows-for-materialized-views} -Sets minimum number of rows in block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. This setting is applied only for blocks inserted into [materialized view](../../sql-reference/statements/create/view.md). By adjusting this setting, you control blocks squashing while pushing to materialized view and avoid excessive memory usage. +Sets the minimum number of rows in the block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. This setting is applied only for blocks inserted into [materialized view](../../sql-reference/statements/create/view.md). By adjusting this setting, you control blocks squashing while pushing to materialized view and avoid excessive memory usage. Possible values: @@ -1800,7 +1800,7 @@ Default value: 1048576. ## min\_insert\_block\_size\_bytes\_for\_materialized\_views {#min-insert-block-size-bytes-for-materialized-views} -Sets minimum number of bytes in block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. This setting is applied only for blocks inserted into [materialized view](../../sql-reference/statements/create/view.md). By adjusting this setting, you control blocks squashing while pushing to materialized view and avoid excessive memory usage. +Sets the minimum number of bytes in the block which can be inserted into a table by an `INSERT` query. Smaller-sized blocks are squashed into bigger ones. This setting is applied only for blocks inserted into [materialized view](../../sql-reference/statements/create/view.md). By adjusting this setting, you control blocks squashing while pushing to materialized view and avoid excessive memory usage. Possible values: @@ -1815,7 +1815,7 @@ Default value: 268435456. ## output\_format\_pretty\_grid\_charset {#output-format-pretty-grid-charset} -Allows to change a charset which is used for printing grids borders. Available charsets are following: UTF-8, ASCII. +Allows changing a charset which is used for printing grids borders. Available charsets are UTF-8, ASCII. **Example** @@ -1872,12 +1872,12 @@ When `ttl_only_drop_parts` is disabled (by default), the ClickHouse server only When `ttl_only_drop_parts` is enabled, the ClickHouse server drops a whole part when all rows in it are expired. -Dropping whole parts instead of partial cleaning TTL-d rows allows to have shorter `merge_with_ttl_timeout` times and lower impact on system performance. +Dropping whole parts instead of partial cleaning TTL-d rows allows having shorter `merge_with_ttl_timeout` times and lower impact on system performance. Possible values: -- 0 — Complete dropping of data parts is disabled. -- 1 — Complete dropping of data parts is enabled. +- 0 — The complete dropping of data parts is disabled. +- 1 — The complete dropping of data parts is enabled. Default value: `0`. @@ -1888,9 +1888,9 @@ Default value: `0`. ## lock_acquire_timeout {#lock_acquire_timeout} -Defines how many seconds locking request waits before failing. +Defines how many seconds a locking request waits before failing. -Locking timeout is used to protect from deadlocks while executing read/write operations with tables. When timeout expires and locking request fails, the ClickHouse server throws an exeption "Locking attempt timed out! Possible deadlock avoided. Client should retry." with error code `DEADLOCK_AVOIDED`. +Locking timeout is used to protect from deadlocks while executing read/write operations with tables. When the timeout expires and the locking request fails, the ClickHouse server throws an exception "Locking attempt timed out! Possible deadlock avoided. Client should retry." with error code `DEADLOCK_AVOIDED`. Possible values: diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index f6ff264e827..a9f9b718de6 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -81,12 +81,12 @@ SYSTEM DROP REPLICA 'replica_name'; SYSTEM DROP REPLICA 'replica_name' FROM ZKPATH '/path/to/table/in/zk'; ``` -Queries will remove the replica path in ZooKeeper. It’s useful when replica is dead and its metadata cannot be removed from ZooKeeper by `DROP TABLE` because there is no such table anymore. It will only drop the inactive/stale replica, and it can’t drop local replica, please use `DROP TABLE` for that. `DROP REPLICA` does not drop any tables and does not remove any data or metadata from disk. +Queries will remove the replica path in ZooKeeper. It is useful when the replica is dead and its metadata cannot be removed from ZooKeeper by `DROP TABLE` because there is no such table anymore. It will only drop the inactive/stale replica, and it cannot drop local replica, please use `DROP TABLE` for that. `DROP REPLICA` does not drop any tables and does not remove any data or metadata from disk. The first one removes metadata of `'replica_name'` replica of `database.table` table. The second one does the same for all replicated tables in the database. -The third one does the same for all replicated tables on local server. -The forth one is useful to remove metadata of dead replica when all other replicas of a table were dropped. It requires the table path to be specified explicitly. It must be the same path as was passed to the first argument of `ReplicatedMergeTree` engine on table creation. +The third one does the same for all replicated tables on the local server. +The fourth one is useful to remove metadata of dead replica when all other replicas of a table were dropped. It requires the table path to be specified explicitly. It must be the same path as was passed to the first argument of `ReplicatedMergeTree` engine on table creation. ## DROP UNCOMPRESSED CACHE {#query_language-system-drop-uncompressed-cache} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 795a9f5893a..0abb568ffc7 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -492,11 +492,11 @@ ClickHouse проверяет условия для `min_part_size` и `min_part ## max\_thread\_pool\_size {#max-thread-pool-size} -Максимальное кол-во потоков в глобальном пуле потоков. +Максимальное количество потоков в глобальном пуле потоков. -Default value: 10000. +Значение по умолчанию: 10000. -**Example** +**Пример** ``` xml 12000 diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 9a487b6c166..15c4139a3f3 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -281,6 +281,14 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( Значение по умолчанию: 1. +## input\_format\_tsv\_empty\_as\_default {#settings-input-format-tsv-empty-as-default} + +Если эта настройка включена, замените пустые поля ввода в TSV значениями по умолчанию. Для сложных выражений по умолчанию также должна быть включена настройка `input_format_defaults_for_omitted_fields`. + +По умолчанию отключена. + +Disabled by default. + ## input\_format\_null\_as\_default {#settings-input-format-null-as-default} Включает или отключает использование значений по умолчанию в случаях, когда во входных данных содержится `NULL`, но тип соответствующего столбца не `Nullable(T)` (для текстовых форматов). @@ -369,7 +377,7 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( Устанавливает строгость по умолчанию для [JOIN](../../sql-reference/statements/select/join.md#select-join). -Возможные значения +Возможные значения: - `ALL` — если в правой таблице несколько совпадающих строк, данные умножаются на количество этих строк. Это нормальное поведение `JOIN` как в стандартном SQL. - `ANY` — если в правой таблице несколько соответствующих строк, то соединяется только первая найденная. Если в «правой» таблице есть не более одной подходящей строки, то результаты `ANY` и `ALL` совпадают. @@ -520,6 +528,31 @@ ClickHouse использует этот параметр при чтении д Значение по умолчанию: 0. +## network_compression_method {#network_compression_method} + +Устанавливает метод сжатия данных, который используется для обмена данными между серверами и между сервером и [clickhouse-client](../../interfaces/cli.md). + +Возможные значения: + +- `LZ4` — устанавливает метод сжатия LZ4. +- `ZSTD` — устанавливает метод сжатия ZSTD. + +Значение по умолчанию: `LZ4`. + +**См. также** + +- [network_zstd_compression_level](#network_zstd_compression_level) + +## network_zstd_compression_level {#network_zstd_compression_level} + +Регулирует уровень сжатия ZSTD. Используется только тогда, когда [network_compression_method](#network_compression_method) установлен на `ZSTD`. + +Возможные значения: + +- Положительное целое число от 1 до 15. + +Значение по умолчанию: `1`. + ## log\_queries {#settings-log-queries} Установка логирования запроса. @@ -534,42 +567,6 @@ log_queries=1 ## log\_queries\_min\_type {#settings-log-queries-min-type} -`query_log` минимальный уровень логирования. - -Возможные значения: -- `QUERY_START` (`=1`) -- `QUERY_FINISH` (`=2`) -- `EXCEPTION_BEFORE_START` (`=3`) -- `EXCEPTION_WHILE_PROCESSING` (`=4`) - -Значение по умолчанию: `QUERY_START`. - -Можно использовать для ограничения того, какие объекты будут записаны в `query_log`, например, если вас интересуют ошибки, тогда вы можете использовать `EXCEPTION_WHILE_PROCESSING`: - -``` text -log_queries_min_type='EXCEPTION_WHILE_PROCESSING' -``` - -## log\_queries\_min\_type {#settings-log-queries-min-type} - -`query_log` минимальный уровень логирования. - -Возможные значения: -- `QUERY_START` (`=1`) -- `QUERY_FINISH` (`=2`) -- `EXCEPTION_BEFORE_START` (`=3`) -- `EXCEPTION_WHILE_PROCESSING` (`=4`) - -Значение по умолчанию: `QUERY_START`. - -Можно использовать для ограничения того, какие объекты будут записаны в `query_log`, например, если вас интересуют ошибки, тогда вы можете использовать `EXCEPTION_WHILE_PROCESSING`: - -``` text -log_queries_min_type='EXCEPTION_WHILE_PROCESSING' -``` - -## log\_queries\_min\_type {#settings-log-queries-min-type} - Задаёт минимальный уровень логирования в `query_log`. Возможные значения: @@ -839,6 +836,11 @@ ClickHouse поддерживает следующие алгоритмы выб - [Nearest hostname](#load_balancing-nearest_hostname) - [In order](#load_balancing-in_order) - [First or random](#load_balancing-first_or_random) +- [Round robin](#load_balancing-round_robin) + +См. также: + +- [distributed\_replica\_max\_ignored\_errors](#settings-distributed_replica_max_ignored_errors) ### Random (by Default) {#load_balancing-random} @@ -882,6 +884,14 @@ load_balancing = first_or_random Алгоритм `first or random` решает проблему алгоритма `in order`. При использовании `in order`, если одна реплика перестаёт отвечать, то следующая за ней принимает двойную нагрузку, в то время как все остальные обрабатываю свой обычный трафик. Алгоритм `first or random` равномерно распределяет нагрузку между репликами. +### Round Robin {#load_balancing-round_robin} + +``` sql +load_balancing = round_robin +``` + +Этот алгоритм использует циклический перебор реплик с одинаковым количеством ошибок (учитываются только запросы с алгоритмом `round_robin`). + ## prefer\_localhost\_replica {#settings-prefer-localhost-replica} Включает или выключает предпочтительное использование localhost реплики при обработке распределенных запросов. @@ -1292,6 +1302,48 @@ ClickHouse генерирует исключение Значение по умолчанию: 0. +## distributed\_replica\_error\_half\_life {#settings-distributed_replica_error_half_life} + +- Тип: секунды +- Значение по умолчанию: 60 секунд + +Управляет скоростью обнуления ошибок в распределенных таблицах. Если реплика недоступна в течение некоторого времени, накапливает 5 ошибок, а distributed\_replica\_error\_half\_life установлена на 1 секунду, то реплика считается нормальной через 3 секунды после последней ошибки. + +См. также: + +- [load\_balancing](#load_balancing-round_robin) +- [Table engine Distributed](../../engines/table-engines/special/distributed.md) +- [distributed\_replica\_error\_cap](#settings-distributed_replica_error_cap) +- [distributed\_replica\_max\_ignored\_errors](#settings-distributed_replica_max_ignored_errors) + +## distributed\_replica\_error\_cap {#settings-distributed_replica_error_cap} + +- Тип: unsigned int +- Значение по умолчанию: 1000 + +Счетчик ошибок каждой реплики ограничен этим значением, чтобы одна реплика не накапливала слишком много ошибок. + +См. также: + +- [load\_balancing](#load_balancing-round_robin) +- [Table engine Distributed](../../engines/table-engines/special/distributed.md) +- [distributed\_replica\_error\_half\_life](#settings-distributed_replica_error_half_life) +- [distributed\_replica\_max\_ignored\_errors](#settings-distributed_replica_max_ignored_errors) + +## distributed\_replica\_max\_ignored\_errors {#settings-distributed_replica_max_ignored_errors} + +- Тип: unsigned int +- Значение по умолчанию: 0 + +Количество ошибок, которые будут проигнорированы при выборе реплик (согласно алгоритму `load_balancing`). + +См. также: + +- [load\_balancing](#load_balancing-round_robin) +- [Table engine Distributed](../../engines/table-engines/special/distributed.md) +- [distributed\_replica\_error\_cap](#settings-distributed_replica_error_cap) +- [distributed\_replica\_error\_half\_life](#settings-distributed_replica_error_half_life) + ## distributed\_directory\_monitor\_sleep\_time\_ms {#distributed_directory_monitor_sleep_time_ms} Основной интервал отправки данных движком таблиц [Distributed](../../engines/table-engines/special/distributed.md). Фактический интервал растёт экспоненциально при возникновении ошибок. @@ -1342,65 +1394,103 @@ ClickHouse генерирует исключение ## query\_profiler\_real\_time\_period\_ns {#query_profiler_real_time_period_ns} -Sets the period for a real clock timer of the [query profiler](../../operations/optimizing-performance/sampling-query-profiler.md). Real clock timer counts wall-clock time. +Устанавливает период для таймера реального времени [профилировщика запросов](../../operations/optimizing-performance/sampling-query-profiler.md). Таймер реального времени считает wall-clock time. -Possible values: +Возможные значения: -- Positive integer number, in nanoseconds. +- Положительное целое число в наносекундах. - Recommended values: + Рекомендуемые значения: - - 10000000 (100 times a second) nanoseconds and less for single queries. - - 1000000000 (once a second) for cluster-wide profiling. + - 10000000 (100 раз в секунду) наносекунд и меньшее значение для одиночных запросов. + - 1000000000 (раз в секунду) для профилирования в масштабе кластера. -- 0 for turning off the timer. +- 0 для выключения таймера. -Type: [UInt64](../../sql-reference/data-types/int-uint.md). +Тип: [UInt64](../../sql-reference/data-types/int-uint.md). -Default value: 1000000000 nanoseconds (once a second). +Значение по умолчанию: 1000000000 наносекунд (раз в секунду). -See also: +См. также: -- System table [trace\_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) +- Системная таблица [trace\_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) ## query\_profiler\_cpu\_time\_period\_ns {#query_profiler_cpu_time_period_ns} -Sets the period for a CPU clock timer of the [query profiler](../../operations/optimizing-performance/sampling-query-profiler.md). This timer counts only CPU time. +Устанавливает период для таймера CPU [query profiler](../../operations/optimizing-performance/sampling-query-profiler.md). Этот таймер считает только время CPU. -Possible values: +Возможные значения: -- Positive integer number of nanoseconds. +- Положительное целое число в наносекундах. - Recommended values: + Рекомендуемые значения: - - 10000000 (100 times a second) nanosecods and more for for single queries. - - 1000000000 (once a second) for cluster-wide profiling. + - 10000000 (100 раз в секунду) наносекунд и большее значение для одиночных запросов. + - 1000000000 (раз в секунду) для профилирования в масштабе кластера. -- 0 for turning off the timer. +- 0 для выключения таймера. -Type: [UInt64](../../sql-reference/data-types/int-uint.md). +Тип: [UInt64](../../sql-reference/data-types/int-uint.md). -Default value: 1000000000 nanoseconds. +Значение по умолчанию: 1000000000 наносекунд. -See also: +См. также: -- System table [trace\_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) +- Системная таблица [trace\_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) ## allow_introspection_functions {#settings-allow_introspection_functions} -Enables of disables [introspections functions](../../sql-reference/functions/introspection.md) for query profiling. +Включает или отключает [функции самоанализа](../../sql-reference/functions/introspection.md) для профилирования запросов. -Possible values: +Возможные значения: -- 1 — Introspection functions enabled. -- 0 — Introspection functions disabled. +- 1 — включены функции самоанализа. +- 0 — функции самоанализа отключены. -Default value: 0. +Значение по умолчанию: 0. -**See Also** +**См. также** - [Sampling Query Profiler](../optimizing-performance/sampling-query-profiler.md) -- System table [trace\_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) +- Системная таблица [trace\_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) + +## input\_format\_parallel\_parsing {#input-format-parallel-parsing} + +- Тип: bool +- Значение по умолчанию: True + +Обеспечивает параллельный анализ форматов данных с сохранением порядка. Поддерживается только для форматов TSV, TKSV, CSV и JSONEachRow. + +## min\_chunk\_bytes\_for\_parallel\_parsing {#min-chunk-bytes-for-parallel-parsing} + +- Тип: unsigned int +- Значение по умолчанию: 1 MiB + +Минимальный размер блока в байтах, который каждый поток будет анализировать параллельно. + +## output\_format\_avro\_codec {#settings-output_format_avro_codec} + +Устанавливает кодек сжатия, используемый для вывода файла Avro. + +Тип: строка + +Возможные значения: + +- `null` — без сжатия +- `deflate` — сжать с помощью Deflate (zlib) +- `snappy` — сжать с помощью [Snappy](https://google.github.io/snappy/) + +Значение по умолчанию: `snappy` (если доступно) или `deflate`. + +## output\_format\_avro\_sync\_interval {#settings-output_format_avro_sync_interval} + +Устанавливает минимальный размер данных (в байтах) между маркерами синхронизации для выходного файла Avro. + +Тип: unsigned int + +озможные значения: 32 (32 байта) - 1073741824 (1 GiB) + +Значение по умолчанию: 32768 (32 KiB) ## background\_pool\_size {#background_pool_size} @@ -1624,6 +1714,26 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; - [min_insert_block_size_bytes](#min-insert-block-size-bytes) +## output\_format\_pretty\_grid\_charset {#output-format-pretty-grid-charset} + +Позволяет изменить кодировку, которая используется для печати грид-границ. Доступны следующие кодировки: UTF-8, ASCII. + +**Пример** + +``` text +SET output_format_pretty_grid_charset = 'UTF-8'; +SELECT * FROM a; +┌─a─┐ +│ 1 │ +└───┘ + +SET output_format_pretty_grid_charset = 'ASCII'; +SELECT * FROM a; ++-a-+ +| 1 | ++---+ +``` + ## optimize_read_in_order {#optimize_read_in_order} Включает или отключает оптимизацию в запросах [SELECT](../../sql-reference/statements/select/index.md) с секцией [ORDER BY](../../sql-reference/statements/select/order-by.md#optimize_read_in_order) при работе с таблицами семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). From c4b9c700c516132471586bff36fcac6f63d5de10 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 26 Jan 2021 02:09:17 +0300 Subject: [PATCH 02/30] Map type and map function. Data type description template also added. --- .../template-data-type.md | 29 +++++++++ docs/en/sql-reference/data-types/map.md | 56 ++++++++++++++++ .../functions/tuple-map-functions.md | 64 ++++++++++++++++++- 3 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 docs/_description_templates/template-data-type.md create mode 100644 docs/en/sql-reference/data-types/map.md diff --git a/docs/_description_templates/template-data-type.md b/docs/_description_templates/template-data-type.md new file mode 100644 index 00000000000..edb6586ee7d --- /dev/null +++ b/docs/_description_templates/template-data-type.md @@ -0,0 +1,29 @@ +--- +toc_priority: +toc_title: +--- + +# data_type_name {#data_type-name} + +Description. + +**Parameters** (Optional) + +- `x` — Description. [Type name](relative/path/to/type/dscr.md#type). +- `y` — Description. [Type name](relative/path/to/type/dscr.md#type). + +**Examples** + +```sql + +``` + +## Additional Info {#additional-info} (Optional) + +The name of an additional section can be any, for example, **Usage**. + +**See Also** (Optional) + +- [link](#) + +[Original article](https://clickhouse.tech/docs/en/data_types//) diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md new file mode 100644 index 00000000000..5f1300896e8 --- /dev/null +++ b/docs/en/sql-reference/data-types/map.md @@ -0,0 +1,56 @@ +--- +toc_priority: 65 +toc_title: Map(key, value) +--- + +# Map(key, value) {#data_type-map} + +`Map(key, value)` data type stores `key:value` pairs in structures like JSON. + +**Parameters** +- `key` — Key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — Value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). + +!!! warning "Warning" + Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. + +To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. + +**Example** + +Query: + +``` sql +CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; +INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); +SELECT a['key2'] FROM table_map; +``` +Result: + +```text +┌─arrayElement(a, 'key2')─┐ +│ 100 │ +│ 200 │ +│ 300 │ +└─────────────────────────┘ +``` + +## Map() and Tuple() Types {#map-and-tuple} + +You can cast `Tuple()` as `Map()`: + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +**See Also** + +- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function + +[Original article](https://clickhouse.tech/docs/en/data_types/map/) diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index a46c36395b8..3de570e6dcc 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -5,6 +5,68 @@ toc_title: Working with maps # Functions for maps {#functions-for-working-with-tuple-maps} +## map {#function-map} + +Arranges `key:value` pairs into a JSON data structure. + +**Syntax** + +``` sql +map(key1, value1[, key2, value2, ...]) +``` + +**Parameters** + +- `key` — Key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — Value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- JSON with `key:value` pairs. + +Type: [Map(key, value)](../../sql-reference/data-types/map.md). + +**Examples** + +Query: + +``` sql +SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); +``` + +Result: + +``` text +┌─map('key1', number, 'key2', multiply(number, 2))─┐ +│ {'key1':0,'key2':0} │ +│ {'key1':1,'key2':2} │ +│ {'key1':2,'key2':4} │ +└──────────────────────────────────────────────────┘ +``` + +Query: + +``` sql +CREATE TABLE table_map (a Map(String, UInt64)) ENGINE = MergeTree() ORDER BY a; +INSERT INTO table_map SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); +SELECT a['key2'] FROM table_map; +``` + +Result: + +``` text +┌─arrayElement(a, 'key2')─┐ +│ 0 │ +│ 2 │ +│ 4 │ +└─────────────────────────┘ +``` + +**See Also** + +- [Map(key, value)](../../sql-reference/data-types/map.md) data type + + ## mapAdd {#function-mapadd} Collect all the keys and sum corresponding values. @@ -112,4 +174,4 @@ Result: └──────────────────────────────┴───────────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/tuple-map-functions/) +[Original article](https://clickhouse.tech/docs/en/sql-reference/functions/tuple-map-functions/) From 297d106f1446790a1a065e2b0ccc416eda658bb8 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 2 Feb 2021 03:24:01 +0300 Subject: [PATCH 03/30] Fixed, translated. Symbolic links added. --- docs/en/sql-reference/data-types/map.md | 11 ++-- .../functions/tuple-map-functions.md | 4 +- docs/es/sql-reference/data-types/map.md | 57 ++++++++++++++++++ docs/fr/sql-reference/data-types/map.md | 57 ++++++++++++++++++ docs/ja/sql-reference/data-types/map.md | 57 ++++++++++++++++++ docs/ru/sql-reference/data-types/map.md | 57 ++++++++++++++++++ .../functions/tuple-map-functions.md | 60 +++++++++++++++++++ docs/zh/sql-reference/data-types/map.md | 57 ++++++++++++++++++ 8 files changed, 353 insertions(+), 7 deletions(-) create mode 100644 docs/es/sql-reference/data-types/map.md create mode 100644 docs/fr/sql-reference/data-types/map.md create mode 100644 docs/ja/sql-reference/data-types/map.md create mode 100644 docs/ru/sql-reference/data-types/map.md create mode 100644 docs/zh/sql-reference/data-types/map.md diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md index 5f1300896e8..0f0f69d421d 100644 --- a/docs/en/sql-reference/data-types/map.md +++ b/docs/en/sql-reference/data-types/map.md @@ -8,8 +8,8 @@ toc_title: Map(key, value) `Map(key, value)` data type stores `key:value` pairs in structures like JSON. **Parameters** -- `key` — Key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). -- `value` — Value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). +- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). !!! warning "Warning" Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. @@ -35,9 +35,9 @@ Result: └─────────────────────────┘ ``` -## Map() and Tuple() Types {#map-and-tuple} +## Convert Tuple to Map Type {#map-and-tuple} -You can cast `Tuple()` as `Map()`: +You can cast `Tuple()` as `Map()` using [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function: ``` sql SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; @@ -52,5 +52,6 @@ SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map **See Also** - [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function +- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function -[Original article](https://clickhouse.tech/docs/en/data_types/map/) +[Original article](https://clickhouse.tech/docs/en/data-types/map/) diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index 3de570e6dcc..b81f971196a 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -17,8 +17,8 @@ map(key1, value1[, key2, value2, ...]) **Parameters** -- `key` — Key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). -- `value` — Value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). +- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). **Returned value** diff --git a/docs/es/sql-reference/data-types/map.md b/docs/es/sql-reference/data-types/map.md new file mode 100644 index 00000000000..0f0f69d421d --- /dev/null +++ b/docs/es/sql-reference/data-types/map.md @@ -0,0 +1,57 @@ +--- +toc_priority: 65 +toc_title: Map(key, value) +--- + +# Map(key, value) {#data_type-map} + +`Map(key, value)` data type stores `key:value` pairs in structures like JSON. + +**Parameters** +- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). + +!!! warning "Warning" + Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. + +To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. + +**Example** + +Query: + +``` sql +CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; +INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); +SELECT a['key2'] FROM table_map; +``` +Result: + +```text +┌─arrayElement(a, 'key2')─┐ +│ 100 │ +│ 200 │ +│ 300 │ +└─────────────────────────┘ +``` + +## Convert Tuple to Map Type {#map-and-tuple} + +You can cast `Tuple()` as `Map()` using [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function: + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +**See Also** + +- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function +- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function + +[Original article](https://clickhouse.tech/docs/en/data-types/map/) diff --git a/docs/fr/sql-reference/data-types/map.md b/docs/fr/sql-reference/data-types/map.md new file mode 100644 index 00000000000..0f0f69d421d --- /dev/null +++ b/docs/fr/sql-reference/data-types/map.md @@ -0,0 +1,57 @@ +--- +toc_priority: 65 +toc_title: Map(key, value) +--- + +# Map(key, value) {#data_type-map} + +`Map(key, value)` data type stores `key:value` pairs in structures like JSON. + +**Parameters** +- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). + +!!! warning "Warning" + Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. + +To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. + +**Example** + +Query: + +``` sql +CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; +INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); +SELECT a['key2'] FROM table_map; +``` +Result: + +```text +┌─arrayElement(a, 'key2')─┐ +│ 100 │ +│ 200 │ +│ 300 │ +└─────────────────────────┘ +``` + +## Convert Tuple to Map Type {#map-and-tuple} + +You can cast `Tuple()` as `Map()` using [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function: + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +**See Also** + +- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function +- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function + +[Original article](https://clickhouse.tech/docs/en/data-types/map/) diff --git a/docs/ja/sql-reference/data-types/map.md b/docs/ja/sql-reference/data-types/map.md new file mode 100644 index 00000000000..0f0f69d421d --- /dev/null +++ b/docs/ja/sql-reference/data-types/map.md @@ -0,0 +1,57 @@ +--- +toc_priority: 65 +toc_title: Map(key, value) +--- + +# Map(key, value) {#data_type-map} + +`Map(key, value)` data type stores `key:value` pairs in structures like JSON. + +**Parameters** +- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). + +!!! warning "Warning" + Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. + +To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. + +**Example** + +Query: + +``` sql +CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; +INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); +SELECT a['key2'] FROM table_map; +``` +Result: + +```text +┌─arrayElement(a, 'key2')─┐ +│ 100 │ +│ 200 │ +│ 300 │ +└─────────────────────────┘ +``` + +## Convert Tuple to Map Type {#map-and-tuple} + +You can cast `Tuple()` as `Map()` using [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function: + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +**See Also** + +- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function +- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function + +[Original article](https://clickhouse.tech/docs/en/data-types/map/) diff --git a/docs/ru/sql-reference/data-types/map.md b/docs/ru/sql-reference/data-types/map.md new file mode 100644 index 00000000000..c1391e37133 --- /dev/null +++ b/docs/ru/sql-reference/data-types/map.md @@ -0,0 +1,57 @@ +--- +toc_priority: 65 +toc_title: Map(key, value) +--- + +# Map(key, value) {#data_type-map} + +Тип данных `Map(key, value)` хранит пары `ключ:значение` в структурах типа JSON. + +**Параметры** +- `key` — ключ. [String](../../sql-reference/data-types/string.md) или [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — значение. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) или [Array](../../sql-reference/data-types/array.md). + +!!! warning "Предупреждение" + Сейчас использование типа данных `Map` является экспериментальной возможностью. Чтобы использовать этот тип данных, включите настройку `allow_experimental_map_type = 1`. + +Чтобы получить значение из колонки `a Map('key', 'value')`, используйте синтаксис `a['key']`. + +**Пример** + +Запрос: + +``` sql +CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; +INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); +SELECT a['key2'] FROM table_map; +``` +Результат: + +```text +┌─arrayElement(a, 'key2')─┐ +│ 100 │ +│ 200 │ +│ 300 │ +└─────────────────────────┘ +``` + +## Преобразование типа данных Tuple в Map {#map-and-tuple} + +Для преобразования данных с типом `Tuple()` в тип `Map()` можно использовать функцию [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast): + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +**См. также** + +- функция [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) +- функция [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) + +[Original article](https://clickhouse.tech/docs/ru/data-types/map/) diff --git a/docs/ru/sql-reference/functions/tuple-map-functions.md b/docs/ru/sql-reference/functions/tuple-map-functions.md index a2b25e68fe5..65e44698008 100644 --- a/docs/ru/sql-reference/functions/tuple-map-functions.md +++ b/docs/ru/sql-reference/functions/tuple-map-functions.md @@ -5,6 +5,66 @@ toc_title: Работа с контейнерами map # Функции для работы с контейнерами map {#functions-for-working-with-tuple-maps} +## map {#function-map} + +Преобразовывает пары `ключ:значение` в структуру JSON. + +**Синтаксис** + +``` sql +map(key1, value1[, key2, value2, ...]) +``` + +**Параметры** + +- `key` — ключ. [String](../../sql-reference/data-types/string.md) или [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — значение. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) или [Array](../../sql-reference/data-types/array.md). + +**Возвращаемое значение** + +- Структура JSON с парами `ключ:значение`. + +Тип: [Map(key, value)](../../sql-reference/data-types/map.md). + +**Примеры** + +Запрос: + +``` sql +SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); +``` + +Результат: + +``` text +┌─map('key1', number, 'key2', multiply(number, 2))─┐ +│ {'key1':0,'key2':0} │ +│ {'key1':1,'key2':2} │ +│ {'key1':2,'key2':4} │ +└──────────────────────────────────────────────────┘ +``` + +Запрос: + +``` sql +CREATE TABLE table_map (a Map(String, UInt64)) ENGINE = MergeTree() ORDER BY a; +INSERT INTO table_map SELECT map('key1', number, 'key2', number * 2) FROM numbers(3); +SELECT a['key2'] FROM table_map; +``` + +Результат: + +``` text +┌─arrayElement(a, 'key2')─┐ +│ 0 │ +│ 2 │ +│ 4 │ +└─────────────────────────┘ +``` + +**См. также** + +- тип данных [Map(key, value)](../../sql-reference/data-types/map.md) ## mapAdd {#function-mapadd} Собирает все ключи и суммирует соответствующие значения. diff --git a/docs/zh/sql-reference/data-types/map.md b/docs/zh/sql-reference/data-types/map.md new file mode 100644 index 00000000000..0f0f69d421d --- /dev/null +++ b/docs/zh/sql-reference/data-types/map.md @@ -0,0 +1,57 @@ +--- +toc_priority: 65 +toc_title: Map(key, value) +--- + +# Map(key, value) {#data_type-map} + +`Map(key, value)` data type stores `key:value` pairs in structures like JSON. + +**Parameters** +- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). +- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). + +!!! warning "Warning" + Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. + +To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. + +**Example** + +Query: + +``` sql +CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; +INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); +SELECT a['key2'] FROM table_map; +``` +Result: + +```text +┌─arrayElement(a, 'key2')─┐ +│ 100 │ +│ 200 │ +│ 300 │ +└─────────────────────────┘ +``` + +## Convert Tuple to Map Type {#map-and-tuple} + +You can cast `Tuple()` as `Map()` using [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function: + +``` sql +SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +``` + +``` text +┌─map───────────────────────────┐ +│ {1:'Ready',2:'Steady',3:'Go'} │ +└───────────────────────────────┘ +``` + +**See Also** + +- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function +- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function + +[Original article](https://clickhouse.tech/docs/en/data-types/map/) From abeeebc66156b85a690e0bc33f7759f26dcd61da Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 2 Feb 2021 22:01:06 +0300 Subject: [PATCH 04/30] Symbolic links deleted --- docs/es/sql-reference/data-types/map.md | 57 ------------------------- docs/fr/sql-reference/data-types/map.md | 57 ------------------------- docs/ja/sql-reference/data-types/map.md | 57 ------------------------- docs/zh/sql-reference/data-types/map.md | 57 ------------------------- 4 files changed, 228 deletions(-) delete mode 100644 docs/es/sql-reference/data-types/map.md delete mode 100644 docs/fr/sql-reference/data-types/map.md delete mode 100644 docs/ja/sql-reference/data-types/map.md delete mode 100644 docs/zh/sql-reference/data-types/map.md diff --git a/docs/es/sql-reference/data-types/map.md b/docs/es/sql-reference/data-types/map.md deleted file mode 100644 index 0f0f69d421d..00000000000 --- a/docs/es/sql-reference/data-types/map.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -toc_priority: 65 -toc_title: Map(key, value) ---- - -# Map(key, value) {#data_type-map} - -`Map(key, value)` data type stores `key:value` pairs in structures like JSON. - -**Parameters** -- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). -- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). - -!!! warning "Warning" - Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. - -To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. - -**Example** - -Query: - -``` sql -CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; -INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); -SELECT a['key2'] FROM table_map; -``` -Result: - -```text -┌─arrayElement(a, 'key2')─┐ -│ 100 │ -│ 200 │ -│ 300 │ -└─────────────────────────┘ -``` - -## Convert Tuple to Map Type {#map-and-tuple} - -You can cast `Tuple()` as `Map()` using [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function: - -``` sql -SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; -``` - -``` text -┌─map───────────────────────────┐ -│ {1:'Ready',2:'Steady',3:'Go'} │ -└───────────────────────────────┘ -``` - -**See Also** - -- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function -- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function - -[Original article](https://clickhouse.tech/docs/en/data-types/map/) diff --git a/docs/fr/sql-reference/data-types/map.md b/docs/fr/sql-reference/data-types/map.md deleted file mode 100644 index 0f0f69d421d..00000000000 --- a/docs/fr/sql-reference/data-types/map.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -toc_priority: 65 -toc_title: Map(key, value) ---- - -# Map(key, value) {#data_type-map} - -`Map(key, value)` data type stores `key:value` pairs in structures like JSON. - -**Parameters** -- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). -- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). - -!!! warning "Warning" - Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. - -To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. - -**Example** - -Query: - -``` sql -CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; -INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); -SELECT a['key2'] FROM table_map; -``` -Result: - -```text -┌─arrayElement(a, 'key2')─┐ -│ 100 │ -│ 200 │ -│ 300 │ -└─────────────────────────┘ -``` - -## Convert Tuple to Map Type {#map-and-tuple} - -You can cast `Tuple()` as `Map()` using [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function: - -``` sql -SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; -``` - -``` text -┌─map───────────────────────────┐ -│ {1:'Ready',2:'Steady',3:'Go'} │ -└───────────────────────────────┘ -``` - -**See Also** - -- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function -- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function - -[Original article](https://clickhouse.tech/docs/en/data-types/map/) diff --git a/docs/ja/sql-reference/data-types/map.md b/docs/ja/sql-reference/data-types/map.md deleted file mode 100644 index 0f0f69d421d..00000000000 --- a/docs/ja/sql-reference/data-types/map.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -toc_priority: 65 -toc_title: Map(key, value) ---- - -# Map(key, value) {#data_type-map} - -`Map(key, value)` data type stores `key:value` pairs in structures like JSON. - -**Parameters** -- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). -- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). - -!!! warning "Warning" - Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. - -To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. - -**Example** - -Query: - -``` sql -CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; -INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); -SELECT a['key2'] FROM table_map; -``` -Result: - -```text -┌─arrayElement(a, 'key2')─┐ -│ 100 │ -│ 200 │ -│ 300 │ -└─────────────────────────┘ -``` - -## Convert Tuple to Map Type {#map-and-tuple} - -You can cast `Tuple()` as `Map()` using [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function: - -``` sql -SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; -``` - -``` text -┌─map───────────────────────────┐ -│ {1:'Ready',2:'Steady',3:'Go'} │ -└───────────────────────────────┘ -``` - -**See Also** - -- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function -- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function - -[Original article](https://clickhouse.tech/docs/en/data-types/map/) diff --git a/docs/zh/sql-reference/data-types/map.md b/docs/zh/sql-reference/data-types/map.md deleted file mode 100644 index 0f0f69d421d..00000000000 --- a/docs/zh/sql-reference/data-types/map.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -toc_priority: 65 -toc_title: Map(key, value) ---- - -# Map(key, value) {#data_type-map} - -`Map(key, value)` data type stores `key:value` pairs in structures like JSON. - -**Parameters** -- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). -- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). - -!!! warning "Warning" - Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. - -To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. - -**Example** - -Query: - -``` sql -CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; -INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); -SELECT a['key2'] FROM table_map; -``` -Result: - -```text -┌─arrayElement(a, 'key2')─┐ -│ 100 │ -│ 200 │ -│ 300 │ -└─────────────────────────┘ -``` - -## Convert Tuple to Map Type {#map-and-tuple} - -You can cast `Tuple()` as `Map()` using [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function: - -``` sql -SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; -``` - -``` text -┌─map───────────────────────────┐ -│ {1:'Ready',2:'Steady',3:'Go'} │ -└───────────────────────────────┘ -``` - -**See Also** - -- [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function -- [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function - -[Original article](https://clickhouse.tech/docs/en/data-types/map/) From 409ff2f6b3f7b16cd9c15cca48b3332574bd8cd5 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 4 Feb 2021 22:13:55 +0300 Subject: [PATCH 05/30] Document system.opentelemetry_span_log system table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Задокументировал системную таблицу system.opentelemetry_span_log. --- .../system-tables/opentelemetry_span_log.md | 49 +++++++++++++++++++ .../system-tables/opentelemetry_span_log.md | 45 +++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 docs/en/operations/system-tables/opentelemetry_span_log.md create mode 100644 docs/ru/operations/system-tables/opentelemetry_span_log.md diff --git a/docs/en/operations/system-tables/opentelemetry_span_log.md b/docs/en/operations/system-tables/opentelemetry_span_log.md new file mode 100644 index 00000000000..64fd549458a --- /dev/null +++ b/docs/en/operations/system-tables/opentelemetry_span_log.md @@ -0,0 +1,49 @@ +# system.opentelemetry_span_log {#system_tables-opentelemetry_span_log} + +Contains information about [trace spans](https://opentracing.io/docs/overview/spans/) for executed queries. + +Columns: + +- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md) — ID of the trace for executed query. + +- `span_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — ID of the `trace span`. + +- `parent_span_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — ID of the parent `trace span`. + +- `operation_name` ([String](../../sql-reference/data-types/string.md)) — The name of the operation. + +- `start_time_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The start time of the `trace span` (in microseconds). + +- `finish_time_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The finish time of the `trace span` (in microseconds). + +- `finish_date` ([Date](../../sql-reference/data-types/date.md)) — The finish date of the `trace span`. + +- `attribute.names` ([Array(String)](../../sql-reference/data-types/array.md)) — [Attribute](https://opentelemetry.io/docs/go/instrumentation/#attributes) names depending on the `trace span`. They are filled in according to the recommendations in the [OpenTelemetry](https://opentelemetry.io/) standard. + +- `attribute.values` ([Array(String)](../../sql-reference/data-types/array.md)) — Attribute values depending on the `trace span`. They are filled in according to the recommendations in the `OpenTelemetry` standard. + +**Example** + +``` sql +SELECT * FROM system.opentelemetry_span_log LIMIT 1 FORMAT Vertical; +``` + +``` text +Row 1: +────── +trace_id: cdab0847-0d62-61d5-4d38-dd65b19a1914 +span_id: 701487461015578150 +parent_span_id: 2991972114672045096 +operation_name: DB::Block DB::InterpreterSelectQuery::getSampleBlockImpl() +start_time_us: 1612374594529090 +finish_time_us: 1612374594529108 +finish_date: 2021-02-03 +attribute.names: [] +attribute.values: [] +``` + +**See Also** + +- [OpenTelemetry](../../operations/opentelemetry.md) + +[Original article](https://clickhouse.tech/docs/en/operations/system_tables/opentelemetry_span_log) diff --git a/docs/ru/operations/system-tables/opentelemetry_span_log.md b/docs/ru/operations/system-tables/opentelemetry_span_log.md new file mode 100644 index 00000000000..5c577eb691d --- /dev/null +++ b/docs/ru/operations/system-tables/opentelemetry_span_log.md @@ -0,0 +1,45 @@ +# system.opentelemetry_span_log {#system_tables-opentelemetry_span_log} + +Содержит информацию о [trace spans](https://opentracing.io/docs/overview/spans/) для выполненных запросов. + +Столбцы: + +- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md) — идентификатор трассировки для выполненного запроса. + +- `span_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — идентификатор `trace span`. + +- `parent_span_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — идентификатор родительского `trace span`. + +- `operation_name` ([String](../../sql-reference/data-types/string.md)) — имя операции. + +- `start_time_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — время начала `trace span` (в микросекундах). + +- `finish_time_us` ([UInt64](../../sql-reference/data-types/int-uint.md)) — время окончания `trace span` (в микросекундах). + +- `finish_date` ([Date](../../sql-reference/data-types/date.md)) — дата окончания `trace span`. + +- `attribute.names` ([Array(String)](../../sql-reference/data-types/array.md)) — имена [атрибутов](https://opentelemetry.io/docs/go/instrumentation/#attributes) в зависимости от `trace span`. Заполняются согласно рекомендациям в стандарте [OpenTelemetry](https://opentelemetry.io/). + +- `attribute.values` ([Array(String)](../../sql-reference/data-types/array.md)) — значения атрибутов в зависимости от `trace span`. Заполняются согласно рекомендациям в стандарте `OpenTelemetry`. + +**Пример** + +``` sql +SELECT * FROM system.opentelemetry_span_log LIMIT 1 FORMAT Vertical; +``` + +``` text +Row 1: +────── +trace_id: cdab0847-0d62-61d5-4d38-dd65b19a1914 +span_id: 701487461015578150 +parent_span_id: 2991972114672045096 +operation_name: DB::Block DB::InterpreterSelectQuery::getSampleBlockImpl() +start_time_us: 1612374594529090 +finish_time_us: 1612374594529108 +finish_date: 2021-02-03 +attribute.names: [] +attribute.values: [] +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/opentelemetry_span_log) From 27933e714b956e34a404f1519b7397f3f93d2d7c Mon Sep 17 00:00:00 2001 From: Marquitos Date: Fri, 5 Feb 2021 17:39:05 +0100 Subject: [PATCH 06/30] Add 'access_management' configuration to initial setup --- docker/server/README.md | 8 ++++---- docker/server/entrypoint.sh | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/server/README.md b/docker/server/README.md index d8e9204dffa..6f799d68185 100644 --- a/docker/server/README.md +++ b/docker/server/README.md @@ -56,7 +56,7 @@ $ echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @- 20.12.3.3 ``` -### Volumes +### Volumes Typically you may want to mount the following folders inside your container to archieve persistency: @@ -76,7 +76,7 @@ You may also want to mount: * `/etc/clickhouse-server/usert.d/*.xml` - files with use settings adjustmenets * `/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below). -### Linux capabilities +### Linux capabilities ClickHouse has some advanced functionality which requite enabling several [linux capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html). @@ -113,10 +113,10 @@ $ docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-serv ### How to create default database and user on starting -Sometimes you may want to create user (user named `default` is used by default) and database on image starting. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD`: +Sometimes you may want to create user (user named `default` is used by default) and database on image starting. You can do it using environment variables `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT` and `CLICKHOUSE_PASSWORD`: ``` -$ docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLICKHOUSE_PASSWORD=password -p 9000:9000/tcp yandex/clickhouse-server +$ docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=password -p 9000:9000/tcp yandex/clickhouse-server ``` ## How to extend this image diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 549ff601c59..0138a165505 100755 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -54,6 +54,7 @@ FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_ CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" CLICKHOUSE_DB="${CLICKHOUSE_DB:-}" +CLICKHOUSE_ACCESS_MANAGEMENT="${CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT:-0}" for dir in "$DATA_DIR" \ "$ERROR_LOG_DIR" \ @@ -97,6 +98,7 @@ if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CL ${CLICKHOUSE_PASSWORD} default + ${CLICKHOUSE_ACCESS_MANAGEMENT} From f0370b241c341ce961bac516afbd909631ec6b3d Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sat, 6 Feb 2021 20:17:25 +0300 Subject: [PATCH 07/30] Document the opentelemetry_start_trace_probability setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Задокументировал настройку. --- docs/en/operations/settings/settings.md | 11 +++++++++++ docs/ru/operations/settings/settings.md | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index edfd391c71e..869c76fb975 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2592,4 +2592,15 @@ Possible values: Default value: `16`. +## opentelemetry_start_trace_probability {#opentelemetry-start-trace-probability} + +Enables a trace for executed queries. + +Possible values: + +- 0 — The trace for a executed query is disabled. +- 1 — The trace for a executed query is enabled. + +Default value: `0`. + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index bacc97dfd14..2aa81daa0b0 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2473,4 +2473,15 @@ SELECT SUM(-1), MAX(0) FROM system.one WHERE 0; Значение по умолчанию: `16`. +## opentelemetry_start_trace_probability {#opentelemetry-start-trace-probability} + +Включает трассировку для выполненных запросов. + +Возможные значения: + +- 0 — трассировка для выполненного запроса отключена. +- 1 — трассировка для выполненного запроса включена. + +Значение по умолчанию: `0`. + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) From ed59b355c0dba42da612546a584b0645ef463019 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 9 Feb 2021 20:34:16 +0300 Subject: [PATCH 08/30] Update the description of the opentelemetry_start_trace_probability setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Обновил документацию настройки. --- docs/en/operations/settings/settings.md | 7 ++++--- docs/ru/operations/settings/settings.md | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 869c76fb975..0554ea79ecd 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2594,12 +2594,13 @@ Default value: `16`. ## opentelemetry_start_trace_probability {#opentelemetry-start-trace-probability} -Enables a trace for executed queries. +Sets the probability that the ClickHouse can start a trace for executed queries (if no parent [trace context](https://www.w3.org/TR/trace-context/) is supplied). Possible values: -- 0 — The trace for a executed query is disabled. -- 1 — The trace for a executed query is enabled. +- 0 — The trace for a executed queries is disabled (if no parent trace context is supplied). +- (0, 1) — The probability with which the ClickHouse can start a trace for executed queries (if no parent trace context is supplied). For example, if the setting value is `0,5`, ClickHouse can start a trace on average for half of the queries. +- 1 — The trace for all executed queries is enabled. Default value: `0`. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 2aa81daa0b0..47e2666e652 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2475,12 +2475,13 @@ SELECT SUM(-1), MAX(0) FROM system.one WHERE 0; ## opentelemetry_start_trace_probability {#opentelemetry-start-trace-probability} -Включает трассировку для выполненных запросов. +Задает вероятность того, что ClickHouse начнет трассировку для выполненных запросов (если не указан [родительский контекст](https://www.w3.org/TR/trace-context/) трассировки). Возможные значения: -- 0 — трассировка для выполненного запроса отключена. -- 1 — трассировка для выполненного запроса включена. +- 0 — трассировка для выполненных запросов отключена (если не указан родительский контекст трассировки). +- (0, 1) — вероятность, с которой ClickHouse начнет трассировку для выполненных запросов (если не указан родительский контекст трассировки). Например, при значении настройки, равной `0,5`, ClickHouse начнет трассировку в среднем для половины запросов. +- 1 — трассировка для всех выполненных запросов включена. Значение по умолчанию: `0`. From 2905df831f9119d414c44a8eedd8df9012825889 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Thu, 11 Feb 2021 10:15:18 +0300 Subject: [PATCH 09/30] JSON deteted --- docs/en/sql-reference/data-types/map.md | 40 +++++++++++++--- .../functions/tuple-map-functions.md | 4 +- docs/ru/sql-reference/data-types/map.md | 46 ++++++++++++------- .../functions/tuple-map-functions.md | 4 +- 4 files changed, 66 insertions(+), 28 deletions(-) diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md index 0f0f69d421d..58634e5b669 100644 --- a/docs/en/sql-reference/data-types/map.md +++ b/docs/en/sql-reference/data-types/map.md @@ -5,7 +5,7 @@ toc_title: Map(key, value) # Map(key, value) {#data_type-map} -`Map(key, value)` data type stores `key:value` pairs in structures like JSON. +`Map(key, value)` data type stores `key:value` pairs. **Parameters** - `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). @@ -14,24 +14,50 @@ toc_title: Map(key, value) !!! warning "Warning" Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. -To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. +To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. This lookup works now with a linear complexity. -**Example** +**Examples** -Query: +Consider the table: ``` sql CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; -INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); +INSERT INTO table_map VALUES ({'key1':1, 'key2':10}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30}); +``` + +Select all `key2` values: + +```sql SELECT a['key2'] FROM table_map; ``` Result: ```text ┌─arrayElement(a, 'key2')─┐ +│ 10 │ +│ 20 │ +│ 30 │ +└─────────────────────────┘ +``` + +If there's no such `key` in the `Map()` column, the query returns zeros for numerical values, empty strings or empty arrays. + +```sql +INSERT INTO table_map VALUES ({'key3':100}), ({}); +SELECT a['key3'] FROM table_map; +``` + +Result: + +```text +┌─arrayElement(a, 'key3')─┐ │ 100 │ -│ 200 │ -│ 300 │ +│ 0 │ +└─────────────────────────┘ +┌─arrayElement(a, 'key3')─┐ +│ 0 │ +│ 0 │ +│ 0 │ └─────────────────────────┘ ``` diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index b81f971196a..18d008f11f2 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -7,7 +7,7 @@ toc_title: Working with maps ## map {#function-map} -Arranges `key:value` pairs into a JSON data structure. +Arranges `key:value` pairs into [Map(key, value)](../../sql-reference/data-types/map.md) data type. **Syntax** @@ -22,7 +22,7 @@ map(key1, value1[, key2, value2, ...]) **Returned value** -- JSON with `key:value` pairs. +- Data structure as `key:value` pairs. Type: [Map(key, value)](../../sql-reference/data-types/map.md). diff --git a/docs/ru/sql-reference/data-types/map.md b/docs/ru/sql-reference/data-types/map.md index c1391e37133..9c2ffedc4a9 100644 --- a/docs/ru/sql-reference/data-types/map.md +++ b/docs/ru/sql-reference/data-types/map.md @@ -5,7 +5,7 @@ toc_title: Map(key, value) # Map(key, value) {#data_type-map} -Тип данных `Map(key, value)` хранит пары `ключ:значение` в структурах типа JSON. +Тип данных `Map(key, value)` хранит пары `ключ:значение`. **Параметры** - `key` — ключ. [String](../../sql-reference/data-types/string.md) или [Integer](../../sql-reference/data-types/int-uint.md). @@ -14,39 +14,51 @@ toc_title: Map(key, value) !!! warning "Предупреждение" Сейчас использование типа данных `Map` является экспериментальной возможностью. Чтобы использовать этот тип данных, включите настройку `allow_experimental_map_type = 1`. -Чтобы получить значение из колонки `a Map('key', 'value')`, используйте синтаксис `a['key']`. +Чтобы получить значение из колонки `a Map('key', 'value')`, используйте синтаксис `a['key']`. В настоящее время такая подстановка работает по алгоритму с линейной сложностью. -**Пример** +**Примеры** -Запрос: +Рассмотрим таблицу: ``` sql CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; -INSERT INTO table_map VALUES ({'key1':1, 'key2':100}), ({'key1':2,'key2':200}), ({'key1':3,'key2':300}); +INSERT INTO table_map VALUES ({'key1':1, 'key2':10}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30}); +``` + +Выборка всем значений ключа `key2`: + +```sql SELECT a['key2'] FROM table_map; ``` Результат: ```text ┌─arrayElement(a, 'key2')─┐ -│ 100 │ -│ 200 │ -│ 300 │ +│ 10 │ +│ 20 │ +│ 30 │ └─────────────────────────┘ ``` -## Преобразование типа данных Tuple в Map {#map-and-tuple} +Если для какого-то ключа `key` в колонке с типом `Map()` нет значения, запрос возвращает нули для числовых колонок, пустые строки или пустые массивы. -Для преобразования данных с типом `Tuple()` в тип `Map()` можно использовать функцию [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast): - -``` sql -SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map; +```sql +INSERT INTO table_map VALUES ({'key3':100}), ({}); +SELECT a['key3'] FROM table_map; ``` -``` text -┌─map───────────────────────────┐ -│ {1:'Ready',2:'Steady',3:'Go'} │ -└───────────────────────────────┘ +Результат: + +```text +┌─arrayElement(a, 'key3')─┐ +│ 100 │ +│ 0 │ +└─────────────────────────┘ +┌─arrayElement(a, 'key3')─┐ +│ 0 │ +│ 0 │ +│ 0 │ +└─────────────────────────┘ ``` **См. также** diff --git a/docs/ru/sql-reference/functions/tuple-map-functions.md b/docs/ru/sql-reference/functions/tuple-map-functions.md index 65e44698008..a36613280a1 100644 --- a/docs/ru/sql-reference/functions/tuple-map-functions.md +++ b/docs/ru/sql-reference/functions/tuple-map-functions.md @@ -7,7 +7,7 @@ toc_title: Работа с контейнерами map ## map {#function-map} -Преобразовывает пары `ключ:значение` в структуру JSON. +Преобразовывает пары `ключ:значение` в тип данных [Map(key, value)](../../sql-reference/data-types/map.md). **Синтаксис** @@ -22,7 +22,7 @@ map(key1, value1[, key2, value2, ...]) **Возвращаемое значение** -- Структура JSON с парами `ключ:значение`. +- Структура данных в виде пар `ключ:значение`. Тип: [Map(key, value)](../../sql-reference/data-types/map.md). From d3549aca95c1bcdc2b65617afd35f71ee51be4a9 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 11 Feb 2021 21:42:15 +0300 Subject: [PATCH 10/30] Fix the description of the table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправил описание таблицы. --- .../en/operations/system-tables/opentelemetry_span_log.md | 8 ++++++-- .../ru/operations/system-tables/opentelemetry_span_log.md | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/system-tables/opentelemetry_span_log.md b/docs/en/operations/system-tables/opentelemetry_span_log.md index 64fd549458a..e45a989742c 100644 --- a/docs/en/operations/system-tables/opentelemetry_span_log.md +++ b/docs/en/operations/system-tables/opentelemetry_span_log.md @@ -18,16 +18,20 @@ Columns: - `finish_date` ([Date](../../sql-reference/data-types/date.md)) — The finish date of the `trace span`. -- `attribute.names` ([Array(String)](../../sql-reference/data-types/array.md)) — [Attribute](https://opentelemetry.io/docs/go/instrumentation/#attributes) names depending on the `trace span`. They are filled in according to the recommendations in the [OpenTelemetry](https://opentelemetry.io/) standard. +- `attribute.names` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — [Attribute](https://opentelemetry.io/docs/go/instrumentation/#attributes) names depending on the `trace span`. They are filled in according to the recommendations in the [OpenTelemetry](https://opentelemetry.io/) standard. -- `attribute.values` ([Array(String)](../../sql-reference/data-types/array.md)) — Attribute values depending on the `trace span`. They are filled in according to the recommendations in the `OpenTelemetry` standard. +- `attribute.values` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Attribute values depending on the `trace span`. They are filled in according to the recommendations in the `OpenTelemetry` standard. **Example** +Query: + ``` sql SELECT * FROM system.opentelemetry_span_log LIMIT 1 FORMAT Vertical; ``` +Result: + ``` text Row 1: ────── diff --git a/docs/ru/operations/system-tables/opentelemetry_span_log.md b/docs/ru/operations/system-tables/opentelemetry_span_log.md index 5c577eb691d..96555064b0e 100644 --- a/docs/ru/operations/system-tables/opentelemetry_span_log.md +++ b/docs/ru/operations/system-tables/opentelemetry_span_log.md @@ -18,16 +18,20 @@ - `finish_date` ([Date](../../sql-reference/data-types/date.md)) — дата окончания `trace span`. -- `attribute.names` ([Array(String)](../../sql-reference/data-types/array.md)) — имена [атрибутов](https://opentelemetry.io/docs/go/instrumentation/#attributes) в зависимости от `trace span`. Заполняются согласно рекомендациям в стандарте [OpenTelemetry](https://opentelemetry.io/). +- `attribute.names` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — имена [атрибутов](https://opentelemetry.io/docs/go/instrumentation/#attributes) в зависимости от `trace span`. Заполняются согласно рекомендациям в стандарте [OpenTelemetry](https://opentelemetry.io/). -- `attribute.values` ([Array(String)](../../sql-reference/data-types/array.md)) — значения атрибутов в зависимости от `trace span`. Заполняются согласно рекомендациям в стандарте `OpenTelemetry`. +- `attribute.values` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — значения атрибутов в зависимости от `trace span`. Заполняются согласно рекомендациям в стандарте `OpenTelemetry`. **Пример** +Запрос: + ``` sql SELECT * FROM system.opentelemetry_span_log LIMIT 1 FORMAT Vertical; ``` +Результат: + ``` text Row 1: ────── From ce1524c4ebaca545feeaa1493d5ae8e66af8dab9 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Thu, 11 Feb 2021 22:06:30 +0300 Subject: [PATCH 11/30] Update docs/en/operations/settings/settings.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 0554ea79ecd..8f1cb186449 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2599,7 +2599,7 @@ Sets the probability that the ClickHouse can start a trace for executed queries Possible values: - 0 — The trace for a executed queries is disabled (if no parent trace context is supplied). -- (0, 1) — The probability with which the ClickHouse can start a trace for executed queries (if no parent trace context is supplied). For example, if the setting value is `0,5`, ClickHouse can start a trace on average for half of the queries. +- Positive floating-point number in the range [0..1]. For example, if the setting value is `0,5`, ClickHouse can start a trace on average for half of the queries. - 1 — The trace for all executed queries is enabled. Default value: `0`. From 6271709efacad598431127808dae44cd1ac6e0bb Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 11 Feb 2021 22:23:19 +0300 Subject: [PATCH 12/30] Fix the description of the setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправил описание настройки. --- docs/en/operations/settings/settings.md | 2 +- docs/ru/operations/settings/settings.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 8f1cb186449..6f028b00a5b 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2599,7 +2599,7 @@ Sets the probability that the ClickHouse can start a trace for executed queries Possible values: - 0 — The trace for a executed queries is disabled (if no parent trace context is supplied). -- Positive floating-point number in the range [0..1]. For example, if the setting value is `0,5`, ClickHouse can start a trace on average for half of the queries. +- Positive floating-point number in the range [0..1]. For example, if the setting value is `0,5`, ClickHouse can start a trace on average for half of the queries. - 1 — The trace for all executed queries is enabled. Default value: `0`. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 47e2666e652..434157401fa 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2475,12 +2475,12 @@ SELECT SUM(-1), MAX(0) FROM system.one WHERE 0; ## opentelemetry_start_trace_probability {#opentelemetry-start-trace-probability} -Задает вероятность того, что ClickHouse начнет трассировку для выполненных запросов (если не указан [родительский контекст](https://www.w3.org/TR/trace-context/) трассировки). +Задает вероятность того, что ClickHouse начнет трассировку для выполненных запросов (если не указан [входящий контекст](https://www.w3.org/TR/trace-context/) трассировки). Возможные значения: -- 0 — трассировка для выполненных запросов отключена (если не указан родительский контекст трассировки). -- (0, 1) — вероятность, с которой ClickHouse начнет трассировку для выполненных запросов (если не указан родительский контекст трассировки). Например, при значении настройки, равной `0,5`, ClickHouse начнет трассировку в среднем для половины запросов. +- 0 — трассировка для выполненных запросов отключена (если не указан входящий контекст трассировки). +- Положительное число с плавающей точкой в диапазоне [0..1]. Например, при значении настройки, равной `0,5`, ClickHouse начнет трассировку в среднем для половины запросов. - 1 — трассировка для всех выполненных запросов включена. Значение по умолчанию: `0`. From bb2a11bcfd94c525238a768ac10bdeaa1fb1d2b5 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Fri, 12 Feb 2021 07:43:33 +0300 Subject: [PATCH 13/30] Misspelling --- docs/ru/sql-reference/data-types/map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/data-types/map.md b/docs/ru/sql-reference/data-types/map.md index 9c2ffedc4a9..6cb8ccf1143 100644 --- a/docs/ru/sql-reference/data-types/map.md +++ b/docs/ru/sql-reference/data-types/map.md @@ -25,7 +25,7 @@ CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory; INSERT INTO table_map VALUES ({'key1':1, 'key2':10}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30}); ``` -Выборка всем значений ключа `key2`: +Выборка всех значений ключа `key2`: ```sql SELECT a['key2'] FROM table_map; From 69d4120982fa2b7cae35da83532c8318f44bfc8f Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 12 Feb 2021 10:22:18 +0800 Subject: [PATCH 14/30] Disable table function view in expression --- src/Parsers/ASTFunction.cpp | 8 ++++++++ .../0_stateless/01715_table_function_view_fix.reference | 0 .../queries/0_stateless/01715_table_function_view_fix.sql | 1 + 3 files changed, 9 insertions(+) create mode 100644 tests/queries/0_stateless/01715_table_function_view_fix.reference create mode 100644 tests/queries/0_stateless/01715_table_function_view_fix.sql diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 806b8e6c5b9..29ac01eefc5 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -15,8 +15,16 @@ namespace DB { +namespace ErrorCodes +{ + extern const int UNEXPECTED_EXPRESSION; +} + void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const { + if (name == "view") + throw Exception("Table function view cannot be used as an expression", ErrorCodes::UNEXPECTED_EXPRESSION); + writeString(name, ostr); if (parameters) diff --git a/tests/queries/0_stateless/01715_table_function_view_fix.reference b/tests/queries/0_stateless/01715_table_function_view_fix.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01715_table_function_view_fix.sql b/tests/queries/0_stateless/01715_table_function_view_fix.sql new file mode 100644 index 00000000000..21da116f6ba --- /dev/null +++ b/tests/queries/0_stateless/01715_table_function_view_fix.sql @@ -0,0 +1 @@ +SELECT view(SELECT 1); -- { serverError 183 } From a551edd8d6e308569433a9158df1ee31a60844de Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 13 Feb 2021 13:18:14 +0800 Subject: [PATCH 15/30] Do not parse view function in expression --- src/Parsers/ASTFunction.cpp | 8 -- src/Parsers/ExpressionElementParsers.cpp | 81 ++++++++++++------- src/Parsers/ExpressionElementParsers.h | 16 +++- src/Parsers/ExpressionListParsers.cpp | 17 +++- src/Parsers/ExpressionListParsers.h | 22 ++++- src/Parsers/ParserTablesInSelectQuery.cpp | 2 +- .../01715_table_function_view_fix.sql | 2 +- 7 files changed, 98 insertions(+), 50 deletions(-) diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 29ac01eefc5..806b8e6c5b9 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -15,16 +15,8 @@ namespace DB { -namespace ErrorCodes -{ - extern const int UNEXPECTED_EXPRESSION; -} - void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const { - if (name == "view") - throw Exception("Table function view cannot be used as an expression", ErrorCodes::UNEXPECTED_EXPRESSION); - writeString(name, ostr); if (parameters) diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index e7cd85798b9..3d868812304 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -266,7 +266,7 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserIdentifier id_parser; ParserKeyword distinct("DISTINCT"); ParserKeyword all("ALL"); - ParserExpressionList contents(false); + ParserExpressionList contents(false, is_table_function); ParserSelectWithUnionQuery select; ParserKeyword over("OVER"); @@ -278,6 +278,12 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr expr_list_args; ASTPtr expr_list_params; + if (is_table_function) + { + if (ParserTableFunctionView().parse(pos, node, expected)) + return true; + } + if (!id_parser.parse(pos, identifier, expected)) return false; @@ -312,36 +318,6 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } } - if (!has_distinct && !has_all) - { - auto old_pos = pos; - auto maybe_an_subquery = pos->type == TokenType::OpeningRoundBracket; - - if (select.parse(pos, query, expected)) - { - auto & select_ast = query->as(); - if (select_ast.list_of_selects->children.size() == 1 && maybe_an_subquery) - { - // It's an subquery. Bail out. - pos = old_pos; - } - else - { - if (pos->type != TokenType::ClosingRoundBracket) - return false; - ++pos; - auto function_node = std::make_shared(); - tryGetIdentifierNameInto(identifier, function_node->name); - auto expr_list_with_single_query = std::make_shared(); - expr_list_with_single_query->children.push_back(query); - function_node->arguments = expr_list_with_single_query; - function_node->children.push_back(function_node->arguments); - node = function_node; - return true; - } - } - } - const char * contents_begin = pos->begin; if (!contents.parse(pos, expr_list_args, expected)) return false; @@ -477,6 +453,49 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return true; } +bool ParserTableFunctionView::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserIdentifier id_parser; + ParserKeyword view("VIEW"); + ParserSelectWithUnionQuery select; + + ASTPtr identifier; + ASTPtr query; + + if (!view.ignore(pos, expected)) + return false; + + if (pos->type != TokenType::OpeningRoundBracket) + return false; + + ++pos; + + bool maybe_an_subquery = pos->type == TokenType::OpeningRoundBracket; + + if (!select.parse(pos, query, expected)) + return false; + + auto & select_ast = query->as(); + if (select_ast.list_of_selects->children.size() == 1 && maybe_an_subquery) + { + // It's an subquery. Bail out. + return false; + } + + if (pos->type != TokenType::ClosingRoundBracket) + return false; + ++pos; + auto function_node = std::make_shared(); + tryGetIdentifierNameInto(identifier, function_node->name); + auto expr_list_with_single_query = std::make_shared(); + expr_list_with_single_query->children.push_back(query); + function_node->name = "view"; + function_node->arguments = expr_list_with_single_query; + function_node->children.push_back(function_node->arguments); + node = function_node; + return true; +} + bool ParserWindowReference::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTFunction * function = dynamic_cast(node.get()); diff --git a/src/Parsers/ExpressionElementParsers.h b/src/Parsers/ExpressionElementParsers.h index ba18fc2cddd..b6194f981fe 100644 --- a/src/Parsers/ExpressionElementParsers.h +++ b/src/Parsers/ExpressionElementParsers.h @@ -149,11 +149,25 @@ protected: class ParserFunction : public IParserBase { public: - ParserFunction(bool allow_function_parameters_ = true) : allow_function_parameters(allow_function_parameters_) {} + ParserFunction(bool allow_function_parameters_ = true, bool is_table_function_ = false) + : allow_function_parameters(allow_function_parameters_), is_table_function(is_table_function_) + { + } + protected: const char * getName() const override { return "function"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; bool allow_function_parameters; + bool is_table_function; +}; + +// A special function parser for view table function. +// It parses an SELECT query as its argument and doesn't support getColumnName(). +class ParserTableFunctionView : public IParserBase +{ +protected: + const char * getName() const override { return "function"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; // Window reference (the thing that goes after OVER) for window function. diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index afe85f069c7..e9ad65af471 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -468,6 +468,14 @@ bool ParserLambdaExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expe } +bool ParserTableFunctionExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + if (ParserTableFunctionView().parse(pos, node, expected)) + return true; + return elem_parser.parse(pos, node, expected); +} + + bool ParserPrefixUnaryOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { /// try to find any of the valid operators @@ -570,9 +578,10 @@ bool ParserTupleElementExpression::parseImpl(Pos & pos, ASTPtr & node, Expected } -ParserExpressionWithOptionalAlias::ParserExpressionWithOptionalAlias(bool allow_alias_without_as_keyword) - : impl(std::make_unique(std::make_unique(), - allow_alias_without_as_keyword)) +ParserExpressionWithOptionalAlias::ParserExpressionWithOptionalAlias(bool allow_alias_without_as_keyword, bool is_table_function) + : impl(std::make_unique( + is_table_function ? ParserPtr(std::make_unique()) : ParserPtr(std::make_unique()), + allow_alias_without_as_keyword)) { } @@ -580,7 +589,7 @@ ParserExpressionWithOptionalAlias::ParserExpressionWithOptionalAlias(bool allow_ bool ParserExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { return ParserList( - std::make_unique(allow_alias_without_as_keyword), + std::make_unique(allow_alias_without_as_keyword, is_table_function), std::make_unique(TokenType::Comma)) .parse(pos, node, expected); } diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index 90b27950873..2371e006c09 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -436,13 +436,26 @@ protected: }; +// It's used to parse expressions in table function. +class ParserTableFunctionExpression : public IParserBase +{ +private: + ParserLambdaExpression elem_parser; + +protected: + const char * getName() const override { return "table function expression"; } + + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + + using ParserExpression = ParserLambdaExpression; class ParserExpressionWithOptionalAlias : public IParserBase { public: - ParserExpressionWithOptionalAlias(bool allow_alias_without_as_keyword); + explicit ParserExpressionWithOptionalAlias(bool allow_alias_without_as_keyword, bool is_table_function = false); protected: ParserPtr impl; @@ -459,11 +472,12 @@ protected: class ParserExpressionList : public IParserBase { public: - ParserExpressionList(bool allow_alias_without_as_keyword_) - : allow_alias_without_as_keyword(allow_alias_without_as_keyword_) {} + explicit ParserExpressionList(bool allow_alias_without_as_keyword_, bool is_table_function_ = false) + : allow_alias_without_as_keyword(allow_alias_without_as_keyword_), is_table_function(is_table_function_) {} protected: bool allow_alias_without_as_keyword; + bool is_table_function; // This expression list is used by a table function const char * getName() const override { return "list of expressions"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; @@ -473,7 +487,7 @@ protected: class ParserNotEmptyExpressionList : public IParserBase { public: - ParserNotEmptyExpressionList(bool allow_alias_without_as_keyword) + explicit ParserNotEmptyExpressionList(bool allow_alias_without_as_keyword) : nested_parser(allow_alias_without_as_keyword) {} private: ParserExpressionList nested_parser; diff --git a/src/Parsers/ParserTablesInSelectQuery.cpp b/src/Parsers/ParserTablesInSelectQuery.cpp index 1264acefe64..2e20279dbe1 100644 --- a/src/Parsers/ParserTablesInSelectQuery.cpp +++ b/src/Parsers/ParserTablesInSelectQuery.cpp @@ -22,7 +22,7 @@ bool ParserTableExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expec auto res = std::make_shared(); if (!ParserWithOptionalAlias(std::make_unique(), true).parse(pos, res->subquery, expected) - && !ParserWithOptionalAlias(std::make_unique(), true).parse(pos, res->table_function, expected) + && !ParserWithOptionalAlias(std::make_unique(true, true), true).parse(pos, res->table_function, expected) && !ParserWithOptionalAlias(std::make_unique(false, true), true).parse(pos, res->database_and_table_name, expected)) return false; diff --git a/tests/queries/0_stateless/01715_table_function_view_fix.sql b/tests/queries/0_stateless/01715_table_function_view_fix.sql index 21da116f6ba..de5150b7b70 100644 --- a/tests/queries/0_stateless/01715_table_function_view_fix.sql +++ b/tests/queries/0_stateless/01715_table_function_view_fix.sql @@ -1 +1 @@ -SELECT view(SELECT 1); -- { serverError 183 } +SELECT view(SELECT 1); -- { clientError 62 } From ed49367fc750d0d50edaa4dde3cc7cb56598c305 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 14 Feb 2021 14:20:23 +0800 Subject: [PATCH 16/30] Fix global-with with subqueries --- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 8 ++++++-- src/Interpreters/InterpreterSelectWithUnionQuery.h | 3 ++- src/Interpreters/getTableExpressions.cpp | 2 +- .../0_stateless/01717_global_with_subquery_fix.reference | 0 .../0_stateless/01717_global_with_subquery_fix.sql | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/01717_global_with_subquery_fix.reference create mode 100644 tests/queries/0_stateless/01717_global_with_subquery_fix.sql diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index e6610df43ff..59fcff61936 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -329,7 +329,7 @@ InterpreterSelectWithUnionQuery::buildCurrentChildInterpreter(const ASTPtr & ast InterpreterSelectWithUnionQuery::~InterpreterSelectWithUnionQuery() = default; -Block InterpreterSelectWithUnionQuery::getSampleBlock(const ASTPtr & query_ptr_, const Context & context_) +Block InterpreterSelectWithUnionQuery::getSampleBlock(const ASTPtr & query_ptr_, const Context & context_, bool is_subquery) { auto & cache = context_.getSampleBlockCache(); /// Using query string because query_ptr changes for every internal SELECT @@ -339,7 +339,11 @@ Block InterpreterSelectWithUnionQuery::getSampleBlock(const ASTPtr & query_ptr_, return cache[key]; } - return cache[key] = InterpreterSelectWithUnionQuery(query_ptr_, context_, SelectQueryOptions().analyze()).getSampleBlock(); + if (is_subquery) + return cache[key] + = InterpreterSelectWithUnionQuery(query_ptr_, context_, SelectQueryOptions().subquery().analyze()).getSampleBlock(); + else + return cache[key] = InterpreterSelectWithUnionQuery(query_ptr_, context_, SelectQueryOptions().analyze()).getSampleBlock(); } diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.h b/src/Interpreters/InterpreterSelectWithUnionQuery.h index cd089a51970..f4062b2005e 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.h +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.h @@ -35,7 +35,8 @@ public: static Block getSampleBlock( const ASTPtr & query_ptr_, - const Context & context_); + const Context & context_, + bool is_subquery = false); virtual void ignoreWithTotals() override; diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index 766ce257530..a4e971c302c 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -84,7 +84,7 @@ static NamesAndTypesList getColumnsFromTableExpression( if (table_expression.subquery) { const auto & subquery = table_expression.subquery->children.at(0); - names_and_type_list = InterpreterSelectWithUnionQuery::getSampleBlock(subquery, context).getNamesAndTypesList(); + names_and_type_list = InterpreterSelectWithUnionQuery::getSampleBlock(subquery, context, true).getNamesAndTypesList(); } else if (table_expression.table_function) { diff --git a/tests/queries/0_stateless/01717_global_with_subquery_fix.reference b/tests/queries/0_stateless/01717_global_with_subquery_fix.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01717_global_with_subquery_fix.sql b/tests/queries/0_stateless/01717_global_with_subquery_fix.sql new file mode 100644 index 00000000000..14c4ac3e4ca --- /dev/null +++ b/tests/queries/0_stateless/01717_global_with_subquery_fix.sql @@ -0,0 +1 @@ +WITH (SELECT count(distinct colU) from tabA) AS withA, (SELECT count(distinct colU) from tabA) AS withB SELECT withA / withB AS ratio FROM (SELECT date AS period, colX FROM (SELECT date, if(colA IN (SELECT colB FROM tabC), 0, colA) AS colX FROM tabB) AS tempB GROUP BY period, colX) AS main; -- {serverError 60} From 48d38e497871556ce6bf3de2b18f8140a5474dbd Mon Sep 17 00:00:00 2001 From: damozhaeva <68770561+damozhaeva@users.noreply.github.com> Date: Sun, 14 Feb 2021 14:19:26 +0300 Subject: [PATCH 17/30] Update docs/ru/operations/settings/settings.md Co-authored-by: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> --- docs/ru/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 716345a9560..bd7fa97db5d 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1985,7 +1985,7 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; ## output_format_pretty_grid_charset {#output-format-pretty-grid-charset} -ППозволяет изменить кодировку, которая используется для рисования таблицы при выводе результатов запросов. Доступны следующие кодировки: UTF-8, ASCII. +Позволяет изменить кодировку, которая используется для отрисовки таблицы при выводе результатов запросов. Доступны следующие кодировки: UTF-8, ASCII. **Пример** From 7f9436381f175eae6326bc7ddc9970f31849e499 Mon Sep 17 00:00:00 2001 From: Daria Mozhaeva Date: Sun, 14 Feb 2021 14:48:26 +0300 Subject: [PATCH 18/30] fixed conflict --- docs/en/operations/settings/settings.md | 79 +------------------------ 1 file changed, 1 insertion(+), 78 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c91ed1f2400..50108531310 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -311,12 +311,8 @@ Enables or disables parsing enum values as enum ids for TSV input format. Possible values: -<<<<<<< HEAD -Enables or disables using default values if input data contain `NULL`, but the data type of the corresponding column in not `Nullable(T)` (for text input formats). -======= - 0 — Enum values are parsed as values. -- 1 — Enum values are parsed as enum IDs ->>>>>>> 6325b15a63335e2efd7de1ae92d2907493a07a9c +- 1 — Enum values are parsed as enum IDs. Default value: 0. @@ -1318,15 +1314,7 @@ See also: ## insert_quorum_timeout {#settings-insert_quorum_timeout} -<<<<<<< HEAD -<<<<<<< HEAD -Write to a quorum timeout in seconds. If the timeout has passed and no write has taken place yet, ClickHouse will generate an exception and the client must repeat the query to write the same block to the same or any other replica. -======= Write to a quorum timeout in milliseconds. If the timeout has passed and no write has taken place yet, ClickHouse will generate an exception and the client must repeat the query to write the same block to the same or any other replica. ->>>>>>> 6325b15a63335e2efd7de1ae92d2907493a07a9c -======= -Write to a quorum timeout in milliseconds. If the timeout has passed and no write has taken place yet, ClickHouse will generate an exception and the client must repeat the query to write the same block to the same or any other replica. ->>>>>>> 547db452d63ba42b88e82cbe9a2aa1f5c683403f Default value: 600 000 milliseconds (ten minutes). @@ -1625,11 +1613,7 @@ Default value: 0. - Type: seconds - Default value: 60 seconds -<<<<<<< HEAD -Controls how fast errors in distributed tables are zeroed. If a replica is unavailable for some time, accumulates 5 errors, and distributed\_replica\_error\_half\_life is set to 1 second, then the replica is considered normal 3 seconds after the last error. -======= Controls how fast errors in distributed tables are zeroed. If a replica is unavailable for some time, accumulates 5 errors, and distributed_replica_error_half_life is set to 1 second, then the replica is considered normal 3 seconds after the last error. ->>>>>>> 6325b15a63335e2efd7de1ae92d2907493a07a9c See also: @@ -1875,8 +1859,6 @@ Default value: `0`. - [Distributed Table Engine](../../engines/table-engines/special/distributed.md#distributed) - [Managing Distributed Tables](../../sql-reference/statements/system.md#query-language-system-distributed) -<<<<<<< HEAD -======= ## insert_distributed_one_random_shard {#insert_distributed_one_random_shard} Enables or disables random shard insertion into a [Distributed](../../engines/table-engines/special/distributed.md#distributed) table when there is no distributed key. @@ -1907,7 +1889,6 @@ Default value: `1`. ## background_buffer_flush_schedule_pool_size {#background_buffer_flush_schedule_pool_size} ->>>>>>> 6325b15a63335e2efd7de1ae92d2907493a07a9c Sets the number of threads performing background flush in [Buffer](../../engines/table-engines/special/buffer.md)-engine tables. This setting is applied at the ClickHouse server start and can’t be changed in a user session. Possible values: @@ -2224,11 +2205,7 @@ Default value: `0`. ## lock_acquire_timeout {#lock_acquire_timeout} -<<<<<<< HEAD -Defines how many seconds a locking request waits before failing. -======= Defines how many seconds a locking request waits before failing. ->>>>>>> 6325b15a63335e2efd7de1ae92d2907493a07a9c Locking timeout is used to protect from deadlocks while executing read/write operations with tables. When the timeout expires and the locking request fails, the ClickHouse server throws an exception "Locking attempt timed out! Possible deadlock avoided. Client should retry." with error code `DEADLOCK_AVOIDED`. @@ -2615,58 +2592,4 @@ Possible values: Default value: `16`. -## optimize_on_insert {#optimize-on-insert} - -Enables or disables data transformation before the insertion, as if merge was done on this block (according to table engine). - -Possible values: - -- 0 — Disabled. -- 1 — Enabled. - -Default value: 1. - -**Example** - -The difference between enabled and disabled: - -Query: - -```sql -SET optimize_on_insert = 1; - -CREATE TABLE test1 (`FirstTable` UInt32) ENGINE = ReplacingMergeTree ORDER BY FirstTable; - -INSERT INTO test1 SELECT number % 2 FROM numbers(5); - -SELECT * FROM test1; - -SET optimize_on_insert = 0; - -CREATE TABLE test2 (`SecondTable` UInt32) ENGINE = ReplacingMergeTree ORDER BY SecondTable; - -INSERT INTO test2 SELECT number % 2 FROM numbers(5); - -SELECT * FROM test2; -``` - -Result: - -``` text -┌─FirstTable─┐ -│ 0 │ -│ 1 │ -└────────────┘ - -┌─SecondTable─┐ -│ 0 │ -│ 0 │ -│ 0 │ -│ 1 │ -│ 1 │ -└─────────────┘ -``` - -Note that this setting influences [Materialized view](../../sql-reference/statements/create/view.md#materialized) and [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md) behaviour. - [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) From 00bb72735eef1b11f406a8e139d4667d8c7e8b4d Mon Sep 17 00:00:00 2001 From: Daria Mozhaeva Date: Sun, 14 Feb 2021 15:55:40 +0300 Subject: [PATCH 19/30] add text --- docs/en/operations/settings/settings.md | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 50108531310..40a68491682 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2592,4 +2592,58 @@ Possible values: Default value: `16`. +## optimize_on_insert {#optimize-on-insert} + +Enables or disables data transformation before the insertion, as if merge was done on this block (according to table engine). + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: 1. + +**Example** + +The difference between enabled and disabled: + +Query: + +```sql +SET optimize_on_insert = 1; + +CREATE TABLE test1 (`FirstTable` UInt32) ENGINE = ReplacingMergeTree ORDER BY FirstTable; + +INSERT INTO test1 SELECT number % 2 FROM numbers(5); + +SELECT * FROM test1; + +SET optimize_on_insert = 0; + +CREATE TABLE test2 (`SecondTable` UInt32) ENGINE = ReplacingMergeTree ORDER BY SecondTable; + +INSERT INTO test2 SELECT number % 2 FROM numbers(5); + +SELECT * FROM test2; +``` + +Result: + +``` text +┌─FirstTable─┐ +│ 0 │ +│ 1 │ +└────────────┘ + +┌─SecondTable─┐ +│ 0 │ +│ 0 │ +│ 0 │ +│ 1 │ +│ 1 │ +└─────────────┘ +``` + +Note that this setting influences [Materialized view](../../sql-reference/statements/create/view.md#materialized) and [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md) behaviour. + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) From c5b96a522e29b38eae3f6c2d945540dd234e3c34 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Feb 2021 04:57:34 +0300 Subject: [PATCH 20/30] Add a test for #8654 --- tests/queries/0_stateless/01718_subtract_seconds_date.reference | 2 ++ tests/queries/0_stateless/01718_subtract_seconds_date.sql | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 tests/queries/0_stateless/01718_subtract_seconds_date.reference create mode 100644 tests/queries/0_stateless/01718_subtract_seconds_date.sql diff --git a/tests/queries/0_stateless/01718_subtract_seconds_date.reference b/tests/queries/0_stateless/01718_subtract_seconds_date.reference new file mode 100644 index 00000000000..97e3da8cc48 --- /dev/null +++ b/tests/queries/0_stateless/01718_subtract_seconds_date.reference @@ -0,0 +1,2 @@ +2021-02-14 23:59:59 +10 diff --git a/tests/queries/0_stateless/01718_subtract_seconds_date.sql b/tests/queries/0_stateless/01718_subtract_seconds_date.sql new file mode 100644 index 00000000000..6bffcd4db5a --- /dev/null +++ b/tests/queries/0_stateless/01718_subtract_seconds_date.sql @@ -0,0 +1,2 @@ +SELECT subtractSeconds(toDate('2021-02-15'), 1); +SELECT subtractSeconds(today(), 1) - subtractSeconds(today(), 11); From c1550814ca770a0ecb9aec0de8eeb77dee266ca4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Feb 2021 10:30:08 +0300 Subject: [PATCH 21/30] Disable snapshots for tests --- src/Coordination/NuKeeperStateMachine.cpp | 2 ++ tests/config/config.d/test_keeper_port.xml | 2 ++ tests/queries/skip_list.json | 14 ++------------ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Coordination/NuKeeperStateMachine.cpp b/src/Coordination/NuKeeperStateMachine.cpp index d282f57ce73..0061645c75c 100644 --- a/src/Coordination/NuKeeperStateMachine.cpp +++ b/src/Coordination/NuKeeperStateMachine.cpp @@ -166,6 +166,8 @@ void NuKeeperStateMachine::create_snapshot( } } + + LOG_DEBUG(log, "Created snapshot {}", s.get_last_log_idx()); nuraft::ptr except(nullptr); bool ret = true; when_done(ret, except); diff --git a/tests/config/config.d/test_keeper_port.xml b/tests/config/config.d/test_keeper_port.xml index 6ca00a972d4..97c6d7c2e33 100644 --- a/tests/config/config.d/test_keeper_port.xml +++ b/tests/config/config.d/test_keeper_port.xml @@ -6,6 +6,8 @@ 10000 30000 + 0 + 0 diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index ee25bee6a0a..e4e7504ba41 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -10,7 +10,6 @@ "00152_insert_different_granularity", "00151_replace_partition_with_different_granularity", "00157_cache_dictionary", - "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01193_metadata_loading", "01473_event_time_microseconds", "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers @@ -26,7 +25,6 @@ "memory_profiler", "odbc_roundtrip", "01103_check_cpu_instructions_at_startup", - "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01473_event_time_microseconds", "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers "01193_metadata_loading" @@ -37,7 +35,6 @@ "memory_profiler", "01103_check_cpu_instructions_at_startup", "00900_orc_load", - "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01473_event_time_microseconds", "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers "01193_metadata_loading" @@ -49,7 +46,6 @@ "01103_check_cpu_instructions_at_startup", "01086_odbc_roundtrip", /// can't pass because odbc libraries are not instrumented "00877_memory_limit_for_new_delete", /// memory limits don't work correctly under msan because it replaces malloc/free - "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01473_event_time_microseconds", "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers "01193_metadata_loading" @@ -61,7 +57,6 @@ "00980_alter_settings_race", "00834_kill_mutation_replicated_zookeeper", "00834_kill_mutation", - "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01200_mutations_memory_consumption", "01103_check_cpu_instructions_at_startup", "01037_polygon_dicts_", @@ -87,7 +82,6 @@ "00505_secure", "00505_shard_secure", "odbc_roundtrip", - "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "01103_check_cpu_instructions_at_startup", "01114_mysql_database_engine_segfault", "00834_cancel_http_readonly_queries_on_client_close", @@ -101,19 +95,16 @@ "01455_time_zones" ], "release-build": [ - "00992_system_parts_race_condition_zookeeper" /// TODO remove me (alesapin) ], "database-ordinary": [ "00604_show_create_database", "00609_mv_index_in_in", "00510_materizlized_view_and_deduplication_zookeeper", - "00738_lock_for_inner_table", - "00992_system_parts_race_condition_zookeeper" /// TODO remove me (alesapin) + "00738_lock_for_inner_table" ], "polymorphic-parts": [ "01508_partition_pruning_long", /// bug, shoud be fixed - "01482_move_to_prewhere_and_cast", /// bug, shoud be fixed - "00992_system_parts_race_condition_zookeeper" /// TODO remove me (alesapin) + "01482_move_to_prewhere_and_cast" /// bug, shoud be fixed ], "antlr": [ "00186_very_long_arrays", @@ -153,7 +144,6 @@ "00982_array_enumerate_uniq_ranked", "00984_materialized_view_to_columns", "00988_constraints_replication_zookeeper", - "00992_system_parts_race_condition_zookeeper", /// TODO remove me (alesapin) "00995_order_by_with_fill", "01001_enums_in_in_section", "01011_group_uniq_array_memsan", From 9686649b0229cc4f492dbf646d6342d587f02657 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Feb 2021 12:42:50 +0300 Subject: [PATCH 22/30] Fix non-zero session reconnect in integration test --- tests/integration/test_testkeeper_back_to_back/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_testkeeper_back_to_back/test.py b/tests/integration/test_testkeeper_back_to_back/test.py index 0f2c1ed19a5..8ec54f1a883 100644 --- a/tests/integration/test_testkeeper_back_to_back/test.py +++ b/tests/integration/test_testkeeper_back_to_back/test.py @@ -29,8 +29,8 @@ def get_fake_zk(): def reset_last_zxid_listener(state): print("Fake zk callback called for state", state) global _fake_zk_instance - # reset last_zxid -- fake server doesn't support it - _fake_zk_instance.last_zxid = 0 + if state != KazooState.CONNECTED: + _fake_zk_instance._reset() _fake_zk_instance.add_listener(reset_last_zxid_listener) _fake_zk_instance.start() From ac476ad83e526d8afec591189f10c5933edf68e7 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 15 Feb 2021 14:27:16 +0300 Subject: [PATCH 23/30] done --- .../1_stateful/00158_cache_dictionary_has.reference | 6 +++--- tests/queries/1_stateful/00158_cache_dictionary_has.sql | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/queries/1_stateful/00158_cache_dictionary_has.reference b/tests/queries/1_stateful/00158_cache_dictionary_has.reference index f8d5cd4f53d..ad4bce6bec5 100644 --- a/tests/queries/1_stateful/00158_cache_dictionary_has.reference +++ b/tests/queries/1_stateful/00158_cache_dictionary_has.reference @@ -1,6 +1,6 @@ +100 6410 -6410 -25323 +100 25323 -1774655 +100 1774655 diff --git a/tests/queries/1_stateful/00158_cache_dictionary_has.sql b/tests/queries/1_stateful/00158_cache_dictionary_has.sql index 063e7843fd4..8461728c58e 100644 --- a/tests/queries/1_stateful/00158_cache_dictionary_has.sql +++ b/tests/queries/1_stateful/00158_cache_dictionary_has.sql @@ -6,15 +6,15 @@ CREATE DICTIONARY db_dict.cache_hits PRIMARY KEY WatchID SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'hits' PASSWORD '' DB 'test')) LIFETIME(MIN 300 MAX 600) -LAYOUT(CACHE(SIZE_IN_CELLS 100000 QUERY_WAIT_TIMEOUT_MILLISECONDS 600000)); +LAYOUT(CACHE(SIZE_IN_CELLS 100 QUERY_WAIT_TIMEOUT_MILLISECONDS 600000)); -SELECT sum(flag) FROM (SELECT dictHas('db_dict.cache_hits', toUInt64(WatchID)) as flag FROM test.hits PREWHERE WatchID % 1400 == 0); +SELECT sum(flag) FROM (SELECT dictHas('db_dict.cache_hits', toUInt64(WatchID)) as flag FROM test.hits PREWHERE WatchID % 1400 == 0 LIMIT 100); SELECT count() from test.hits PREWHERE WatchID % 1400 == 0; -SELECT sum(flag) FROM (SELECT dictHas('db_dict.cache_hits', toUInt64(WatchID)) as flag FROM test.hits PREWHERE WatchID % 350 == 0); +SELECT sum(flag) FROM (SELECT dictHas('db_dict.cache_hits', toUInt64(WatchID)) as flag FROM test.hits PREWHERE WatchID % 350 == 0 LIMIT 100); SELECT count() from test.hits PREWHERE WatchID % 350 == 0; -SELECT sum(flag) FROM (SELECT dictHas('db_dict.cache_hits', toUInt64(WatchID)) as flag FROM test.hits PREWHERE WatchID % 5 == 0); +SELECT sum(flag) FROM (SELECT dictHas('db_dict.cache_hits', toUInt64(WatchID)) as flag FROM test.hits PREWHERE WatchID % 5 == 0 LIMIT 100); SELECT count() from test.hits PREWHERE WatchID % 5 == 0; DROP DICTIONARY IF EXISTS db_dict.cache_hits; From d615b8e516569ddf69ad92cd3b73f6591c0b7248 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 15 Feb 2021 16:10:14 +0300 Subject: [PATCH 24/30] more checks (cherry picked from commit b45168ecaf37d0061edfd12c67a8c5300d45d2e3) --- src/Formats/JSONEachRowUtils.cpp | 11 ++++++++--- src/IO/BufferWithOwnMemory.h | 6 +++--- src/Processors/Formats/Impl/CSVRowInputFormat.cpp | 13 +++++++++---- .../Formats/Impl/RegexpRowInputFormat.cpp | 5 ++++- .../Formats/Impl/TabSeparatedRowInputFormat.cpp | 8 +++++--- 5 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/Formats/JSONEachRowUtils.cpp b/src/Formats/JSONEachRowUtils.cpp index 56bef9e09ea..28ba625d9fb 100644 --- a/src/Formats/JSONEachRowUtils.cpp +++ b/src/Formats/JSONEachRowUtils.cpp @@ -6,6 +6,7 @@ namespace DB namespace ErrorCodes { extern const int INCORRECT_DATA; + extern const int LOGICAL_ERROR; } std::pair fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, DB::Memory<> & memory, size_t min_chunk_size) @@ -28,7 +29,9 @@ std::pair fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, D if (quotes) { pos = find_first_symbols<'\\', '"'>(pos, in.buffer().end()); - if (pos == in.buffer().end()) + if (pos > in.buffer().end()) + throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); + else if (pos == in.buffer().end()) continue; if (*pos == '\\') { @@ -45,9 +48,11 @@ std::pair fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, D else { pos = find_first_symbols<'{', '}', '\\', '"'>(pos, in.buffer().end()); - if (pos == in.buffer().end()) + if (pos > in.buffer().end()) + throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); + else if (pos == in.buffer().end()) continue; - if (*pos == '{') + else if (*pos == '{') { ++balance; ++pos; diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index 782eea84ed7..f8cc8b7febb 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -35,10 +35,10 @@ struct Memory : boost::noncopyable, Allocator char * m_data = nullptr; size_t alignment = 0; - Memory() {} + Memory() = default; /// If alignment != 0, then allocate memory aligned to specified value. - Memory(size_t size_, size_t alignment_ = 0) : m_capacity(size_), m_size(m_capacity), alignment(alignment_) + explicit Memory(size_t size_, size_t alignment_ = 0) : m_capacity(size_), m_size(m_capacity), alignment(alignment_) { alloc(); } @@ -140,7 +140,7 @@ protected: Memory<> memory; public: /// If non-nullptr 'existing_memory' is passed, then buffer will not create its own memory and will use existing_memory without ownership. - BufferWithOwnMemory(size_t size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0) + explicit BufferWithOwnMemory(size_t size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0) : Base(nullptr, 0), memory(existing_memory ? 0 : size, alignment) { Base::set(existing_memory ? existing_memory : memory.data(), size); diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index 8422f09e364..f7f08411dfa 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -15,6 +15,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int INCORRECT_DATA; + extern const int LOGICAL_ERROR; } @@ -436,9 +437,11 @@ static std::pair fileSegmentationEngineCSVImpl(ReadBuffer & in, DB if (quotes) { pos = find_first_symbols<'"'>(pos, in.buffer().end()); - if (pos == in.buffer().end()) + if (pos > in.buffer().end()) + throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); + else if (pos == in.buffer().end()) continue; - if (*pos == '"') + else if (*pos == '"') { ++pos; if (loadAtPosition(in, memory, pos) && *pos == '"') @@ -450,9 +453,11 @@ static std::pair fileSegmentationEngineCSVImpl(ReadBuffer & in, DB else { pos = find_first_symbols<'"', '\r', '\n'>(pos, in.buffer().end()); - if (pos == in.buffer().end()) + if (pos > in.buffer().end()) + throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); + else if (pos == in.buffer().end()) continue; - if (*pos == '"') + else if (*pos == '"') { quotes = true; ++pos; diff --git a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp index 6e14a1dc3c8..108f4d9d321 100644 --- a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp @@ -11,6 +11,7 @@ namespace ErrorCodes { extern const int INCORRECT_DATA; extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; } RegexpRowInputFormat::RegexpRowInputFormat( @@ -182,7 +183,9 @@ static std::pair fileSegmentationEngineRegexpImpl(ReadBuffer & in, while (loadAtPosition(in, memory, pos) && need_more_data) { pos = find_first_symbols<'\n', '\r'>(pos, in.buffer().end()); - if (pos == in.buffer().end()) + if (pos > in.buffer().end()) + throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); + else if (pos == in.buffer().end()) continue; // Support DOS-style newline ("\r\n") diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 69a5e61caf2..96b01a5bd9b 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -15,6 +15,7 @@ namespace DB namespace ErrorCodes { extern const int INCORRECT_DATA; + extern const int LOGICAL_ERROR; } @@ -433,10 +434,11 @@ static std::pair fileSegmentationEngineTabSeparatedImpl(ReadBuffer { pos = find_first_symbols<'\\', '\r', '\n'>(pos, in.buffer().end()); - if (pos == in.buffer().end()) + if (pos > in.buffer().end()) + throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); + else if (pos == in.buffer().end()) continue; - - if (*pos == '\\') + else if (*pos == '\\') { ++pos; if (loadAtPosition(in, memory, pos)) From d08dcb1958a565ad62d2e688413c3942c20e91f6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 15 Feb 2021 22:35:49 +0300 Subject: [PATCH 25/30] Update docs/en/operations/settings/settings.md --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f64c623415b..963f9fa18bd 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2598,7 +2598,7 @@ Sets the probability that the ClickHouse can start a trace for executed queries Possible values: -- 0 — The trace for a executed queries is disabled (if no parent trace context is supplied). +- 0 — The trace for all executed queries is disabled (if no parent trace context is supplied). - Positive floating-point number in the range [0..1]. For example, if the setting value is `0,5`, ClickHouse can start a trace on average for half of the queries. - 1 — The trace for all executed queries is enabled. From 7f21a216941ae6557e8ac5f75d9093635ec71919 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 15 Feb 2021 22:40:55 +0300 Subject: [PATCH 26/30] Update index.md --- docs/en/sql-reference/window-functions/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md index 46f7ed3824e..07a7f2f6978 100644 --- a/docs/en/sql-reference/window-functions/index.md +++ b/docs/en/sql-reference/window-functions/index.md @@ -15,6 +15,7 @@ ClickHouse supports the standard grammar for defining windows and window functio | Feature | Support or workaround | | --------| ----------| | ad hoc window specification (`count(*) over (partition by id order by time desc)`) | yes | +| expressions involving window functions, e.g. `(count(*) over ()) / 2)` | not supported, wrap in a subquery ([feature request](https://github.com/ClickHouse/ClickHouse/issues/19857)) | | `WINDOW` clause (`select ... from table window w as (partiton by id)`) | yes | | `ROWS` frame | yes | | `RANGE` frame | yes, it is the default | From 2de6d550cc04d62c8189ca225c4016efe8c1847a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 15 Feb 2021 22:42:10 +0300 Subject: [PATCH 27/30] Update index.md --- docs/en/sql-reference/window-functions/index.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md index 07a7f2f6978..0a19b4a8da4 100644 --- a/docs/en/sql-reference/window-functions/index.md +++ b/docs/en/sql-reference/window-functions/index.md @@ -14,15 +14,15 @@ ClickHouse supports the standard grammar for defining windows and window functio | Feature | Support or workaround | | --------| ----------| -| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | yes | +| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | supported | | expressions involving window functions, e.g. `(count(*) over ()) / 2)` | not supported, wrap in a subquery ([feature request](https://github.com/ClickHouse/ClickHouse/issues/19857)) | -| `WINDOW` clause (`select ... from table window w as (partiton by id)`) | yes | -| `ROWS` frame | yes | -| `RANGE` frame | yes, it is the default | -| `GROUPS` frame | no | +| `WINDOW` clause (`select ... from table window w as (partiton by id)`) | supported | +| `ROWS` frame | supported | +| `RANGE` frame | supported, the default | +| `GROUPS` frame | not supported | | Calculating aggregate functions over a frame (`sum(value) over (order by time)`) | all aggregate functions are supported | -| `rank()`, `dense_rank()`, `row_number()` | yes | -| `lag/lead(value, offset)` | no, replace with `any(value) over (.... rows between preceding and preceding)`, or `following` for `lead`| +| `rank()`, `dense_rank()`, `row_number()` | supported | +| `lag/lead(value, offset)` | not supported, replace with `any(value) over (.... rows between preceding and preceding)`, or `following` for `lead`| ## References From c9dd1aa58b831835a801bb886c77ccc712febcd9 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 15 Feb 2021 22:56:26 +0300 Subject: [PATCH 28/30] Update index.md --- docs/en/sql-reference/window-functions/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md index 0a19b4a8da4..cbf03a44d46 100644 --- a/docs/en/sql-reference/window-functions/index.md +++ b/docs/en/sql-reference/window-functions/index.md @@ -19,6 +19,7 @@ ClickHouse supports the standard grammar for defining windows and window functio | `WINDOW` clause (`select ... from table window w as (partiton by id)`) | supported | | `ROWS` frame | supported | | `RANGE` frame | supported, the default | +| `INTERVAL` syntax for `DateTime` `RANGE OFFSET` frame | not supported, specify the number of seconds instead | | `GROUPS` frame | not supported | | Calculating aggregate functions over a frame (`sum(value) over (order by time)`) | all aggregate functions are supported | | `rank()`, `dense_rank()`, `row_number()` | supported | From e485d4cad8e21e721ad250f9117b5717a6d64fd7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 16 Feb 2021 09:27:48 +0300 Subject: [PATCH 29/30] Fix SIGSEGV on Unknown packet for Distributed queries On Unknown packet disconnect() will be called, which will reset the input stream, so no need to call setAsyncCallback(): [ 42015 ] {} BaseDaemon: (version 21.3.1.1, build id: 4F9644AF560F6BB6) (from thread 45051) (no query) Received signal Segmentation fault (11) [ 42015 ] {} BaseDaemon: Address: 0x90 Access: read. Address not mapped to object. [ 42015 ] {} BaseDaemon: Stack trace: 0xf82e0f4 0xf82de19 0xf83b9a5 0xf83c0e0 0xe9a6fa7 0xf95016c 0xf950939 0xf95020c 0xf950939 0xf95020c 0xf950939 0xf95020c 0xf9508f9 0xf953e40 0xf958376 0x88056af 0x8809143 0x7f4b3e1aaf27 0x7f4b3e0dc31f [ 42015 ] {} BaseDaemon: 2. ext::basic_scope_guard)::$_3>::~basic_scope_guard() @ 0xf82e0f4 in /usr/lib/debug/.build-id/4f/9644af560f6bb6.debug [ 42015 ] {} BaseDaemon: 3. DB::Connection::receivePacket(std::__1::function) @ 0xf82de19 in /usr/lib/debug/.build-id/4f/9644af560f6bb6.debug [ 42015 ] {} BaseDaemon: 4. DB::MultiplexedConnections::receivePacketUnlocked(std::__1::function) @ 0xf83b9a5 in /usr/lib/debug/.build-id/4f/9644af560f6bb6.debug [ 42015 ] {} BaseDaemon: 5. DB::MultiplexedConnections::drain() @ 0xf83c0e0 in /usr/lib/debug/.build-id/4f/9644af560f6bb6.debug [ 42015 ] {} BaseDaemon: 6. DB::RemoteQueryExecutor::finish(std::__1::unique_ptr >*) @ 0xe9a6fa7 in /usr/lib/debug/.build-id/4f/9644af560f6bb6.debug [ 42015 ] {} BaseDaemon: 7. DB::PipelineExecutor::tryAddProcessorToStackIfUpdated() @ 0xf95016c in /usr/lib/debug/.build-id/4f/9644af560f6bb6.debug ... --- src/Client/Connection.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index e38a6b240a6..164b9565633 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -756,7 +756,11 @@ std::optional Connection::checkPacket(size_t timeout_microseconds) Packet Connection::receivePacket(std::function async_callback) { in->setAsyncCallback(std::move(async_callback)); - SCOPE_EXIT(in->setAsyncCallback({})); + SCOPE_EXIT({ + /// disconnect() will reset "in". + if (in) + in->setAsyncCallback({}); + }); try { From 0b0b481825ba2e71074823d2d0bbce043e6e9b4f Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 16 Feb 2021 14:02:33 +0300 Subject: [PATCH 30/30] DOCSUP-5602: Edited and translated to russian (#20302) * Edited and added translation * Minor fixes * Fix typo Co-authored-by: George Co-authored-by: Ivan <5627721+abyss7@users.noreply.github.com> --- .../sql-reference/functions/url-functions.md | 146 +++++++++++++++- .../sql-reference/functions/url-functions.md | 162 ++++++++++++++++++ 2 files changed, 299 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index 006542f494a..975695f40b3 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -133,10 +133,9 @@ For example: ### cutToFirstSignificantSubdomainCustom {#cuttofirstsignificantsubdomaincustom} -Same as `cutToFirstSignificantSubdomain` but accept custom TLD list name, useful if: +Returns the part of the domain that includes top-level subdomains up to the first significant subdomain. Accepts custom [TLD list](https://en.wikipedia.org/wiki/List_of_Internet_top-level_domains) name. -- you need fresh TLD list, -- or you have custom. +Can be useful if you need fresh TLD list or you have custom. Configuration example: @@ -149,21 +148,150 @@ Configuration example: ``` -Example: +**Syntax** -- `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/', 'public_suffix_list') = 'yandex.com.tr'`. +``` sql +cutToFirstSignificantSubdomain(URL, TLD) +``` + +**Parameters** + +- `URL` — URL. [String](../../sql-reference/data-types/string.md). +- `TLD` — Custom TLD list name. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- Part of the domain that includes top-level subdomains up to the first significant subdomain. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +```sql +SELECT cutToFirstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list'); +``` + +Result: + +```text +┌─cutToFirstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list')─┐ +│ foo.there-is-no-such-domain │ +└───────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +**See Also** + +- [firstSignificantSubdomain](#firstsignificantsubdomain). ### cutToFirstSignificantSubdomainCustomWithWWW {#cuttofirstsignificantsubdomaincustomwithwww} -Same as `cutToFirstSignificantSubdomainWithWWW` but accept custom TLD list name. +Returns the part of the domain that includes top-level subdomains up to the first significant subdomain without stripping `www`. Accepts custom TLD list name. + +Can be useful if you need fresh TLD list or you have custom. + +Configuration example: + +```xml + + + + public_suffix_list.dat + + +``` + +**Syntax** + +```sql +cutToFirstSignificantSubdomainCustomWithWWW(URL, TLD) +``` + +**Parameters** + +- `URL` — URL. [String](../../sql-reference/data-types/string.md). +- `TLD` — Custom TLD list name. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- Part of the domain that includes top-level subdomains up to the first significant subdomain without stripping `www`. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +```sql +SELECT cutToFirstSignificantSubdomainCustomWithWWW('www.foo', 'public_suffix_list'); +``` + +Result: + +```text +┌─cutToFirstSignificantSubdomainCustomWithWWW('www.foo', 'public_suffix_list')─┐ +│ www.foo │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` + +**See Also** + +- [firstSignificantSubdomain](#firstsignificantsubdomain). ### firstSignificantSubdomainCustom {#firstsignificantsubdomaincustom} -Same as `firstSignificantSubdomain` but accept custom TLD list name. +Returns the first significant subdomain. Accepts customs TLD list name. -### cutToFirstSignificantSubdomainCustomWithWWW {#cuttofirstsignificantsubdomaincustomwithwww} +Can be useful if you need fresh TLD list or you have custom. -Same as `cutToFirstSignificantSubdomainWithWWW` but accept custom TLD list name. +Configuration example: + +```xml + + + + public_suffix_list.dat + + +``` + +**Syntax** + +```sql +firstSignificantSubdomainCustom(URL, TLD) +``` + +**Parameters** + +- `URL` — URL. [String](../../sql-reference/data-types/string.md). +- `TLD` — Custom TLD list name. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- First significant subdomain. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +```sql +SELECT firstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list'); +``` + +Result: + +```text +┌─firstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list')─┐ +│ foo │ +└──────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +**See Also** + +- [firstSignificantSubdomain](#firstsignificantsubdomain). ### port(URL\[, default_port = 0\]) {#port} diff --git a/docs/ru/sql-reference/functions/url-functions.md b/docs/ru/sql-reference/functions/url-functions.md index 1008e2a359c..7541e16bed4 100644 --- a/docs/ru/sql-reference/functions/url-functions.md +++ b/docs/ru/sql-reference/functions/url-functions.md @@ -115,6 +115,168 @@ SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk') Например, `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`. +### cutToFirstSignificantSubdomainCustom {#cuttofirstsignificantsubdomaincustom} + +Возвращает часть домена, включающую поддомены верхнего уровня до первого существенного поддомена. Принимает имя пользовательского [списка доменов верхнего уровня](https://ru.wikipedia.org/wiki/Список_доменов_верхнего_уровня). + +Полезно, если требуется актуальный список доменов верхнего уровня или если есть пользовательский. + +Пример конфигурации: + +```xml + + + + public_suffix_list.dat + + +``` + +**Синтаксис** + +``` sql +cutToFirstSignificantSubdomain(URL, TLD) +``` + +**Parameters** + +- `URL` — URL. [String](../../sql-reference/data-types/string.md). +- `TLD` — имя пользовательского списка доменов верхнего уровня. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Часть домена, включающая поддомены верхнего уровня до первого существенного поддомена. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Запрос: + +```sql +SELECT cutToFirstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list'); +``` + +Результат: + +```text +┌─cutToFirstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list')─┐ +│ foo.there-is-no-such-domain │ +└───────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +**Смотрите также** + +- [firstSignificantSubdomain](#firstsignificantsubdomain). + +### cutToFirstSignificantSubdomainCustomWithWWW {#cuttofirstsignificantsubdomaincustomwithwww} + +Возвращает часть домена, включающую поддомены верхнего уровня до первого существенного поддомена, не опуская "www". Принимает имя пользовательского списка доменов верхнего уровня. + +Полезно, если требуется актуальный список доменов верхнего уровня или если есть пользовательский. + +Пример конфигурации: + +```xml + + + + public_suffix_list.dat + + +``` + +**Синтаксис** + +```sql +cutToFirstSignificantSubdomainCustomWithWWW(URL, TLD) +``` + +**Параметры** + +- `URL` — URL. [String](../../sql-reference/data-types/string.md). +- `TLD` — имя пользовательского списка доменов верхнего уровня. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Часть домена, включающая поддомены верхнего уровня до первого существенного поддомена, без удаления `www`. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Запрос: + +```sql +SELECT cutToFirstSignificantSubdomainCustomWithWWW('www.foo', 'public_suffix_list'); +``` + +Результат: + +```text +┌─cutToFirstSignificantSubdomainCustomWithWWW('www.foo', 'public_suffix_list')─┐ +│ www.foo │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` + +**Смотрите также** + +- [firstSignificantSubdomain](#firstsignificantsubdomain). + +### firstSignificantSubdomainCustom {#firstsignificantsubdomaincustom} + +Возвращает первый существенный поддомен. Принимает имя пользовательского списка доменов верхнего уровня. + +Полезно, если требуется актуальный список доменов верхнего уровня или если есть пользовательский. + +Пример конфигурации: + +```xml + + + + public_suffix_list.dat + + +``` + +**Синтаксис** + +```sql +firstSignificantSubdomainCustom(URL, TLD) +``` + +**Параметры** + +- `URL` — URL. [String](../../sql-reference/data-types/string.md). +- `TLD` — имя пользовательского списка доменов верхнего уровня. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Первый существенный поддомен. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Запрос: + +```sql +SELECT firstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list'); +``` + +Результат: + +```text +┌─firstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', 'public_suffix_list')─┐ +│ foo │ +└──────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +**Смотрите также** + +- [firstSignificantSubdomain](#firstsignificantsubdomain). + ### port(URL[, default_port = 0]) {#port} Возвращает порт или значение `default_port`, если в URL-адресе нет порта (или передан невалидный URL)